mirror of
https://github.com/simstudioai/sim.git
synced 2026-10-30 19:32:46 +08:00
feat(connectors): add Bitbucket, Databricks, Google Chat, and Workday Help KB connectors (#6895)
* feat(connectors): add Bitbucket, Databricks, Google Chat, and Workday Help KB connectors
Adds four knowledge base connectors, closing the gap where Sim shipped tool
blocks for these services but could not index their content.
- Bitbucket: repository source files and pull request descriptions over the
existing Bitbucket OAuth credential
- Databricks: notebooks (Workspace API) and saved SQL queries, PAT auth
- Google Chat: spaces indexed as message transcripts, new google-chat OAuth
service under the shared Google client
- Workday Help: knowledge article versions via the public helpArticle/v1 API
* fix(connectors): second-pass validation fixes and test coverage
Adversarial re-validation of all four connectors plus a combined-change
regression audit.
- bitbucket: stop declaring incremental sync (deletion reconciliation is
disabled for incremental runs, so deleted files were never removed on the
default code configuration); drop a wasted listing round-trip after the
frontier drains; add 33 tests
- databricks: reject an explicit maxDocuments of 0, which meant unlimited;
add 34 tests
- google-chat: correct the sender displayName documentation (user auth
populates only name and type) and emit second-precision RFC-3339 in the
message filter; 15 -> 24 tests
- workday: fix a crash when maxVersions is persisted as a number; refuse a
configuration whose status filter Workday did not honor; cap the
unresolved-name error; 18 -> 27 tests
- document the Google Chat service-account omission
- docs: list all four connectors and correct the connector count
* fix(connectors): index Google Chat spaces with no messages in the window
Review round 1.
- orderBy takes a full ordering expression, not a bare direction. The reference
documents the default as `createTime ASC`, so send `createTime DESC`; a bare
`DESC` either 400s every hydration or is ignored, which would make the cap keep
the oldest traffic and the later reverse render the transcript backwards.
- getDocument no longer returns null when the message window is empty. A space
with no messages is still a live space, and null is the "document is gone"
signal the engine treats as last-known-good: returning it dropped spaces whose
only prose is their description or guidelines, and left a stale transcript
indexed after a space was cleared or lookbackDays was tightened past every
message.
- The transcript header is omitted when no message contributed text.
* fix(connectors): only flag a Bitbucket listing capped when the cap withheld something
Review round 2.
takeIndexableWithinCap reports capReached as soon as the running total equals
maxItems, which is also true of a listing that ended at exactly that count.
Setting listingCapped there suppressed deletion reconciliation for a complete
listing, so upstream-deleted files and pull requests could stay in the knowledge
base indefinitely. applyMaxItemsCap now takes whether Bitbucket had more content
beyond the page -- a next link, or directories still queued on the frontier --
and flags the listing only when the cap actually withheld something, matching the
Databricks, Google Chat, and Workday connectors.
* fix(connectors): keep the Bitbucket cap flag set when it skips the pull request phase
Review round 3. Fixes a regression from 622a21bd9a.
maxItems is shared across the code and pull request phases, so a code walk that
ends with exactly maxItems documents and no next link or frontier stops
pagination before the pull request phase runs. Scoping listingCapped to "this
phase had more" left the flag unset in that case, and the engine then treated the
run as a complete enumeration and could hard-delete previously indexed pr:*
documents that were never listed.
The cap flag now asks whether anything the connector was configured to list
remains unlisted -- including a later phase the cap is about to stop us reaching.
This commit is contained in:
Waleed
@@ -7269,6 +7269,44 @@ export function GoogleGroupsIcon(props: SVGProps<SVGSVGElement>) {
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Google Chat product mark: a hollow four-colour speech bubble with a chamfered
|
||||
* upper-right corner, traced from the official 2023 icon
|
||||
* (https://commons.wikimedia.org/wiki/File:Google_Chat_icon_(2023).svg) and
|
||||
* using the shared Google Workspace two-tone palette the other Google marks in
|
||||
* this file already carry.
|
||||
*/
|
||||
export function GoogleChatIcon(props: SVGProps<SVGSVGElement>) {
|
||||
return (
|
||||
<svg {...props} xmlns='http://www.w3.org/2000/svg' fill='none' viewBox='0 0 311 320'>
|
||||
<path
|
||||
fill='#0066da'
|
||||
d='M76.37 0.51L76.38 76.98L0 76.96L0 20.77Q0.85 14.81 3.53 10.76Q10.14 0.74 22.75 0.67Q49.41 0.53 76.37 0.51Z'
|
||||
/>
|
||||
<path
|
||||
fill='#fbbc04'
|
||||
d='M76.37 0.51L233.79 0.53A1.61 1.57 -26.7 0 1 234.71 0.82L235.08 1.09Q234.92 1.15 234.81 1.22Q234.64 1.31 234.64 1.5L234.62 77.01Q234.6 77.01 234.57 77.01L76.41 77.01Q76.4 77 76.38 76.98L76.37 0.51Z'
|
||||
/>
|
||||
<path
|
||||
fill='#ea4335'
|
||||
d='M235.08 1.09L310.53 76.77L234.62 77.01L234.64 1.5Q234.64 1.31 234.81 1.22Q234.92 1.15 235.08 1.09Z'
|
||||
/>
|
||||
<path
|
||||
fill='#2684fc'
|
||||
d='M0 76.96L76.38 76.98Q76.4 77 76.41 77.01L76.43 182.69L0 182.67L0 76.96Z'
|
||||
/>
|
||||
<path
|
||||
fill='#00ac47'
|
||||
d='M310.53 76.77L311 77.11L311 239.01Q308.34 253.54 295.94 257.78Q291.52 259.3 282.91 259.28Q227.02 259.19 169.99 259.11Q161.71 259.1 153.19 259.23Q152.72 259.24 152.39 259.57Q124.49 287.34 96.39 315.59C93.52 318.48 90.27 320.09 86.15 319.48Q80.39 318.63 77.66 313.54Q76.51 311.38 76.49 305.66Q76.42 282.47 76.44 259.13L76.43 220.85L114.21 183.07A1.79 1.77 22.3 0 1 115.47 182.55L233.77 182.59A0.83 0.83 0 0 0 234.6 181.76L234.57 77.01Q234.6 77.01 234.62 77.01L310.53 76.77Z'
|
||||
/>
|
||||
<path
|
||||
fill='#00832d'
|
||||
d='M76.43 182.69L76.43 220.85L76.44 259.13Q52.47 259.27 28.91 259.22Q19.09 259.2 14.76 257.68Q2.62 253.44 0 238.88L0 182.67L76.43 182.69Z'
|
||||
/>
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
|
||||
export function GoogleMeetIcon(props: SVGProps<SVGSVGElement>) {
|
||||
return (
|
||||
<svg {...props} xmlns='http://www.w3.org/2000/svg' fill='none' viewBox='0 0 87.5 72'>
|
||||
|
||||
@@ -14,23 +14,24 @@ Connectors continuously sync documents from external services into your knowledg
|
||||
|
||||
<Image src="/static/connectors/connectors-sources.png" alt="Connect Source picker showing a searchable list of available connectors including Airtable, Asana, Confluence, Discord, Dropbox, Fireflies, GitHub, and Gmail" width={800} height={500} />
|
||||
|
||||
Sim ships with 60 built-in connectors:
|
||||
Sim ships with 64 built-in connectors:
|
||||
|
||||
| Category | Connectors |
|
||||
|----------|-----------|
|
||||
| **Productivity** | Notion, Confluence, Asana, Linear, Jira, Jira Service Management, Monday, Trello, ClickUp, Google Calendar, Google Sheets, Google Forms, Microsoft Excel, Typeform |
|
||||
| **Cloud Storage** | Google Drive, Dropbox, OneDrive, SharePoint, Box, Amazon S3, SFTP |
|
||||
| **Documents** | Google Docs, Google Slides, Mintlify, WordPress, Webflow, DocuSign |
|
||||
| **Development** | GitHub, GitLab, Azure DevOps, Sentry |
|
||||
| **Communication** | Slack, Discord, Microsoft Teams, Reddit, X, YouTube |
|
||||
| **Development** | GitHub, GitLab, Bitbucket, Azure DevOps, Sentry |
|
||||
| **Communication** | Slack, Discord, Microsoft Teams, Google Chat, Reddit, X, YouTube |
|
||||
| **Email** | Gmail, Outlook |
|
||||
| **CRM** | HubSpot, Salesforce |
|
||||
| **Support** | Intercom, ServiceNow, Zendesk, Zoho Desk |
|
||||
| **Incident Management** | incident.io, Rootly, PagerDuty |
|
||||
| **Data** | Airtable |
|
||||
| **Data** | Airtable, Databricks |
|
||||
| **Note-taking** | Obsidian |
|
||||
| **Meetings** | Zoom, Google Meet, Gong, Grain, Granola, Fathom, Fireflies |
|
||||
| **Recruiting** | Greenhouse, Ashby |
|
||||
| **HR** | Workday Help |
|
||||
| **Compliance** | Google Vault |
|
||||
|
||||
## Adding a Connector
|
||||
@@ -58,6 +59,8 @@ Other connectors use **API keys** or **personal access tokens** instead. The set
|
||||
| **PagerDuty** | REST API key from Integrations → API Access Keys |
|
||||
| **SFTP** | Password or unencrypted private key, plus a required SHA-256 host key fingerprint (host, port, username, and root path are entered as config fields) |
|
||||
| **Mintlify** | API key — optional for public documentation sites, which sync from `llms.txt` |
|
||||
| **Databricks** | Personal access token from your workspace's user settings (the workspace host is entered as a config field) |
|
||||
| **Workday Help** | Register an API client for integrations in your tenant, then enter the client secret and refresh token together as `clientSecret:refreshToken` (the client ID, tenant host, and tenant name are entered as config fields) |
|
||||
|
||||
<Callout type="info">
|
||||
If you rotate an API key in the external service, update it in Sim as well — OAuth tokens refresh automatically, but API keys do not.
|
||||
|
||||
@@ -100,7 +100,7 @@ One OAuth client in [Google Cloud Console](https://console.cloud.google.com/apis
|
||||
|
||||
| Environment variables | Provider IDs |
|
||||
|---|---|
|
||||
| `GOOGLE_CLIENT_ID`<br />`GOOGLE_CLIENT_SECRET` | `google-email`, `google-drive`, `google-sheets`, `google-docs`, `google-calendar`, `google-contacts`, `google-forms`, `google-tasks`, `google-meet`, `google-groups`, `google-ads`, `google-bigquery`, `google-vault`, `vertex-ai` |
|
||||
| `GOOGLE_CLIENT_ID`<br />`GOOGLE_CLIENT_SECRET` | `google-email`, `google-drive`, `google-sheets`, `google-docs`, `google-calendar`, `google-contacts`, `google-forms`, `google-tasks`, `google-meet`, `google-groups`, `google-chat`, `google-ads`, `google-bigquery`, `google-vault`, `vertex-ai` |
|
||||
|
||||
The same variables also power "Sign in with Google". See [Authentication](/platform/self-hosting/authentication).
|
||||
|
||||
|
||||
@@ -7269,6 +7269,44 @@ export function GoogleGroupsIcon(props: SVGProps<SVGSVGElement>) {
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Google Chat product mark: a hollow four-colour speech bubble with a chamfered
|
||||
* upper-right corner, traced from the official 2023 icon
|
||||
* (https://commons.wikimedia.org/wiki/File:Google_Chat_icon_(2023).svg) and
|
||||
* using the shared Google Workspace two-tone palette the other Google marks in
|
||||
* this file already carry.
|
||||
*/
|
||||
export function GoogleChatIcon(props: SVGProps<SVGSVGElement>) {
|
||||
return (
|
||||
<svg {...props} xmlns='http://www.w3.org/2000/svg' fill='none' viewBox='0 0 311 320'>
|
||||
<path
|
||||
fill='#0066da'
|
||||
d='M76.37 0.51L76.38 76.98L0 76.96L0 20.77Q0.85 14.81 3.53 10.76Q10.14 0.74 22.75 0.67Q49.41 0.53 76.37 0.51Z'
|
||||
/>
|
||||
<path
|
||||
fill='#fbbc04'
|
||||
d='M76.37 0.51L233.79 0.53A1.61 1.57 -26.7 0 1 234.71 0.82L235.08 1.09Q234.92 1.15 234.81 1.22Q234.64 1.31 234.64 1.5L234.62 77.01Q234.6 77.01 234.57 77.01L76.41 77.01Q76.4 77 76.38 76.98L76.37 0.51Z'
|
||||
/>
|
||||
<path
|
||||
fill='#ea4335'
|
||||
d='M235.08 1.09L310.53 76.77L234.62 77.01L234.64 1.5Q234.64 1.31 234.81 1.22Q234.92 1.15 235.08 1.09Z'
|
||||
/>
|
||||
<path
|
||||
fill='#2684fc'
|
||||
d='M0 76.96L76.38 76.98Q76.4 77 76.41 77.01L76.43 182.69L0 182.67L0 76.96Z'
|
||||
/>
|
||||
<path
|
||||
fill='#00ac47'
|
||||
d='M310.53 76.77L311 77.11L311 239.01Q308.34 253.54 295.94 257.78Q291.52 259.3 282.91 259.28Q227.02 259.19 169.99 259.11Q161.71 259.1 153.19 259.23Q152.72 259.24 152.39 259.57Q124.49 287.34 96.39 315.59C93.52 318.48 90.27 320.09 86.15 319.48Q80.39 318.63 77.66 313.54Q76.51 311.38 76.49 305.66Q76.42 282.47 76.44 259.13L76.43 220.85L114.21 183.07A1.79 1.77 22.3 0 1 115.47 182.55L233.77 182.59A0.83 0.83 0 0 0 234.6 181.76L234.57 77.01Q234.6 77.01 234.62 77.01L310.53 76.77Z'
|
||||
/>
|
||||
<path
|
||||
fill='#00832d'
|
||||
d='M76.43 182.69L76.43 220.85L76.44 259.13Q52.47 259.27 28.91 259.22Q19.09 259.2 14.76 257.68Q2.62 253.44 0 238.88L0 182.67L76.43 182.69Z'
|
||||
/>
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
|
||||
export function GoogleMeetIcon(props: SVGProps<SVGSVGElement>) {
|
||||
return (
|
||||
<svg {...props} xmlns='http://www.w3.org/2000/svg' fill='none' viewBox='0 0 87.5 72'>
|
||||
|
||||
@@ -0,0 +1,718 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*
|
||||
* The load-bearing logic here is the repository tree walk: a `max_depth` listing
|
||||
* that Bitbucket may answer with a `555` timeout, an opaque `next` cursor that
|
||||
* cannot be re-cut, and a per-run frontier of directories the walk has not reached.
|
||||
* Every path that shortens the listing has to leave `syncContext.listingCapped` set,
|
||||
* because the sync engine hard-deletes whatever a full listing omits.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { bitbucketConnector } from '@/connectors/bitbucket/bitbucket'
|
||||
|
||||
const ACCESS_TOKEN = 'bitbucket-token'
|
||||
const COMMIT = 'e7d158ff7ed5538c28f94cd97a9ad569680fc94e'
|
||||
const CONFIG = { workspaceSlug: 'acme', repoSlug: 'widgets' }
|
||||
const PR_CONFIG = { ...CONFIG, contentTypes: 'pullrequests' }
|
||||
|
||||
const mockFetch = vi.fn()
|
||||
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
})
|
||||
}
|
||||
|
||||
const REPOSITORY = {
|
||||
full_name: 'acme/widgets',
|
||||
mainbranch: { name: 'main', target: { hash: COMMIT } },
|
||||
links: { html: { href: 'https://bitbucket.org/acme/widgets' } },
|
||||
}
|
||||
|
||||
function fileEntry(path: string, size = 10, attributes: string[] = []) {
|
||||
return { type: 'commit_file', path, size, attributes, commit: { hash: COMMIT } }
|
||||
}
|
||||
|
||||
function dirEntry(path: string) {
|
||||
return { type: 'commit_directory', path, commit: { hash: COMMIT } }
|
||||
}
|
||||
|
||||
function pullRequestFixture(id: number, overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id,
|
||||
title: `PR ${id}`,
|
||||
state: 'OPEN',
|
||||
summary: { raw: `Body of ${id}`, markup: 'markdown', html: `<p>Body of ${id}</p>` },
|
||||
author: { display_name: 'Ada Lovelace', nickname: 'ada' },
|
||||
created_on: '2026-01-01T00:00:00.000000+00:00',
|
||||
updated_on: '2026-02-01T00:00:00.000000+00:00',
|
||||
links: { html: { href: `https://bitbucket.org/acme/widgets/pull-requests/${id}` } },
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
/** Routes by URL, because the number of lookups before a listing varies by phase. */
|
||||
function mockApi(routes: Array<[RegExp, () => Response]>) {
|
||||
mockFetch.mockImplementation(async (url: string) => {
|
||||
for (const [pattern, respond] of routes) {
|
||||
if (pattern.test(url)) return respond()
|
||||
}
|
||||
if (/\/repositories\/acme\/widgets$/.test(url)) return jsonResponse(REPOSITORY)
|
||||
throw new Error(`unrouted request: ${url}`)
|
||||
})
|
||||
}
|
||||
|
||||
function requestedUrls(pattern: RegExp): string[] {
|
||||
return mockFetch.mock.calls.map(([url]) => url as string).filter((url) => pattern.test(url))
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
describe('bitbucket repository file listing', () => {
|
||||
it('lists text files as deferred stubs and drops binary, symlink, and submodule entries', async () => {
|
||||
mockApi([
|
||||
[
|
||||
/\/src\//,
|
||||
() =>
|
||||
jsonResponse({
|
||||
values: [
|
||||
fileEntry('README.md'),
|
||||
fileEntry('logo.png', 2048, ['binary']),
|
||||
fileEntry('link.txt', 5, ['link']),
|
||||
fileEntry('vendor', 40, ['subrepository']),
|
||||
fileEntry('run.sh', 12, ['executable']),
|
||||
],
|
||||
}),
|
||||
],
|
||||
])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
CONFIG,
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(result.documents.map((d) => d.externalId)).toEqual(['file:README.md', 'file:run.sh'])
|
||||
expect(result.documents[0].contentDeferred).toBe(true)
|
||||
expect(result.documents[0].content).toBe('')
|
||||
expect(result.documents[0].contentHash).toBe(`bitbucket:file:acme/widgets:${COMMIT}:README.md`)
|
||||
expect(result.hasMore).toBe(false)
|
||||
expect(syncContext.listingCapped).toBeUndefined()
|
||||
})
|
||||
|
||||
it('pins the listing to the resolved tip commit and asks for the documented depth', async () => {
|
||||
mockApi([[/\/src\//, () => jsonResponse({ values: [] })]])
|
||||
|
||||
await bitbucketConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {})
|
||||
|
||||
const url = requestedUrls(/\/src\//)[0]
|
||||
expect(url).toContain(`/src/${COMMIT}/`)
|
||||
expect(url).toContain('max_depth=5')
|
||||
expect(url).toContain('pagelen=100')
|
||||
})
|
||||
|
||||
it('applies the path prefix and extension filters to files and prunes unrelated directories', async () => {
|
||||
mockApi([
|
||||
[
|
||||
/\/src\//,
|
||||
() =>
|
||||
jsonResponse({
|
||||
values: [
|
||||
fileEntry('docs/guide.md'),
|
||||
fileEntry('docs/guide.txt'),
|
||||
fileEntry('src/index.md'),
|
||||
dirEntry('docs/a/b/c/d'),
|
||||
dirEntry('src/a/b/c/d'),
|
||||
],
|
||||
}),
|
||||
],
|
||||
])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, pathPrefix: '/docs', fileExtensions: 'md' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(result.documents.map((d) => d.externalId)).toEqual(['file:docs/guide.md'])
|
||||
expect(syncContext.pendingDirs).toEqual(['docs/a/b/c/d'])
|
||||
})
|
||||
|
||||
it('queues only the frontier directories the walk actually stopped at', async () => {
|
||||
mockApi([
|
||||
[
|
||||
/\/src\//,
|
||||
() =>
|
||||
jsonResponse({
|
||||
values: [dirEntry('a'), dirEntry('a/b'), dirEntry('a/b/c/d/e'), fileEntry('a/b/x.md')],
|
||||
}),
|
||||
],
|
||||
])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
await bitbucketConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, syncContext)
|
||||
|
||||
expect(syncContext.pendingDirs).toEqual(['a/b/c/d/e'])
|
||||
})
|
||||
|
||||
it('walks the frontier on a following page and finishes without flagging the listing', async () => {
|
||||
mockApi([
|
||||
[
|
||||
/\/src\/[a-f0-9]+\/\?/,
|
||||
() => jsonResponse({ values: [dirEntry('a/b/c/d/e'), fileEntry('root.md')] }),
|
||||
],
|
||||
[
|
||||
/\/src\/[a-f0-9]+\/a\/b\/c\/d\/e\/\?/,
|
||||
() => jsonResponse({ values: [fileEntry('a/b/c/d/e/deep.md')] }),
|
||||
],
|
||||
])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const first = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
CONFIG,
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
expect(first.hasMore).toBe(true)
|
||||
|
||||
const second = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
CONFIG,
|
||||
first.nextCursor,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(second.documents.map((d) => d.externalId)).toEqual(['file:a/b/c/d/e/deep.md'])
|
||||
expect(second.hasMore).toBe(false)
|
||||
expect(syncContext.listingCapped).toBeUndefined()
|
||||
})
|
||||
|
||||
it('replays the opaque next link verbatim instead of rebuilding it', async () => {
|
||||
const nextUrl =
|
||||
'https://api.bitbucket.org/2.0/repositories/acme/widgets/src/abc/?page=2&opaque=xyz'
|
||||
mockApi([
|
||||
[/page=2/, () => jsonResponse({ values: [fileEntry('second.md')] })],
|
||||
[/\/src\//, () => jsonResponse({ values: [fileEntry('first.md')], next: nextUrl })],
|
||||
])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const first = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
CONFIG,
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
expect(first.hasMore).toBe(true)
|
||||
|
||||
await bitbucketConnector.listDocuments(ACCESS_TOKEN, CONFIG, first.nextCursor, syncContext)
|
||||
|
||||
expect(requestedUrls(/page=2/)).toEqual([nextUrl])
|
||||
})
|
||||
|
||||
it('rejects a next cursor that does not point at the Bitbucket Cloud API', async () => {
|
||||
mockApi([
|
||||
[
|
||||
/\/src\//,
|
||||
() => jsonResponse({ values: [], next: 'https://evil.example.com/2.0/repositories/x' }),
|
||||
],
|
||||
])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const first = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
CONFIG,
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
await expect(
|
||||
bitbucketConnector.listDocuments(ACCESS_TOKEN, CONFIG, first.nextCursor, syncContext)
|
||||
).rejects.toThrow(/Bitbucket Cloud API/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('bitbucket source listing timeouts', () => {
|
||||
it('retries a first-page 555 at depth 1 and keeps the listing reconcilable', async () => {
|
||||
mockApi([
|
||||
[/max_depth=1/, () => jsonResponse({ values: [dirEntry('a'), fileEntry('root.md')] })],
|
||||
[/max_depth=5/, () => new Response('timeout', { status: 555 })],
|
||||
])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
CONFIG,
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(result.documents.map((d) => d.externalId)).toEqual(['file:root.md'])
|
||||
expect(syncContext.listingCapped).toBeUndefined()
|
||||
/** The depth-1 response only reached one level, so `a` is now unexplored. */
|
||||
expect(syncContext.pendingDirs).toEqual(['a'])
|
||||
expect(result.hasMore).toBe(true)
|
||||
})
|
||||
|
||||
it('flags the listing capped when even the depth-1 retry times out', async () => {
|
||||
mockApi([[/\/src\//, () => new Response('timeout', { status: 555 })]])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
CONFIG,
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(syncContext.listingCapped).toBe(true)
|
||||
expect(result.documents).toEqual([])
|
||||
expect(requestedUrls(/\/src\//)).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('flags the listing capped when a replayed next cursor times out, without re-cutting it', async () => {
|
||||
const nextUrl =
|
||||
'https://api.bitbucket.org/2.0/repositories/acme/widgets/src/abc/?page=2&max_depth=5'
|
||||
mockApi([
|
||||
[/page=2/, () => new Response('timeout', { status: 555 })],
|
||||
[/\/src\//, () => jsonResponse({ values: [fileEntry('first.md')], next: nextUrl })],
|
||||
])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const first = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
CONFIG,
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
const second = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
CONFIG,
|
||||
first.nextCursor,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(syncContext.listingCapped).toBe(true)
|
||||
expect(second.documents).toEqual([])
|
||||
/** Exactly one attempt: an opaque cursor is never rebuilt at another depth. */
|
||||
expect(requestedUrls(/page=2/)).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('leaves the listing reconcilable when a directory is simply absent', async () => {
|
||||
mockApi([[/\/src\//, () => jsonResponse({ type: 'error' }, 404)]])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
CONFIG,
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(syncContext.listingCapped).toBeUndefined()
|
||||
expect(result.hasMore).toBe(false)
|
||||
})
|
||||
|
||||
it('flags the listing capped when the token stops working mid-walk', async () => {
|
||||
mockApi([[/\/src\//, () => jsonResponse({ type: 'error' }, 403)]])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
await bitbucketConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, syncContext)
|
||||
|
||||
expect(syncContext.listingCapped).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('bitbucket maxItems cap', () => {
|
||||
it('flags the listing capped when the cap truncates a page', async () => {
|
||||
mockApi([
|
||||
[
|
||||
/\/src\//,
|
||||
() => jsonResponse({ values: [fileEntry('a.md'), fileEntry('b.md'), fileEntry('c.md')] }),
|
||||
],
|
||||
])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, maxItems: '2' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(result.documents).toHaveLength(2)
|
||||
expect(result.hasMore).toBe(false)
|
||||
expect(syncContext.listingCapped).toBe(true)
|
||||
})
|
||||
|
||||
it('leaves the listing reconcilable when a complete listing ends exactly on the cap', async () => {
|
||||
mockApi([[/\/src\//, () => jsonResponse({ values: [fileEntry('a.md'), fileEntry('b.md')] })]])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, maxItems: '2' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(result.documents).toHaveLength(2)
|
||||
expect(syncContext.listingCapped).toBeUndefined()
|
||||
})
|
||||
|
||||
it('flags the listing capped when the code walk ends on the cap and the pull request phase never runs', async () => {
|
||||
mockApi([[/\/src\//, () => jsonResponse({ values: [fileEntry('a.md'), fileEntry('b.md')] })]])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, contentTypes: 'all', maxItems: '2' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(result.documents).toHaveLength(2)
|
||||
expect(result.hasMore).toBe(false)
|
||||
expect(syncContext.listingCapped).toBe(true)
|
||||
})
|
||||
|
||||
it('flags the listing capped when the cap lands on a page boundary with more to come', async () => {
|
||||
mockApi([
|
||||
[
|
||||
/\/src\//,
|
||||
() =>
|
||||
jsonResponse({
|
||||
values: [fileEntry('a.md'), fileEntry('b.md')],
|
||||
next: 'https://api.bitbucket.org/2.0/repositories/acme/widgets/src/abc/?page=2',
|
||||
}),
|
||||
],
|
||||
])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, maxItems: '2' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(syncContext.listingCapped).toBe(true)
|
||||
})
|
||||
|
||||
it('leaves the listing reconcilable when no cap is configured', async () => {
|
||||
mockApi([[/\/src\//, () => jsonResponse({ values: [fileEntry('a.md'), fileEntry('b.md')] })]])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
CONFIG,
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(result.documents).toHaveLength(2)
|
||||
expect(syncContext.listingCapped).toBeUndefined()
|
||||
})
|
||||
|
||||
it('lets an oversized file ride along without consuming the cap', async () => {
|
||||
mockApi([
|
||||
[
|
||||
/\/src\//,
|
||||
() =>
|
||||
jsonResponse({
|
||||
values: [fileEntry('huge.md', 500 * 1024 * 1024), fileEntry('a.md'), fileEntry('b.md')],
|
||||
}),
|
||||
],
|
||||
])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, maxItems: '2' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(result.documents.map((d) => d.externalId)).toEqual([
|
||||
'file:huge.md',
|
||||
'file:a.md',
|
||||
'file:b.md',
|
||||
])
|
||||
expect(result.documents[0].skippedReason).toMatch(/size limit/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('bitbucket pull request listing', () => {
|
||||
it('builds the documented collection query', async () => {
|
||||
mockApi([[/\/pullrequests/, () => jsonResponse({ values: [pullRequestFixture(7)] })]])
|
||||
|
||||
await bitbucketConnector.listDocuments(ACCESS_TOKEN, PR_CONFIG, undefined, {})
|
||||
|
||||
const url = requestedUrls(/\/pullrequests/)[0]
|
||||
expect(url).toContain('pagelen=50')
|
||||
/** `+` must reach Bitbucket percent-encoded or it is read as a space. */
|
||||
expect(url).toContain('fields=%2Bvalues.summary')
|
||||
expect(url).toContain('sort=-id')
|
||||
expect(url).toContain('state=OPEN')
|
||||
})
|
||||
|
||||
it('repeats the state parameter for each requested state', async () => {
|
||||
mockApi([[/\/pullrequests/, () => jsonResponse({ values: [] })]])
|
||||
|
||||
await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...PR_CONFIG, pullRequestState: 'all' },
|
||||
undefined,
|
||||
{}
|
||||
)
|
||||
|
||||
const url = requestedUrls(/\/pullrequests/)[0]
|
||||
expect(url).toContain('state=OPEN')
|
||||
expect(url).toContain('state=MERGED')
|
||||
expect(url).toContain('state=DECLINED')
|
||||
expect(url).toContain('state=SUPERSEDED')
|
||||
})
|
||||
|
||||
it('renders the incremental BBQL filter with an explicit numeric UTC offset', async () => {
|
||||
mockApi([[/\/pullrequests/, () => jsonResponse({ values: [] })]])
|
||||
|
||||
await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
PR_CONFIG,
|
||||
undefined,
|
||||
{},
|
||||
new Date('2026-02-03T04:05:06.007Z')
|
||||
)
|
||||
|
||||
const url = new URL(requestedUrls(/\/pullrequests/)[0])
|
||||
expect(url.searchParams.get('q')).toBe('updated_on > 2026-02-03T04:05:06.007+00:00')
|
||||
})
|
||||
|
||||
it('maps the author-typed body and falls back to the rendered description', async () => {
|
||||
mockApi([
|
||||
[
|
||||
/\/pullrequests/,
|
||||
() =>
|
||||
jsonResponse({
|
||||
values: [
|
||||
pullRequestFixture(7),
|
||||
pullRequestFixture(8, {
|
||||
summary: undefined,
|
||||
rendered: { description: { raw: 'Rendered body' } },
|
||||
}),
|
||||
],
|
||||
}),
|
||||
],
|
||||
])
|
||||
|
||||
const result = await bitbucketConnector.listDocuments(ACCESS_TOKEN, PR_CONFIG, undefined, {})
|
||||
|
||||
expect(result.documents[0].content).toBe('PR 7\n\nBody of 7')
|
||||
expect(result.documents[0].contentDeferred).toBe(false)
|
||||
expect(result.documents[0].contentHash).toBe(
|
||||
'bitbucket:pr:acme/widgets:7:2026-02-01T00:00:00.000000+00:00'
|
||||
)
|
||||
expect(result.documents[1].content).toBe('PR 8\n\nRendered body')
|
||||
})
|
||||
|
||||
it('walks from the code phase into the pull request phase for combined content', async () => {
|
||||
mockApi([
|
||||
[/\/src\//, () => jsonResponse({ values: [fileEntry('a.md')] })],
|
||||
[/\/pullrequests/, () => jsonResponse({ values: [pullRequestFixture(7)] })],
|
||||
])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const first = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, contentTypes: 'all' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
expect(first.hasMore).toBe(true)
|
||||
|
||||
const second = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, contentTypes: 'all' },
|
||||
first.nextCursor,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(second.documents.map((d) => d.externalId)).toEqual(['pr:7'])
|
||||
expect(second.hasMore).toBe(false)
|
||||
expect(syncContext.listingCapped).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('bitbucket getDocument', () => {
|
||||
it('hydrates a file with the same contentHash the listing produced', async () => {
|
||||
mockApi([[/\/src\//, () => jsonResponse({ values: [fileEntry('README.md')] })]])
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const listed = await bitbucketConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
CONFIG,
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
mockApi([[/\/src\/[a-f0-9]+\/README\.md$/, () => new Response('# Widgets', { status: 200 })]])
|
||||
const fetched = await bitbucketConnector.getDocument(
|
||||
ACCESS_TOKEN,
|
||||
CONFIG,
|
||||
'file:README.md',
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(fetched?.contentHash).toBe(listed.documents[0].contentHash)
|
||||
expect(fetched?.contentDeferred).toBe(false)
|
||||
expect(fetched?.content).toBe('README.md\n\n# Widgets')
|
||||
expect(fetched?.metadata?.size).toBe(9)
|
||||
})
|
||||
|
||||
it('surfaces an LFS-managed file as skipped rather than following the media redirect', async () => {
|
||||
mockApi([
|
||||
[
|
||||
/\/src\/[a-f0-9]+\/big\.bin$/,
|
||||
() =>
|
||||
new Response(null, {
|
||||
status: 301,
|
||||
headers: { Location: 'https://media.atlassian.com/file/abc' },
|
||||
}),
|
||||
],
|
||||
])
|
||||
|
||||
const doc = await bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'file:big.bin', {})
|
||||
|
||||
expect(doc?.skippedReason).toMatch(/LFS/)
|
||||
expect(doc?.content).toBe('')
|
||||
})
|
||||
|
||||
it('surfaces a file with a NUL byte as a skipped binary', async () => {
|
||||
mockApi([
|
||||
[
|
||||
/\/src\/[a-f0-9]+\/data\.md$/,
|
||||
() => new Response(Buffer.from([0x61, 0x00, 0x62]), { status: 200 }),
|
||||
],
|
||||
])
|
||||
|
||||
const doc = await bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'file:data.md', {})
|
||||
|
||||
expect(doc?.skippedReason).toMatch(/Binary/)
|
||||
})
|
||||
|
||||
it('returns null for a file the ref no longer carries', async () => {
|
||||
mockApi([[/\/src\/[a-f0-9]+\/gone\.md$/, () => jsonResponse({ type: 'error' }, 404)]])
|
||||
|
||||
expect(
|
||||
await bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'file:gone.md', {})
|
||||
).toBeNull()
|
||||
})
|
||||
|
||||
it('rethrows a transient failure so the sync records it instead of reading it as a deletion', async () => {
|
||||
mockApi([[/\/src\/[a-f0-9]+\/flaky\.md$/, () => jsonResponse({ type: 'error' }, 500)]])
|
||||
|
||||
await expect(
|
||||
bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'file:flaky.md', {})
|
||||
).rejects.toThrow(/500/)
|
||||
})
|
||||
|
||||
it('returns null for a pull request the repository no longer has', async () => {
|
||||
mockApi([[/\/pullrequests\/9$/, () => jsonResponse({ type: 'error' }, 404)]])
|
||||
|
||||
expect(await bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'pr:9', {})).toBeNull()
|
||||
})
|
||||
|
||||
it('rejects an externalId with no recognized resource prefix', async () => {
|
||||
mockApi([])
|
||||
|
||||
expect(await bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'wiki:Home', {})).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('bitbucket tag mapping', () => {
|
||||
it('maps every declared file tag from the listing metadata', async () => {
|
||||
mockApi([[/\/src\//, () => jsonResponse({ values: [fileEntry('docs/guide.md', 42)] })]])
|
||||
|
||||
const result = await bitbucketConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {})
|
||||
const tags = bitbucketConnector.mapTags?.(result.documents[0].metadata ?? {})
|
||||
|
||||
expect(tags).toEqual({
|
||||
contentType: 'file',
|
||||
title: 'guide.md',
|
||||
repository: 'acme/widgets',
|
||||
path: 'docs/guide.md',
|
||||
size: 42,
|
||||
})
|
||||
})
|
||||
|
||||
it('maps every declared pull request tag from the listing metadata', async () => {
|
||||
mockApi([[/\/pullrequests/, () => jsonResponse({ values: [pullRequestFixture(7)] })]])
|
||||
|
||||
const result = await bitbucketConnector.listDocuments(ACCESS_TOKEN, PR_CONFIG, undefined, {})
|
||||
const tags = bitbucketConnector.mapTags?.(result.documents[0].metadata ?? {})
|
||||
|
||||
expect(tags).toEqual({
|
||||
contentType: 'pull_request',
|
||||
title: 'PR 7',
|
||||
repository: 'acme/widgets',
|
||||
state: 'OPEN',
|
||||
author: 'Ada Lovelace',
|
||||
createdAt: new Date('2026-01-01T00:00:00.000000+00:00'),
|
||||
updatedAt: new Date('2026-02-01T00:00:00.000000+00:00'),
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('bitbucket validateConfig', () => {
|
||||
it('rejects a repository the credential cannot read', async () => {
|
||||
mockApi([[/\/repositories\/acme\/widgets$/, () => jsonResponse({ type: 'error' }, 404)]])
|
||||
|
||||
expect(await bitbucketConnector.validateConfig(ACCESS_TOKEN, CONFIG)).toEqual({
|
||||
valid: false,
|
||||
error: 'Repository "acme/widgets" not found',
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a branch or tag that resolves to neither', async () => {
|
||||
mockApi([[/\/refs\//, () => jsonResponse({ type: 'error' }, 404)]])
|
||||
|
||||
const result = await bitbucketConnector.validateConfig(ACCESS_TOKEN, {
|
||||
...CONFIG,
|
||||
ref: 'nope',
|
||||
})
|
||||
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.error).toMatch(/Branch or tag "nope" not found/)
|
||||
})
|
||||
|
||||
it('accepts a tag when no branch by that name exists', async () => {
|
||||
mockApi([
|
||||
[/\/refs\/branches\//, () => jsonResponse({ type: 'error' }, 404)],
|
||||
[/\/refs\/tags\//, () => jsonResponse({ name: 'v1.0', target: { hash: COMMIT } })],
|
||||
])
|
||||
|
||||
expect(
|
||||
await bitbucketConnector.validateConfig(ACCESS_TOKEN, { ...CONFIG, ref: 'v1.0' })
|
||||
).toEqual({ valid: true })
|
||||
})
|
||||
|
||||
it('rejects a non-positive maxItems before spending a request', async () => {
|
||||
mockApi([])
|
||||
|
||||
expect(
|
||||
await bitbucketConnector.validateConfig(ACCESS_TOKEN, { ...CONFIG, maxItems: '0' })
|
||||
).toEqual({ valid: false, error: 'Max items must be a positive number' })
|
||||
})
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
export { bitbucketConnector } from '@/connectors/bitbucket/bitbucket'
|
||||
@@ -0,0 +1,156 @@
|
||||
import { BitbucketIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const bitbucketConnectorMeta: ConnectorMeta = {
|
||||
id: 'bitbucket',
|
||||
name: 'Bitbucket',
|
||||
description:
|
||||
'Sync repository files and pull request descriptions from a Bitbucket Cloud repository into your knowledge base',
|
||||
version: '1.0.0',
|
||||
icon: BitbucketIcon,
|
||||
|
||||
/**
|
||||
* Deliberately disabled. Incremental sync would only ever apply to pull requests,
|
||||
* via a BBQL `q=updated_on > <timestamp>` filter derived from lastSyncAt; repository
|
||||
* files carry no per-file change timestamp in the source listing and are always
|
||||
* re-listed in full regardless. The sync engine disables deletion reconciliation for
|
||||
* every incremental run, so declaring support would leave files deleted upstream
|
||||
* indexed indefinitely on the default `code` configuration -- a complete listing whose
|
||||
* deletions are never applied. Re-listing pull requests in full each run is the
|
||||
* cheaper tradeoff.
|
||||
*/
|
||||
supportsIncrementalSync: false,
|
||||
|
||||
/**
|
||||
* Bitbucket Cloud REST API 2.0 authenticates with an OAuth 2.0 bearer token, the
|
||||
* same credential the Bitbucket tools and selectors already use. The scopes named
|
||||
* here are the ones Sim's Bitbucket OAuth service already requests
|
||||
* (`lib/oauth/oauth.ts`); the connector deliberately indexes nothing that would
|
||||
* need a scope outside that set.
|
||||
*/
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'bitbucket',
|
||||
requiredScopes: ['repository', 'pullrequest'],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'workspaceSelector',
|
||||
title: 'Workspace',
|
||||
type: 'selector',
|
||||
selectorKey: 'bitbucket.workspaces',
|
||||
canonicalParamId: 'workspaceSlug',
|
||||
mode: 'basic',
|
||||
placeholder: 'Select a workspace',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'workspaceSlug',
|
||||
title: 'Workspace Slug',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'workspaceSlug',
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. my-team',
|
||||
required: true,
|
||||
description: 'Workspace ID (slug) or workspace UUID in curly braces.',
|
||||
},
|
||||
{
|
||||
id: 'repoSelector',
|
||||
title: 'Repository',
|
||||
type: 'selector',
|
||||
selectorKey: 'bitbucket.repositories',
|
||||
canonicalParamId: 'repoSlug',
|
||||
mode: 'basic',
|
||||
dependsOn: ['workspaceSelector'],
|
||||
placeholder: 'Select a repository',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'repoSlug',
|
||||
title: 'Repository Slug',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'repoSlug',
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. my-repo',
|
||||
required: true,
|
||||
description: 'Repository slug or repository UUID in curly braces.',
|
||||
},
|
||||
{
|
||||
id: 'contentTypes',
|
||||
title: 'Content',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Code (repository files) only', id: 'code' },
|
||||
{ label: 'Pull requests only', id: 'pullrequests' },
|
||||
{ label: 'Code & Pull Requests', id: 'all' },
|
||||
],
|
||||
placeholder: 'Code (repository files) only',
|
||||
description:
|
||||
'Which content to index. Defaults to repository files when left unset. Bitbucket Cloud wikis and the issue tracker are not indexed — see the connector notes.',
|
||||
},
|
||||
{
|
||||
id: 'ref',
|
||||
title: 'Branch or Tag',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'Default branch',
|
||||
description: 'Branch or tag to sync repository files from. Applies only when syncing Code.',
|
||||
},
|
||||
{
|
||||
id: 'pathPrefix',
|
||||
title: 'Path Filter',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. docs/',
|
||||
description:
|
||||
'Only sync repository files under this path prefix. Applies only when syncing Code.',
|
||||
},
|
||||
{
|
||||
id: 'fileExtensions',
|
||||
title: 'File Extensions',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. .md, .txt, .mdx',
|
||||
description:
|
||||
'Only sync repository files with these extensions (comma-separated). Leave blank for all text files. Applies only when syncing Code.',
|
||||
},
|
||||
{
|
||||
id: 'pullRequestState',
|
||||
title: 'Pull Request State',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
options: [
|
||||
{ label: 'Open only', id: 'open' },
|
||||
{ label: 'Merged only', id: 'merged' },
|
||||
{ label: 'Open & Merged', id: 'openMerged' },
|
||||
{ label: 'All', id: 'all' },
|
||||
],
|
||||
description: 'Which pull requests to sync by state. Applies only when syncing pull requests.',
|
||||
},
|
||||
{
|
||||
id: 'maxItems',
|
||||
title: 'Max Items',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'contentType', displayName: 'Content Type', fieldType: 'text' },
|
||||
{ id: 'title', displayName: 'Title', fieldType: 'text' },
|
||||
{ id: 'repository', displayName: 'Repository', fieldType: 'text' },
|
||||
{ id: 'path', displayName: 'File Path', fieldType: 'text' },
|
||||
{ id: 'state', displayName: 'State', fieldType: 'text' },
|
||||
{ id: 'author', displayName: 'Author', fieldType: 'text' },
|
||||
{ id: 'size', displayName: 'File Size (bytes)', fieldType: 'number' },
|
||||
{ id: 'createdAt', displayName: 'Created At', fieldType: 'date' },
|
||||
{ id: 'updatedAt', displayName: 'Updated At', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,650 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*
|
||||
* `GET /api/2.0/workspace/list` is unpaginated and returns one directory at a
|
||||
* time, so the notebook side of this connector is a hand-rolled breadth-first
|
||||
* walk whose pending-directory queue rides in the cursor. Everything that can
|
||||
* silently shrink a listing lives in that walk — a container type the walk does
|
||||
* not recognise, a directory the token cannot read, the document cap — and the
|
||||
* sync engine hard-deletes whatever a listing leaves out unless
|
||||
* `syncContext.listingCapped` says the listing was partial. Those paths are the
|
||||
* bulk of what is asserted here, alongside the stub/`getDocument` hash equality
|
||||
* that keeps every sync from re-indexing every notebook.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { databricksConnector } from '@/connectors/databricks/databricks'
|
||||
import { databricksConnectorMeta } from '@/connectors/databricks/meta'
|
||||
|
||||
const TOKEN = 'dapi-personal-access-token'
|
||||
const HOST = 'dbc-1234abcd-5678.cloud.databricks.com'
|
||||
const ORIGIN = `https://${HOST}`
|
||||
|
||||
const NOTEBOOK_CONFIG = { workspaceHost: HOST, contentType: 'notebooks', rootPath: '/Shared' }
|
||||
const QUERY_CONFIG = { workspaceHost: HOST, contentType: 'queries' }
|
||||
|
||||
const fetchMock = vi.fn()
|
||||
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
})
|
||||
}
|
||||
|
||||
function dir(path: string) {
|
||||
return { object_type: 'DIRECTORY', path, object_id: path.length }
|
||||
}
|
||||
|
||||
function repo(path: string) {
|
||||
return { object_type: 'REPO', path, object_id: path.length }
|
||||
}
|
||||
|
||||
function notebook(path: string, modifiedAt?: number) {
|
||||
return {
|
||||
object_type: 'NOTEBOOK',
|
||||
path,
|
||||
object_id: path.length * 1000,
|
||||
language: 'PYTHON',
|
||||
...(modifiedAt === undefined ? {} : { modified_at: modifiedAt }),
|
||||
}
|
||||
}
|
||||
|
||||
function query(id: string, overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id,
|
||||
display_name: `Query ${id}`,
|
||||
query_text: `SELECT ${id}`,
|
||||
catalog: 'main',
|
||||
schema: 'default',
|
||||
owner_user_name: 'ada@example.com',
|
||||
lifecycle_state: 'ACTIVE',
|
||||
update_time: '2026-01-01T00:00:00Z',
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads the `path` query parameter off a recorded `workspace/list` call. */
|
||||
function listedPaths(): string[] {
|
||||
return fetchMock.mock.calls
|
||||
.map(([url]) => new URL(url as string))
|
||||
.filter((url) => url.pathname === '/api/2.0/workspace/list')
|
||||
.map((url) => url.searchParams.get('path') ?? '')
|
||||
}
|
||||
|
||||
function queryListingUrls(): URL[] {
|
||||
return fetchMock.mock.calls
|
||||
.map(([url]) => new URL(url as string))
|
||||
.filter((url) => url.pathname === '/api/2.0/sql/queries')
|
||||
}
|
||||
|
||||
/**
|
||||
* Serves `workspace/list` from a path→objects map. An unmapped path is a 404,
|
||||
* which is the Databricks answer for a path that no longer exists.
|
||||
*/
|
||||
function mockWorkspace(tree: Record<string, unknown[] | (() => Response)>) {
|
||||
fetchMock.mockImplementation(async (url: string) => {
|
||||
const parsed = new URL(url)
|
||||
if (parsed.pathname === '/api/2.0/workspace/list') {
|
||||
const path = parsed.searchParams.get('path') ?? ''
|
||||
const entry = tree[path]
|
||||
if (entry === undefined) {
|
||||
return jsonResponse({ error_code: 'RESOURCE_DOES_NOT_EXIST', message: 'not found' }, 404)
|
||||
}
|
||||
/** A factory, not a value: a retried request must not re-read a used body. */
|
||||
if (typeof entry === 'function') return entry()
|
||||
return jsonResponse({ objects: entry })
|
||||
}
|
||||
throw new Error(`Unexpected request: ${url}`)
|
||||
})
|
||||
}
|
||||
|
||||
/** Wraps an error body so each (possibly retried) request gets a fresh Response. */
|
||||
function errorResponse(body: Record<string, string>, status: number): () => Response {
|
||||
return () => jsonResponse(body, status)
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
describe('databricks notebook traversal', () => {
|
||||
it('walks DIRECTORY and REPO containers and indexes only notebooks', async () => {
|
||||
mockWorkspace({
|
||||
'/Shared': [dir('/Shared/team'), repo('/Shared/git-folder')],
|
||||
'/Shared/team': [
|
||||
notebook('/Shared/team/etl', 1),
|
||||
{ object_type: 'FILE', path: '/Shared/team/readme.md', object_id: 5 },
|
||||
{ object_type: 'DASHBOARD', path: '/Shared/team/board.lvdash.json', object_id: 6 },
|
||||
{ object_type: 'LIBRARY', path: '/Shared/team/lib.jar', object_id: 7 },
|
||||
],
|
||||
'/Shared/git-folder': [notebook('/Shared/git-folder/model', 2)],
|
||||
})
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await databricksConnector.listDocuments(
|
||||
TOKEN,
|
||||
NOTEBOOK_CONFIG,
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(listedPaths().sort()).toEqual(['/Shared', '/Shared/git-folder', '/Shared/team'])
|
||||
expect(result.documents.map((doc) => doc.externalId).sort()).toEqual([
|
||||
'notebook:/Shared/git-folder/model',
|
||||
'notebook:/Shared/team/etl',
|
||||
])
|
||||
expect(result.hasMore).toBe(false)
|
||||
expect(syncContext.listingCapped).toBeUndefined()
|
||||
})
|
||||
|
||||
/**
|
||||
* `workspace/list` answers with the object itself when the path is not a
|
||||
* directory. Re-queueing that self-reference would list the same path forever.
|
||||
*/
|
||||
it('does not re-queue a container that lists itself', async () => {
|
||||
mockWorkspace({
|
||||
'/Shared': [dir('/Shared'), notebook('/Shared/etl', 1)],
|
||||
})
|
||||
|
||||
const result = await databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {})
|
||||
|
||||
expect(listedPaths()).toEqual(['/Shared'])
|
||||
expect(result.documents).toHaveLength(1)
|
||||
expect(result.hasMore).toBe(false)
|
||||
})
|
||||
|
||||
it('carries the pending queue through the cursor without re-listing or duplicating', async () => {
|
||||
/** 60 sibling directories: more than the 50 a single call will list. */
|
||||
const children = Array.from({ length: 60 }, (_, i) => dir(`/Shared/d${i}`))
|
||||
const tree: Record<string, unknown[]> = { '/Shared': children }
|
||||
for (let i = 0; i < 60; i++) {
|
||||
tree[`/Shared/d${i}`] = [notebook(`/Shared/d${i}/nb`, i + 1)]
|
||||
}
|
||||
mockWorkspace(tree)
|
||||
|
||||
const first = await databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {})
|
||||
expect(first.hasMore).toBe(true)
|
||||
expect(first.nextCursor).toBeDefined()
|
||||
expect(first.documents).toHaveLength(49)
|
||||
|
||||
const firstPaths = listedPaths()
|
||||
fetchMock.mockClear()
|
||||
|
||||
const second = await databricksConnector.listDocuments(
|
||||
TOKEN,
|
||||
NOTEBOOK_CONFIG,
|
||||
first.nextCursor,
|
||||
{}
|
||||
)
|
||||
expect(second.hasMore).toBe(false)
|
||||
expect(second.documents).toHaveLength(11)
|
||||
|
||||
const secondPaths = listedPaths()
|
||||
expect(secondPaths.some((path) => firstPaths.includes(path))).toBe(false)
|
||||
|
||||
const ids = [...first.documents, ...second.documents].map((doc) => doc.externalId)
|
||||
expect(new Set(ids).size).toBe(ids.length)
|
||||
expect(ids).toHaveLength(60)
|
||||
})
|
||||
})
|
||||
|
||||
describe('databricks notebook listingCapped', () => {
|
||||
it('flags listingCapped when a subdirectory is unreadable', async () => {
|
||||
mockWorkspace({
|
||||
'/Shared': [dir('/Shared/open'), dir('/Shared/locked')],
|
||||
'/Shared/open': [notebook('/Shared/open/nb', 1)],
|
||||
'/Shared/locked': errorResponse({ error_code: 'PERMISSION_DENIED' }, 403),
|
||||
})
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await databricksConnector.listDocuments(
|
||||
TOKEN,
|
||||
NOTEBOOK_CONFIG,
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(result.documents).toHaveLength(1)
|
||||
expect(syncContext.listingCapped).toBe(true)
|
||||
})
|
||||
|
||||
it('flags listingCapped when a subdirectory has been deleted mid-walk', async () => {
|
||||
mockWorkspace({
|
||||
'/Shared': [dir('/Shared/open'), dir('/Shared/gone')],
|
||||
'/Shared/open': [notebook('/Shared/open/nb', 1)],
|
||||
})
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
await databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, syncContext)
|
||||
|
||||
expect(syncContext.listingCapped).toBe(true)
|
||||
})
|
||||
|
||||
/**
|
||||
* A denied root yields an empty listing, not a partial one. Reporting that as
|
||||
* a successful sync of zero documents is worse than failing.
|
||||
*/
|
||||
it('throws rather than reporting an empty sync when the root itself is denied', async () => {
|
||||
mockWorkspace({ '/Shared': errorResponse({ error_code: 'PERMISSION_DENIED' }, 403) })
|
||||
|
||||
await expect(
|
||||
databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {})
|
||||
).rejects.toThrow(/denied access to \/Shared/)
|
||||
})
|
||||
|
||||
it('flags listingCapped when maxDocuments hides notebooks from the same page', async () => {
|
||||
mockWorkspace({
|
||||
'/Shared': [notebook('/Shared/a', 1), notebook('/Shared/b', 2), notebook('/Shared/c', 3)],
|
||||
})
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await databricksConnector.listDocuments(
|
||||
TOKEN,
|
||||
{ ...NOTEBOOK_CONFIG, maxDocuments: '2' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(result.documents).toHaveLength(2)
|
||||
expect(result.hasMore).toBe(false)
|
||||
expect(syncContext.listingCapped).toBe(true)
|
||||
})
|
||||
|
||||
it('leaves listingCapped unset when maxDocuments lands exactly on an exhausted walk', async () => {
|
||||
mockWorkspace({
|
||||
'/Shared': [notebook('/Shared/a', 1), notebook('/Shared/b', 2)],
|
||||
})
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await databricksConnector.listDocuments(
|
||||
TOKEN,
|
||||
{ ...NOTEBOOK_CONFIG, maxDocuments: '2' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(result.documents).toHaveLength(2)
|
||||
expect(result.hasMore).toBe(false)
|
||||
expect(syncContext.listingCapped).toBeUndefined()
|
||||
})
|
||||
|
||||
it('flags listingCapped when maxDocuments stops the walk with directories pending', async () => {
|
||||
mockWorkspace({
|
||||
'/Shared': [dir('/Shared/a'), dir('/Shared/b')],
|
||||
'/Shared/a': [notebook('/Shared/a/nb', 1)],
|
||||
'/Shared/b': [notebook('/Shared/b/nb', 2)],
|
||||
})
|
||||
|
||||
const syncContext: Record<string, unknown> = { totalDocsFetched: 0 }
|
||||
const result = await databricksConnector.listDocuments(
|
||||
TOKEN,
|
||||
{ ...NOTEBOOK_CONFIG, maxDocuments: '1' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(result.documents).toHaveLength(1)
|
||||
expect(syncContext.listingCapped).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('databricks notebook error semantics under concurrency', () => {
|
||||
/**
|
||||
* The walk lists siblings in parallel. A permission failure on one sibling has
|
||||
* to stay a skip while a server failure on another still aborts the whole
|
||||
* listing — a `Promise.all` that swallowed either would be invisible in the
|
||||
* happy path.
|
||||
*/
|
||||
it('propagates a 500 on one sibling even when another sibling is a skippable 403', async () => {
|
||||
mockWorkspace({
|
||||
'/Shared': [dir('/Shared/locked'), dir('/Shared/broken')],
|
||||
'/Shared/locked': errorResponse({ error_code: 'PERMISSION_DENIED' }, 403),
|
||||
'/Shared/broken': errorResponse({ error_code: 'INTERNAL_ERROR', message: 'boom' }, 500),
|
||||
})
|
||||
|
||||
await expect(
|
||||
databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {})
|
||||
).rejects.toThrow(/Failed to list Databricks workspace path \/Shared\/broken: 500/)
|
||||
})
|
||||
|
||||
it('propagates a 401 rather than treating a bad token as an empty directory', async () => {
|
||||
mockWorkspace({
|
||||
'/Shared': [dir('/Shared/a')],
|
||||
'/Shared/a': errorResponse({ error_code: 'PERMISSION_DENIED' }, 401),
|
||||
})
|
||||
|
||||
await expect(
|
||||
databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {})
|
||||
).rejects.toThrow(/401/)
|
||||
})
|
||||
|
||||
it('propagates a 429 after the shared retry budget is spent', async () => {
|
||||
vi.useFakeTimers()
|
||||
mockWorkspace({
|
||||
'/Shared': [dir('/Shared/a')],
|
||||
'/Shared/a': errorResponse({ error_code: 'REQUEST_LIMIT_EXCEEDED' }, 429),
|
||||
})
|
||||
|
||||
const pending = databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {})
|
||||
const assertion = expect(pending).rejects.toThrow(/429/)
|
||||
await vi.runAllTimersAsync()
|
||||
await assertion
|
||||
})
|
||||
})
|
||||
|
||||
describe('databricks saved SQL queries', () => {
|
||||
it('drops trashed queries without suppressing deletion reconciliation', async () => {
|
||||
fetchMock.mockResolvedValue(
|
||||
jsonResponse({
|
||||
results: [query('a'), query('b', { lifecycle_state: 'TRASHED' })],
|
||||
})
|
||||
)
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await databricksConnector.listDocuments(
|
||||
TOKEN,
|
||||
QUERY_CONFIG,
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(result.documents.map((doc) => doc.externalId)).toEqual(['query:a'])
|
||||
expect(result.hasMore).toBe(false)
|
||||
expect(syncContext.listingCapped).toBeUndefined()
|
||||
})
|
||||
|
||||
it('flags listingCapped when the cap stops paging while a page token remains', async () => {
|
||||
fetchMock.mockResolvedValue(
|
||||
jsonResponse({ results: [query('a'), query('b')], next_page_token: 'tok' })
|
||||
)
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await databricksConnector.listDocuments(
|
||||
TOKEN,
|
||||
{ ...QUERY_CONFIG, maxDocuments: '2' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(result.documents).toHaveLength(2)
|
||||
expect(result.hasMore).toBe(false)
|
||||
expect(syncContext.listingCapped).toBe(true)
|
||||
})
|
||||
|
||||
it('leaves listingCapped unset when the cap lands on the last page', async () => {
|
||||
fetchMock.mockResolvedValue(jsonResponse({ results: [query('a'), query('b')] }))
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
await databricksConnector.listDocuments(
|
||||
TOKEN,
|
||||
{ ...QUERY_CONFIG, maxDocuments: '2' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(syncContext.listingCapped).toBeUndefined()
|
||||
})
|
||||
|
||||
/**
|
||||
* An echoed token on an empty tail would re-request the same page until the
|
||||
* engine truncates pagination, which permanently disables deletion
|
||||
* reconciliation.
|
||||
*/
|
||||
it('stops paging on an empty page even when a token is echoed back', async () => {
|
||||
fetchMock.mockResolvedValue(jsonResponse({ results: [], next_page_token: 'tok' }))
|
||||
|
||||
const result = await databricksConnector.listDocuments(TOKEN, QUERY_CONFIG, undefined, {})
|
||||
|
||||
expect(result.hasMore).toBe(false)
|
||||
expect(result.nextCursor).toBeUndefined()
|
||||
})
|
||||
|
||||
it('asks for the full page size when no cap is configured', async () => {
|
||||
fetchMock.mockResolvedValue(jsonResponse({ results: [query('a')] }))
|
||||
|
||||
await databricksConnector.listDocuments(TOKEN, QUERY_CONFIG, undefined, {})
|
||||
|
||||
expect(queryListingUrls()[0].searchParams.get('page_size')).toBe('100')
|
||||
})
|
||||
|
||||
it('shrinks the last page to what the cap still allows', async () => {
|
||||
fetchMock.mockResolvedValue(jsonResponse({ results: [query('a')] }))
|
||||
|
||||
await databricksConnector.listDocuments(
|
||||
TOKEN,
|
||||
{ ...QUERY_CONFIG, maxDocuments: '10' },
|
||||
undefined,
|
||||
{ totalDocsFetched: 7 }
|
||||
)
|
||||
|
||||
expect(queryListingUrls()[0].searchParams.get('page_size')).toBe('3')
|
||||
})
|
||||
|
||||
/**
|
||||
* `page_size=0` is not a valid request. The cap arithmetic must floor at one
|
||||
* even when the running total has already reached (or overshot) the cap.
|
||||
*/
|
||||
it.each([
|
||||
['exactly at the cap', 10],
|
||||
['past the cap', 12],
|
||||
])('never requests a non-positive page size when already %s', async (_label, fetched) => {
|
||||
fetchMock.mockResolvedValue(jsonResponse({ results: [] }))
|
||||
|
||||
await databricksConnector.listDocuments(
|
||||
TOKEN,
|
||||
{ ...QUERY_CONFIG, maxDocuments: '10' },
|
||||
undefined,
|
||||
{ totalDocsFetched: fetched }
|
||||
)
|
||||
|
||||
const pageSize = Number(queryListingUrls()[0].searchParams.get('page_size'))
|
||||
expect(pageSize).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('databricks contentHash consistency', () => {
|
||||
it('produces the same notebook hash from the listing stub and from getDocument', async () => {
|
||||
mockWorkspace({ '/Shared': [notebook('/Shared/etl', 1700000000000)] })
|
||||
|
||||
const listed = await databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {})
|
||||
const stub = listed.documents[0]
|
||||
expect(stub.contentDeferred).toBe(true)
|
||||
expect(stub.content).toBe('')
|
||||
|
||||
fetchMock.mockImplementation(async (url: string) => {
|
||||
const parsed = new URL(url)
|
||||
if (parsed.pathname === '/api/2.0/workspace/get-status') {
|
||||
return jsonResponse(notebook('/Shared/etl', 1700000000000))
|
||||
}
|
||||
if (parsed.pathname === '/api/2.0/workspace/export') {
|
||||
expect(parsed.searchParams.get('format')).toBe('SOURCE')
|
||||
return jsonResponse({
|
||||
content: Buffer.from('print("hi")', 'utf8').toString('base64'),
|
||||
file_type: 'py',
|
||||
})
|
||||
}
|
||||
throw new Error(`Unexpected request: ${url}`)
|
||||
})
|
||||
|
||||
const hydrated = await databricksConnector.getDocument(TOKEN, NOTEBOOK_CONFIG, stub.externalId)
|
||||
|
||||
expect(hydrated?.contentHash).toBe(stub.contentHash)
|
||||
expect(hydrated?.externalId).toBe(stub.externalId)
|
||||
expect(hydrated?.contentDeferred).toBe(false)
|
||||
expect(hydrated?.content).toBe('print("hi")')
|
||||
expect(hydrated?.sourceUrl).toBe(`${ORIGIN}/#notebook/${'/Shared/etl'.length * 1000}`)
|
||||
})
|
||||
|
||||
/**
|
||||
* `modified_at` is documented as file-only, so a notebook routinely carries no
|
||||
* timestamp at all. Its hash must still be stable across syncs — a clock- or
|
||||
* fetch-derived component would make every sync re-index every notebook. The
|
||||
* cost of that stability is that edits go unseen until a full resync, which is
|
||||
* exactly what `rehydrateOnFullSync` covers.
|
||||
*/
|
||||
it('keeps an undated notebook hash stable across repeated listings', async () => {
|
||||
mockWorkspace({ '/Shared': [notebook('/Shared/etl')] })
|
||||
|
||||
const first = await databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {})
|
||||
const second = await databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {})
|
||||
|
||||
expect(first.documents[0].contentHash).toBe('databricks:notebook:/Shared/etl:')
|
||||
expect(second.documents[0].contentHash).toBe(first.documents[0].contentHash)
|
||||
})
|
||||
|
||||
it('produces the same query hash from the listing and from getDocument', async () => {
|
||||
fetchMock.mockResolvedValue(jsonResponse({ results: [query('q1')] }))
|
||||
const listed = await databricksConnector.listDocuments(TOKEN, QUERY_CONFIG, undefined, {})
|
||||
const stub = listed.documents[0]
|
||||
|
||||
fetchMock.mockResolvedValue(jsonResponse(query('q1')))
|
||||
const refetched = await databricksConnector.getDocument(TOKEN, QUERY_CONFIG, stub.externalId)
|
||||
|
||||
expect(refetched?.contentHash).toBe(stub.contentHash)
|
||||
expect(refetched?.content).toBe(stub.content)
|
||||
})
|
||||
|
||||
it('returns null for a notebook that has been deleted since the listing', async () => {
|
||||
fetchMock.mockResolvedValue(
|
||||
jsonResponse({ error_code: 'RESOURCE_DOES_NOT_EXIST', message: 'gone' }, 404)
|
||||
)
|
||||
|
||||
await expect(
|
||||
databricksConnector.getDocument(TOKEN, NOTEBOOK_CONFIG, 'notebook:/Shared/gone')
|
||||
).resolves.toBeNull()
|
||||
})
|
||||
|
||||
it('reports an oversize export as a skip rather than losing the notebook', async () => {
|
||||
fetchMock.mockImplementation(async (url: string) => {
|
||||
const parsed = new URL(url)
|
||||
if (parsed.pathname === '/api/2.0/workspace/get-status') {
|
||||
return jsonResponse(notebook('/Shared/big', 5))
|
||||
}
|
||||
return jsonResponse({ error_code: 'MAX_NOTEBOOK_SIZE_EXCEEDED', message: 'too big' }, 400)
|
||||
})
|
||||
|
||||
const doc = await databricksConnector.getDocument(
|
||||
TOKEN,
|
||||
NOTEBOOK_CONFIG,
|
||||
'notebook:/Shared/big'
|
||||
)
|
||||
|
||||
expect(doc?.skippedReason).toMatch(/10MB size limit/)
|
||||
expect(doc?.contentHash).toBe(`databricks:notebook:/Shared/big:5`)
|
||||
})
|
||||
})
|
||||
|
||||
describe('databricks configuration', () => {
|
||||
it('declares rehydrateOnFullSync so undated notebook edits stay recoverable', () => {
|
||||
expect(databricksConnectorMeta.rehydrateOnFullSync).toBe(true)
|
||||
})
|
||||
|
||||
it('refuses a workspace host outside the Databricks-owned domains', async () => {
|
||||
const result = await databricksConnector.validateConfig(TOKEN, {
|
||||
workspaceHost: 'evil.example.com',
|
||||
contentType: 'notebooks',
|
||||
})
|
||||
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.error).toMatch(/Databricks-hosted domain/)
|
||||
expect(fetchMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('refuses to list against a host outside the allowlist', async () => {
|
||||
await expect(
|
||||
databricksConnector.listDocuments(
|
||||
TOKEN,
|
||||
{ ...NOTEBOOK_CONFIG, workspaceHost: 'evil.example.com' },
|
||||
undefined,
|
||||
{}
|
||||
)
|
||||
).rejects.toThrow(/Databricks-hosted domain/)
|
||||
})
|
||||
|
||||
it('rejects a root path that names a notebook instead of a folder', async () => {
|
||||
fetchMock.mockResolvedValue(jsonResponse(notebook('/Shared/etl', 1)))
|
||||
|
||||
const result = await databricksConnector.validateConfig(TOKEN, {
|
||||
...NOTEBOOK_CONFIG,
|
||||
rootPath: '/Shared/etl',
|
||||
})
|
||||
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.error).toMatch(/is a notebook, not a folder/)
|
||||
})
|
||||
|
||||
it('accepts a Git folder as the notebook root', async () => {
|
||||
fetchMock.mockResolvedValue(jsonResponse(repo('/Repos/ada/project')))
|
||||
|
||||
await expect(
|
||||
databricksConnector.validateConfig(TOKEN, {
|
||||
...NOTEBOOK_CONFIG,
|
||||
rootPath: '/Repos/ada/project',
|
||||
})
|
||||
).resolves.toEqual({ valid: true })
|
||||
})
|
||||
|
||||
it('normalizes a relative, trailing-slashed root path to an absolute one', async () => {
|
||||
fetchMock.mockResolvedValue(jsonResponse(dir('/Shared/team')))
|
||||
|
||||
await databricksConnector.validateConfig(TOKEN, {
|
||||
...NOTEBOOK_CONFIG,
|
||||
rootPath: ' Shared/team/ ',
|
||||
})
|
||||
|
||||
const url = new URL(fetchMock.mock.calls[0][0] as string)
|
||||
expect(url.searchParams.get('path')).toBe('/Shared/team')
|
||||
})
|
||||
})
|
||||
|
||||
describe('databricks mapTags', () => {
|
||||
it('maps notebook metadata to the declared tag slots', () => {
|
||||
const tags = databricksConnector.mapTags?.({
|
||||
language: 'PYTHON',
|
||||
lastModified: '2026-01-01T00:00:00.000Z',
|
||||
path: '/Shared/etl',
|
||||
})
|
||||
|
||||
expect(tags?.language).toBe('PYTHON')
|
||||
expect(tags?.lastModified).toBeInstanceOf(Date)
|
||||
})
|
||||
|
||||
it('maps query metadata, including the joined tag array', () => {
|
||||
const tags = databricksConnector.mapTags?.({
|
||||
owner: 'ada@example.com',
|
||||
catalog: 'main',
|
||||
schema: 'default',
|
||||
labels: ['finance', 'daily'],
|
||||
lastModified: '2026-01-01T00:00:00Z',
|
||||
})
|
||||
|
||||
expect(tags).toMatchObject({
|
||||
owner: 'ada@example.com',
|
||||
catalog: 'main',
|
||||
schema: 'default',
|
||||
labels: 'finance, daily',
|
||||
})
|
||||
})
|
||||
|
||||
it('emits only tag ids the connector declares', () => {
|
||||
const declared = new Set(databricksConnectorMeta.tagDefinitions?.map((tag) => tag.id))
|
||||
const produced = databricksConnector.mapTags?.({
|
||||
language: 'SQL',
|
||||
owner: 'ada@example.com',
|
||||
catalog: 'main',
|
||||
schema: 'default',
|
||||
labels: ['a'],
|
||||
lastModified: '2026-01-01T00:00:00Z',
|
||||
})
|
||||
|
||||
for (const key of Object.keys(produced ?? {})) {
|
||||
expect(declared.has(key)).toBe(true)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,814 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { truncate } from '@sim/utils/string'
|
||||
import { validateDatabricksWorkspaceHost } from '@/lib/core/security/input-validation'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import {
|
||||
DATABRICKS_CONTENT_TYPES,
|
||||
type DatabricksContentType,
|
||||
DEFAULT_NOTEBOOK_ROOT_PATH,
|
||||
databricksConnectorMeta,
|
||||
} from '@/connectors/databricks/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import {
|
||||
CONNECTOR_MAX_FILE_BYTES,
|
||||
isSkippedDocument,
|
||||
joinTagArray,
|
||||
markSkipped,
|
||||
parseTagDate,
|
||||
sizeLimitSkipReason,
|
||||
takeIndexableWithinCap,
|
||||
} from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('DatabricksConnector')
|
||||
|
||||
/** Page size for `GET /api/2.0/sql/queries`. */
|
||||
const QUERY_PAGE_SIZE = 100
|
||||
|
||||
/**
|
||||
* Directories listed per `listDocuments` call.
|
||||
*
|
||||
* `GET /api/2.0/workspace/list` is not paginated — it returns a whole directory
|
||||
* in one response and offers no page token — so the connector walks the tree
|
||||
* breadth-first and carries the pending-directory queue in the cursor. This
|
||||
* bounds how much of the walk happens before the engine gets a batch to persist.
|
||||
*/
|
||||
const DIRECTORIES_PER_CALL = 50
|
||||
|
||||
/** Sibling directories listed in parallel while walking the tree. */
|
||||
const DIRECTORY_CONCURRENCY = 5
|
||||
|
||||
/** Notebooks collected before a `listDocuments` call yields, even mid-walk. */
|
||||
const MAX_NOTEBOOKS_PER_CALL = 500
|
||||
|
||||
/**
|
||||
* Workspace object types that contain other objects and are therefore walked.
|
||||
*
|
||||
* `REPO` is a Git folder. It is a container just like `DIRECTORY` — every
|
||||
* notebook inside a Git folder is only reachable by listing the repo path — so
|
||||
* omitting it would silently skip every notebook under `/Repos` and every Git
|
||||
* folder in a user's home.
|
||||
*/
|
||||
const TRAVERSABLE_OBJECT_TYPES = new Set(['DIRECTORY', 'REPO'])
|
||||
|
||||
/**
|
||||
* Maximum size of a single `workspace/export`, documented alongside the
|
||||
* `MAX_NOTEBOOK_SIZE_EXCEEDED` error. Independent of (and far below) the
|
||||
* knowledge base's own document cap, so the skip reason has to quote it rather
|
||||
* than `CONNECTOR_MAX_FILE_BYTES`.
|
||||
*/
|
||||
const DATABRICKS_MAX_EXPORT_BYTES = 10 * 1024 * 1024
|
||||
|
||||
/** `externalId` prefixes, so a document always names the API it came from. */
|
||||
const NOTEBOOK_ID_PREFIX = 'notebook:'
|
||||
const QUERY_ID_PREFIX = 'query:'
|
||||
|
||||
/**
|
||||
* `ObjectInfo` as returned by `GET /api/2.0/workspace/list` and
|
||||
* `GET /api/2.0/workspace/get-status`.
|
||||
*
|
||||
* `created_at`, `modified_at` and `size` are documented as "only applicable to
|
||||
* files", so they are all optional here — a `NOTEBOOK` entry frequently carries
|
||||
* none of them.
|
||||
*/
|
||||
interface DatabricksObjectInfo {
|
||||
object_type?: string
|
||||
path?: string
|
||||
object_id?: number
|
||||
language?: string
|
||||
created_at?: number
|
||||
modified_at?: number
|
||||
resource_id?: string
|
||||
size?: number
|
||||
}
|
||||
|
||||
interface DatabricksListResponse {
|
||||
objects?: DatabricksObjectInfo[]
|
||||
}
|
||||
|
||||
/** `GET /api/2.0/workspace/export` response (no `direct_download`, so JSON). */
|
||||
interface DatabricksExportResponse {
|
||||
content?: string
|
||||
file_type?: string
|
||||
}
|
||||
|
||||
/** A `results` entry from `GET /api/2.0/sql/queries`. */
|
||||
interface DatabricksQuery {
|
||||
id?: string
|
||||
display_name?: string
|
||||
description?: string
|
||||
query_text?: string
|
||||
catalog?: string
|
||||
schema?: string
|
||||
warehouse_id?: string
|
||||
owner_user_name?: string
|
||||
last_modifier_user_name?: string
|
||||
lifecycle_state?: string
|
||||
tags?: string[]
|
||||
create_time?: string
|
||||
update_time?: string
|
||||
}
|
||||
|
||||
interface DatabricksQueryListResponse {
|
||||
results?: DatabricksQuery[]
|
||||
next_page_token?: string
|
||||
}
|
||||
|
||||
/** Databricks REST error envelope: `{ error_code, message }`. */
|
||||
interface DatabricksErrorBody {
|
||||
error_code?: string
|
||||
message?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Traversal position across pages of a single notebook sync run.
|
||||
*
|
||||
* Only the pending-directory queue needs carrying: each entry is listed in full
|
||||
* by a single unpaginated `workspace/list` call, so there is no intra-directory
|
||||
* marker to resume from.
|
||||
*/
|
||||
interface DatabricksTraversalState {
|
||||
queue: string[]
|
||||
}
|
||||
|
||||
function encodeCursor(state: DatabricksTraversalState): string {
|
||||
return Buffer.from(JSON.stringify(state), 'utf8').toString('base64url')
|
||||
}
|
||||
|
||||
function decodeCursor(cursor: string): DatabricksTraversalState | null {
|
||||
try {
|
||||
const parsed = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8')) as unknown
|
||||
if (typeof parsed !== 'object' || parsed === null) return null
|
||||
const candidate = parsed as Partial<DatabricksTraversalState>
|
||||
if (!Array.isArray(candidate.queue)) return null
|
||||
return { queue: candidate.queue.filter((path): path is string => typeof path === 'string') }
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes the configured workspace host to an https origin, rejecting hosts
|
||||
* outside Databricks-owned domains.
|
||||
*
|
||||
* Validated here rather than only in `validateConfig` because `listDocuments`
|
||||
* and `getDocument` run against a stored `sourceConfig` that may have been
|
||||
* written after validation, and the host is interpolated into every request URL.
|
||||
*/
|
||||
function resolveWorkspaceOrigin(sourceConfig: Record<string, unknown>): string {
|
||||
const result = validateDatabricksWorkspaceHost(sourceConfig.workspaceHost as string | undefined)
|
||||
if (!result.isValid || !result.sanitized) {
|
||||
throw new Error(result.error || 'Invalid Databricks workspace host')
|
||||
}
|
||||
return result.sanitized
|
||||
}
|
||||
|
||||
function resolveContentType(sourceConfig: Record<string, unknown>): DatabricksContentType {
|
||||
return sourceConfig.contentType === DATABRICKS_CONTENT_TYPES.queries
|
||||
? DATABRICKS_CONTENT_TYPES.queries
|
||||
: DATABRICKS_CONTENT_TYPES.notebooks
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes the notebook root to an absolute workspace path.
|
||||
*
|
||||
* Every Workspace API path parameter is documented as absolute, so a relative
|
||||
* value is corrected rather than sent through as a guaranteed 404. Trailing
|
||||
* slashes are dropped (except on the root itself) so the path matches what
|
||||
* `workspace/list` echoes back in `ObjectInfo.path`.
|
||||
*/
|
||||
function normalizeRootPath(value: unknown): string {
|
||||
const raw = typeof value === 'string' ? value.trim() : ''
|
||||
if (!raw) return DEFAULT_NOTEBOOK_ROOT_PATH
|
||||
const absolute = raw.startsWith('/') ? raw : `/${raw}`
|
||||
const trimmed = absolute.replace(/\/+$/, '')
|
||||
return trimmed || DEFAULT_NOTEBOOK_ROOT_PATH
|
||||
}
|
||||
|
||||
function resolveMaxDocuments(value: unknown): number {
|
||||
const parsed = Number(value)
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : 0
|
||||
}
|
||||
|
||||
async function readErrorBody(response: Response): Promise<DatabricksErrorBody> {
|
||||
try {
|
||||
return (await response.json()) as DatabricksErrorBody
|
||||
} catch {
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
function describeError(status: number, body: DatabricksErrorBody): string {
|
||||
const detail = body.message?.trim() || body.error_code?.trim()
|
||||
return detail ? `${status} — ${truncate(detail, 200)}` : String(status)
|
||||
}
|
||||
|
||||
/**
|
||||
* Issues an authenticated GET against a workspace REST endpoint.
|
||||
*
|
||||
* Databricks authenticates every workspace API with a bearer personal access
|
||||
* token, which is exactly what the sync engine hands connectors as
|
||||
* `accessToken`.
|
||||
*/
|
||||
async function databricksGet(
|
||||
origin: string,
|
||||
path: string,
|
||||
accessToken: string,
|
||||
params: Record<string, string>,
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<Response> {
|
||||
const url = new URL(`${origin}${path}`)
|
||||
for (const [key, value] of Object.entries(params)) {
|
||||
url.searchParams.set(key, value)
|
||||
}
|
||||
return fetchWithRetry(
|
||||
url.toString(),
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
retryOptions
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the listing stub for a workspace notebook.
|
||||
*
|
||||
* `externalId` carries the notebook's absolute path rather than its `object_id`
|
||||
* because `workspace/export` is addressed by path only — an id-keyed document
|
||||
* could never be re-hydrated. A moved notebook therefore reads as a delete plus
|
||||
* an add, which is the same behaviour every path-addressed connector has.
|
||||
*
|
||||
* The hash is metadata-only so change detection never needs an export. When
|
||||
* `modified_at` is absent (it is documented as file-only) the hash degenerates
|
||||
* to a constant and only an explicit full resync — see `rehydrateOnFullSync` on
|
||||
* the meta — will pick up later edits.
|
||||
*/
|
||||
function notebookToStub(origin: string, object: DatabricksObjectInfo): ExternalDocument {
|
||||
const path = object.path ?? ''
|
||||
const title = path.slice(path.lastIndexOf('/') + 1) || path || 'Untitled notebook'
|
||||
|
||||
return {
|
||||
externalId: `${NOTEBOOK_ID_PREFIX}${path}`,
|
||||
title,
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: object.object_id ? `${origin}/#notebook/${object.object_id}` : undefined,
|
||||
contentHash: `databricks:notebook:${path}:${object.modified_at ?? ''}`,
|
||||
metadata: {
|
||||
path,
|
||||
objectId: object.object_id,
|
||||
language: object.language,
|
||||
lastModified: object.modified_at ? new Date(object.modified_at).toISOString() : undefined,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders a saved SQL query as indexable text.
|
||||
*
|
||||
* The description is the human-authored half (Databricks surfaces it as "usage
|
||||
* notes") and the query text is the machine half; both are indexed so a search
|
||||
* matches either.
|
||||
*/
|
||||
function formatQueryContent(query: DatabricksQuery): string {
|
||||
const sections: string[] = []
|
||||
const description = query.description?.trim()
|
||||
if (description) sections.push(description)
|
||||
const text = query.query_text?.trim()
|
||||
if (text) sections.push(text)
|
||||
return sections.join('\n\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a document for a saved SQL query.
|
||||
*
|
||||
* `GET /api/2.0/sql/queries` returns `query_text` inline, so there is nothing
|
||||
* left to defer — the listing already carries the full content.
|
||||
*
|
||||
* No `sourceUrl` is set: Databricks documents no stable URL for opening a saved
|
||||
* query by id, and inventing one would produce dead links on every document.
|
||||
*/
|
||||
function queryToDocument(query: DatabricksQuery): ExternalDocument | null {
|
||||
const id = query.id?.trim()
|
||||
if (!id) return null
|
||||
|
||||
const content = formatQueryContent(query)
|
||||
if (!content) return null
|
||||
|
||||
return {
|
||||
externalId: `${QUERY_ID_PREFIX}${id}`,
|
||||
title: query.display_name?.trim() || `Query ${id}`,
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
contentHash: `databricks:query:${id}:${query.update_time ?? ''}`,
|
||||
metadata: {
|
||||
queryId: id,
|
||||
catalog: query.catalog,
|
||||
schema: query.schema,
|
||||
warehouseId: query.warehouse_id,
|
||||
owner: query.owner_user_name ?? query.last_modifier_user_name,
|
||||
labels: query.tags,
|
||||
lastModified: query.update_time,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists one workspace directory.
|
||||
*
|
||||
* Returns `null` when the directory is unreadable (deleted, or the token lacks
|
||||
* permission on that subtree) so the caller can decide whether that is fatal.
|
||||
*/
|
||||
async function listDirectory(
|
||||
origin: string,
|
||||
accessToken: string,
|
||||
accessPath: string
|
||||
): Promise<DatabricksObjectInfo[] | null> {
|
||||
const response = await databricksGet(origin, '/api/2.0/workspace/list', accessToken, {
|
||||
path: accessPath,
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 403 || response.status === 404) {
|
||||
const body = await readErrorBody(response)
|
||||
logger.warn('Skipping unreadable Databricks workspace directory', {
|
||||
path: accessPath,
|
||||
error: describeError(response.status, body),
|
||||
})
|
||||
return null
|
||||
}
|
||||
const body = await readErrorBody(response)
|
||||
throw new Error(
|
||||
`Failed to list Databricks workspace path ${accessPath}: ${describeError(response.status, body)}`
|
||||
)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as DatabricksListResponse
|
||||
return data.objects ?? []
|
||||
}
|
||||
|
||||
/** Walks the workspace tree breadth-first, one bounded slice per call. */
|
||||
async function listNotebooks(
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> {
|
||||
const origin = resolveWorkspaceOrigin(sourceConfig)
|
||||
const rootPath = normalizeRootPath(sourceConfig.rootPath)
|
||||
const maxDocuments = resolveMaxDocuments(sourceConfig.maxDocuments)
|
||||
|
||||
const state = cursor ? (decodeCursor(cursor) ?? { queue: [rootPath] }) : { queue: [rootPath] }
|
||||
const queue = [...state.queue]
|
||||
const notebooks: DatabricksObjectInfo[] = []
|
||||
|
||||
logger.info('Listing Databricks notebooks', {
|
||||
rootPath,
|
||||
pending: queue.length,
|
||||
resumed: Boolean(cursor),
|
||||
})
|
||||
|
||||
for (let listed = 0; listed < DIRECTORIES_PER_CALL && queue.length > 0; ) {
|
||||
const batch = queue.splice(0, Math.min(DIRECTORY_CONCURRENCY, DIRECTORIES_PER_CALL - listed))
|
||||
listed += batch.length
|
||||
|
||||
const listings = await Promise.all(
|
||||
batch.map(async (path) => ({
|
||||
path,
|
||||
objects: await listDirectory(origin, accessToken, path),
|
||||
}))
|
||||
)
|
||||
|
||||
for (const { path: currentPath, objects } of listings) {
|
||||
/**
|
||||
* Losing access to a *sub*directory is survivable, but losing the configured
|
||||
* root means the listing is empty. Failing loudly beats reporting a
|
||||
* successful sync that indexed nothing.
|
||||
*/
|
||||
if (!objects) {
|
||||
if (currentPath === rootPath) {
|
||||
throw new Error(
|
||||
`Databricks denied access to ${rootPath}. Check the token's workspace permissions or choose another root path.`
|
||||
)
|
||||
}
|
||||
/**
|
||||
* A directory was skipped, so notebooks that still exist in the workspace
|
||||
* are absent from this listing. Without this flag the engine would
|
||||
* reconcile them as deleted.
|
||||
*/
|
||||
if (syncContext) syncContext.listingCapped = true
|
||||
continue
|
||||
}
|
||||
|
||||
for (const object of objects) {
|
||||
const path = object.path
|
||||
if (!path) continue
|
||||
if (object.object_type && TRAVERSABLE_OBJECT_TYPES.has(object.object_type)) {
|
||||
/**
|
||||
* `workspace/list` returns the object itself when the path is not a
|
||||
* directory; re-queueing a self-reference would loop forever.
|
||||
*/
|
||||
if (path !== currentPath) queue.push(path)
|
||||
} else if (object.object_type === 'NOTEBOOK') {
|
||||
notebooks.push(object)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (notebooks.length >= MAX_NOTEBOOKS_PER_CALL) break
|
||||
}
|
||||
|
||||
/**
|
||||
* A notebook with no `modified_at` gets a constant `contentHash`, so an edit to
|
||||
* it will never be detected by an ordinary sync — only the "Full resync" action
|
||||
* re-exports it (see `rehydrateOnFullSync`). Surface that here so the condition
|
||||
* is diagnosable from the sync logs instead of presenting as silently stale
|
||||
* content.
|
||||
*/
|
||||
const undatedCount = notebooks.reduce((count, object) => count + (object.modified_at ? 0 : 1), 0)
|
||||
if (undatedCount > 0) {
|
||||
logger.warn(
|
||||
'Databricks listed notebooks without a modification timestamp; their edits are only picked up by a full resync',
|
||||
{ rootPath, undatedCount, listedNotebooks: notebooks.length }
|
||||
)
|
||||
}
|
||||
|
||||
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
const stubs = notebooks.map((object) => notebookToStub(origin, object))
|
||||
const { documents, indexableCount, capReached } = takeIndexableWithinCap(
|
||||
stubs,
|
||||
isSkippedDocument,
|
||||
maxDocuments,
|
||||
previouslyFetched
|
||||
)
|
||||
|
||||
if (syncContext) syncContext.totalDocsFetched = previouslyFetched + indexableCount
|
||||
|
||||
/**
|
||||
* The cap truncates the listing only when it stopped the walk with directories
|
||||
* still pending, or dropped notebooks from this very page. Reaching the cap on
|
||||
* the last notebook of an exhausted walk hides nothing, so deletion
|
||||
* reconciliation stays enabled in that case.
|
||||
*/
|
||||
const hitLimit = capReached && (queue.length > 0 || documents.length < stubs.length)
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
const hasMore = !hitLimit && queue.length > 0
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? encodeCursor({ queue }) : undefined,
|
||||
hasMore,
|
||||
}
|
||||
}
|
||||
|
||||
/** Pages through the workspace's saved SQL queries. */
|
||||
async function listQueries(
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> {
|
||||
const origin = resolveWorkspaceOrigin(sourceConfig)
|
||||
const maxDocuments = resolveMaxDocuments(sourceConfig.maxDocuments)
|
||||
|
||||
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
|
||||
/**
|
||||
* Ask for only what the cap still allows, so the last page of a capped sync
|
||||
* does not pull a full 100 rows to discard most of them. Trashed and empty
|
||||
* rows are dropped after the fetch, so a short page can still leave the cap
|
||||
* unfilled — `next_page_token` then simply carries the sync into another page.
|
||||
*/
|
||||
const remaining =
|
||||
maxDocuments > 0 ? Math.max(1, maxDocuments - previouslyFetched) : QUERY_PAGE_SIZE
|
||||
const params: Record<string, string> = {
|
||||
page_size: String(Math.min(QUERY_PAGE_SIZE, remaining)),
|
||||
}
|
||||
if (cursor) params.page_token = cursor
|
||||
|
||||
logger.info('Listing Databricks SQL queries', { cursor: cursor ?? 'initial', maxDocuments })
|
||||
|
||||
const response = await databricksGet(origin, '/api/2.0/sql/queries', accessToken, params)
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await readErrorBody(response)
|
||||
throw new Error(
|
||||
`Failed to list Databricks SQL queries: ${describeError(response.status, body)}`
|
||||
)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as DatabricksQueryListResponse
|
||||
const results = data.results ?? []
|
||||
|
||||
/**
|
||||
* A trashed query is Databricks' delete: the row stays listable but is no
|
||||
* longer part of the workspace. Dropping it here lets deletion reconciliation
|
||||
* remove any copy already in the knowledge base, so `listingCapped` must NOT
|
||||
* be set for it.
|
||||
*/
|
||||
const stubs = results
|
||||
.filter((query) => query.lifecycle_state !== 'TRASHED')
|
||||
.map(queryToDocument)
|
||||
.filter((document): document is ExternalDocument => document !== null)
|
||||
|
||||
const { documents, indexableCount, capReached } = takeIndexableWithinCap(
|
||||
stubs,
|
||||
isSkippedDocument,
|
||||
maxDocuments,
|
||||
previouslyFetched
|
||||
)
|
||||
|
||||
if (syncContext) syncContext.totalDocsFetched = previouslyFetched + indexableCount
|
||||
|
||||
/**
|
||||
* `next_page_token` is only a usable cursor when this page actually returned
|
||||
* results — an echoed token on an empty tail would re-request the same page
|
||||
* until the engine truncates pagination, permanently disabling deletion
|
||||
* reconciliation.
|
||||
*/
|
||||
const nextPageToken = data.next_page_token?.trim() || undefined
|
||||
const sourceHasMore = Boolean(nextPageToken) && results.length > 0
|
||||
|
||||
const hitLimit = capReached && (sourceHasMore || documents.length < stubs.length)
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
const hasMore = !hitLimit && sourceHasMore
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? nextPageToken : undefined,
|
||||
hasMore,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Exports a notebook and returns its decoded source.
|
||||
*
|
||||
* `workspace/export` answers with base64 in `content` (no `direct_download`, so
|
||||
* the JSON form) and caps a single export at `DATABRICKS_MAX_EXPORT_BYTES`, past
|
||||
* which it fails with `MAX_NOTEBOOK_SIZE_EXCEEDED`. That case is reported as an
|
||||
* oversize skip rather than an error so the notebook stays visible in the
|
||||
* knowledge base.
|
||||
*/
|
||||
async function exportNotebook(
|
||||
origin: string,
|
||||
accessToken: string,
|
||||
path: string
|
||||
): Promise<{ content: string } | { skippedReason: string } | null> {
|
||||
const response = await databricksGet(origin, '/api/2.0/workspace/export', accessToken, {
|
||||
path,
|
||||
format: 'SOURCE',
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await readErrorBody(response)
|
||||
if (response.status === 404 || body.error_code === 'RESOURCE_DOES_NOT_EXIST') return null
|
||||
if (body.error_code === 'MAX_NOTEBOOK_SIZE_EXCEEDED') {
|
||||
return { skippedReason: sizeLimitSkipReason(DATABRICKS_MAX_EXPORT_BYTES) }
|
||||
}
|
||||
throw new Error(
|
||||
`Failed to export Databricks notebook ${path}: ${describeError(response.status, body)}`
|
||||
)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as DatabricksExportResponse
|
||||
if (!data.content) return null
|
||||
|
||||
const decoded = Buffer.from(data.content, 'base64')
|
||||
if (decoded.byteLength > CONNECTOR_MAX_FILE_BYTES) {
|
||||
return { skippedReason: sizeLimitSkipReason(CONNECTOR_MAX_FILE_BYTES) }
|
||||
}
|
||||
|
||||
return { content: decoded.toString('utf8') }
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-hydrates one notebook.
|
||||
*
|
||||
* The stub is rebuilt from `workspace/get-status` — the same `ObjectInfo` shape
|
||||
* `workspace/list` returns — rather than assembled by hand, so the hash produced
|
||||
* here is identical to the listing's by construction.
|
||||
*/
|
||||
async function getNotebook(
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
path: string
|
||||
): Promise<ExternalDocument | null> {
|
||||
const origin = resolveWorkspaceOrigin(sourceConfig)
|
||||
|
||||
const statusResponse = await databricksGet(origin, '/api/2.0/workspace/get-status', accessToken, {
|
||||
path,
|
||||
})
|
||||
|
||||
if (!statusResponse.ok) {
|
||||
const body = await readErrorBody(statusResponse)
|
||||
if (statusResponse.status === 404 || body.error_code === 'RESOURCE_DOES_NOT_EXIST') return null
|
||||
throw new Error(
|
||||
`Failed to read Databricks notebook status for ${path}: ${describeError(statusResponse.status, body)}`
|
||||
)
|
||||
}
|
||||
|
||||
const object = (await statusResponse.json()) as DatabricksObjectInfo
|
||||
if (object.object_type !== 'NOTEBOOK') return null
|
||||
|
||||
const stub = notebookToStub(origin, { ...object, path: object.path ?? path })
|
||||
|
||||
const exported = await exportNotebook(origin, accessToken, path)
|
||||
if (exported === null) return null
|
||||
if ('skippedReason' in exported) return markSkipped(stub, exported.skippedReason)
|
||||
if (!exported.content.trim()) return null
|
||||
|
||||
return { ...stub, content: exported.content, contentDeferred: false }
|
||||
}
|
||||
|
||||
/** Re-fetches one saved SQL query by id. */
|
||||
async function getQuery(
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
queryId: string
|
||||
): Promise<ExternalDocument | null> {
|
||||
const origin = resolveWorkspaceOrigin(sourceConfig)
|
||||
|
||||
const response = await databricksGet(
|
||||
origin,
|
||||
`/api/2.0/sql/queries/${encodeURIComponent(queryId)}`,
|
||||
accessToken,
|
||||
{}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await readErrorBody(response)
|
||||
if (response.status === 404 || body.error_code === 'RESOURCE_DOES_NOT_EXIST') return null
|
||||
throw new Error(
|
||||
`Failed to fetch Databricks SQL query ${queryId}: ${describeError(response.status, body)}`
|
||||
)
|
||||
}
|
||||
|
||||
const query = (await response.json()) as DatabricksQuery
|
||||
if (query.lifecycle_state === 'TRASHED') return null
|
||||
|
||||
return queryToDocument({ ...query, id: query.id ?? queryId })
|
||||
}
|
||||
|
||||
export const databricksConnector: ConnectorConfig = {
|
||||
...databricksConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
return resolveContentType(sourceConfig) === DATABRICKS_CONTENT_TYPES.queries
|
||||
? listQueries(accessToken, sourceConfig, cursor, syncContext)
|
||||
: listNotebooks(accessToken, sourceConfig, cursor, syncContext)
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
if (externalId.startsWith(QUERY_ID_PREFIX)) {
|
||||
return await getQuery(accessToken, sourceConfig, externalId.slice(QUERY_ID_PREFIX.length))
|
||||
}
|
||||
if (externalId.startsWith(NOTEBOOK_ID_PREFIX)) {
|
||||
return await getNotebook(
|
||||
accessToken,
|
||||
sourceConfig,
|
||||
externalId.slice(NOTEBOOK_ID_PREFIX.length)
|
||||
)
|
||||
}
|
||||
return null
|
||||
} catch (error) {
|
||||
/**
|
||||
* Only an explicit "gone" above returns null. Everything else — 429, 5xx,
|
||||
* network faults — is rethrown so the sync engine records a failed row and
|
||||
* keeps the already-indexed document out of deletion reconciliation.
|
||||
*/
|
||||
logger.warn('Failed to get Databricks document', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
throw toError(error)
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const hostResult = validateDatabricksWorkspaceHost(
|
||||
sourceConfig.workspaceHost as string | undefined
|
||||
)
|
||||
if (!hostResult.isValid || !hostResult.sanitized) {
|
||||
return { valid: false, error: hostResult.error || 'Invalid Databricks workspace host' }
|
||||
}
|
||||
const origin = hostResult.sanitized
|
||||
|
||||
const maxDocuments = sourceConfig.maxDocuments
|
||||
if (maxDocuments !== undefined && maxDocuments !== null && String(maxDocuments).trim()) {
|
||||
const parsed = Number(maxDocuments)
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
return { valid: false, error: 'Max documents must be a positive number' }
|
||||
}
|
||||
}
|
||||
|
||||
const contentType = resolveContentType(sourceConfig)
|
||||
|
||||
try {
|
||||
if (contentType === DATABRICKS_CONTENT_TYPES.queries) {
|
||||
const response = await databricksGet(
|
||||
origin,
|
||||
'/api/2.0/sql/queries',
|
||||
accessToken,
|
||||
{ page_size: '1' },
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
if (!response.ok) {
|
||||
const body = await readErrorBody(response)
|
||||
return {
|
||||
valid: false,
|
||||
error: `Databricks SQL queries access failed: ${describeError(response.status, body)}`,
|
||||
}
|
||||
}
|
||||
return { valid: true }
|
||||
}
|
||||
|
||||
const rootPath = normalizeRootPath(sourceConfig.rootPath)
|
||||
const response = await databricksGet(
|
||||
origin,
|
||||
'/api/2.0/workspace/get-status',
|
||||
accessToken,
|
||||
{ path: rootPath },
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await readErrorBody(response)
|
||||
return {
|
||||
valid: false,
|
||||
error: `Databricks workspace access failed for ${rootPath}: ${describeError(response.status, body)}`,
|
||||
}
|
||||
}
|
||||
|
||||
const object = (await response.json()) as DatabricksObjectInfo
|
||||
if (
|
||||
object.object_type &&
|
||||
object.object_type !== 'DIRECTORY' &&
|
||||
object.object_type !== 'REPO'
|
||||
) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `${rootPath} is a ${object.object_type.toLowerCase()}, not a folder. Enter the folder that contains the notebooks.`,
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
return { valid: false, error: getErrorMessage(error, 'Failed to validate configuration') }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.language === 'string' && metadata.language.trim()) {
|
||||
result.language = metadata.language
|
||||
}
|
||||
|
||||
if (typeof metadata.owner === 'string' && metadata.owner.trim()) {
|
||||
result.owner = metadata.owner
|
||||
}
|
||||
|
||||
if (typeof metadata.catalog === 'string' && metadata.catalog.trim()) {
|
||||
result.catalog = metadata.catalog
|
||||
}
|
||||
|
||||
if (typeof metadata.schema === 'string' && metadata.schema.trim()) {
|
||||
result.schema = metadata.schema
|
||||
}
|
||||
|
||||
const labels = joinTagArray(metadata.labels)
|
||||
if (labels) result.labels = labels
|
||||
|
||||
/**
|
||||
* Both stubs normalize their source timestamp to an ISO string — notebooks
|
||||
* from `modified_at` (epoch milliseconds), queries from `update_time` — so
|
||||
* `parseTagDate`, which only accepts strings, resolves both. Notebooks that
|
||||
* carry no timestamp simply produce no tag.
|
||||
*/
|
||||
const lastModified = parseTagDate(metadata.lastModified)
|
||||
if (lastModified) result.lastModified = lastModified
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { databricksConnector } from '@/connectors/databricks/databricks'
|
||||
@@ -0,0 +1,86 @@
|
||||
import { DatabricksIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
/** Default root of the notebook walk when the user leaves `rootPath` empty. */
|
||||
export const DEFAULT_NOTEBOOK_ROOT_PATH = '/'
|
||||
|
||||
/** Content kinds this connector can index, keyed by the `contentType` config field. */
|
||||
export const DATABRICKS_CONTENT_TYPES = {
|
||||
notebooks: 'notebooks',
|
||||
queries: 'queries',
|
||||
} as const
|
||||
|
||||
export type DatabricksContentType =
|
||||
(typeof DATABRICKS_CONTENT_TYPES)[keyof typeof DATABRICKS_CONTENT_TYPES]
|
||||
|
||||
export const databricksConnectorMeta: ConnectorMeta = {
|
||||
id: 'databricks',
|
||||
name: 'Databricks',
|
||||
description: 'Sync notebooks and saved SQL queries from Databricks into your knowledge base',
|
||||
version: '1.0.0',
|
||||
icon: DatabricksIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'Personal Access Token',
|
||||
placeholder: 'Enter your Databricks personal access token',
|
||||
},
|
||||
|
||||
/**
|
||||
* `GET /api/2.0/workspace/list` documents `created_at`, `modified_at` and `size`
|
||||
* as "only applicable to files", so a `NOTEBOOK` entry can come back with no
|
||||
* change indicator at all. Its metadata-derived `contentHash` then stays
|
||||
* identical across edits and an incremental sync would never re-export it.
|
||||
* Opting into full-resync rehydration gives users a way to pick those edits up.
|
||||
*/
|
||||
rehydrateOnFullSync: true,
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'workspaceHost',
|
||||
title: 'Workspace Host',
|
||||
type: 'short-input',
|
||||
placeholder: 'dbc-1234abcd-5678.cloud.databricks.com',
|
||||
required: true,
|
||||
description:
|
||||
'Your per-workspace Databricks URL. Azure and GCP hosts (adb-*.azuredatabricks.net, *.gcp.databricks.com) are also accepted.',
|
||||
},
|
||||
{
|
||||
id: 'contentType',
|
||||
title: 'Content Type',
|
||||
type: 'dropdown',
|
||||
required: true,
|
||||
description: 'What to sync from the workspace. Add a second connector to sync both.',
|
||||
options: [
|
||||
{ label: 'Notebooks', id: DATABRICKS_CONTENT_TYPES.notebooks },
|
||||
{ label: 'Saved SQL Queries', id: DATABRICKS_CONTENT_TYPES.queries },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'rootPath',
|
||||
title: 'Notebook Root Path',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: '/Users/you@example.com (default: /)',
|
||||
description:
|
||||
'Absolute workspace path to walk for notebooks. Ignored when syncing saved SQL queries.',
|
||||
},
|
||||
{
|
||||
id: 'maxDocuments',
|
||||
title: 'Max Documents',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
description: 'Cap the number of documents synced. Leave empty to sync everything.',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'language', displayName: 'Language', fieldType: 'text' },
|
||||
{ id: 'owner', displayName: 'Owner', fieldType: 'text' },
|
||||
{ id: 'catalog', displayName: 'Catalog', fieldType: 'text' },
|
||||
{ id: 'schema', displayName: 'Schema', fieldType: 'text' },
|
||||
{ id: 'labels', displayName: 'Tags', fieldType: 'text' },
|
||||
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { googleChatConnector } from '@/connectors/google-chat/google-chat'
|
||||
import { googleChatConnectorMeta } from '@/connectors/google-chat/meta'
|
||||
|
||||
const SPACE_NAME = 'spaces/AAAA1111'
|
||||
|
||||
const SPACE = {
|
||||
name: SPACE_NAME,
|
||||
displayName: 'Platform Team',
|
||||
spaceType: 'SPACE' as const,
|
||||
spaceUri: 'https://mail.google.com/chat/u/0/#chat/space/AAAA1111',
|
||||
lastActiveTime: '2026-02-01T10:30:00Z',
|
||||
}
|
||||
|
||||
const MESSAGES = [
|
||||
{
|
||||
name: `${SPACE_NAME}/messages/m2`,
|
||||
sender: { name: 'users/2', displayName: 'Grace Hopper' },
|
||||
createTime: '2026-02-01T10:30:00Z',
|
||||
text: 'Shipping today',
|
||||
},
|
||||
{
|
||||
name: `${SPACE_NAME}/messages/m1`,
|
||||
sender: { name: 'users/1', displayName: 'Ada Lovelace' },
|
||||
createTime: '2026-02-01T10:00:00Z',
|
||||
text: 'Morning',
|
||||
},
|
||||
]
|
||||
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
})
|
||||
}
|
||||
|
||||
const requestedUrls: string[] = []
|
||||
const fetchMock = vi.fn<(input: string | URL | Request, init?: RequestInit) => Promise<Response>>()
|
||||
|
||||
/** Spaces returned by `spaces.list`; per-test overridable. */
|
||||
let listedSpaces: Record<string, unknown>[] = [SPACE]
|
||||
/** `nextPageToken` returned by `spaces.list`; per-test overridable. */
|
||||
let listNextPageToken: string | undefined
|
||||
/** Messages returned by `spaces.messages.list`; per-test overridable. */
|
||||
let listedMessages: Record<string, unknown>[] = MESSAGES
|
||||
/** Space returned by `spaces.list` / `spaces.get`; per-test overridable. */
|
||||
let fetchedSpace: Record<string, unknown> = SPACE
|
||||
|
||||
beforeEach(() => {
|
||||
requestedUrls.length = 0
|
||||
listedSpaces = [SPACE]
|
||||
listNextPageToken = undefined
|
||||
listedMessages = MESSAGES
|
||||
fetchedSpace = SPACE
|
||||
fetchMock.mockReset()
|
||||
fetchMock.mockImplementation(async (input) => {
|
||||
const url = String(input)
|
||||
requestedUrls.push(url)
|
||||
if (url.includes('/messages?')) return jsonResponse({ messages: listedMessages })
|
||||
if (url.includes('/spaces?')) {
|
||||
return jsonResponse({ spaces: listedSpaces, nextPageToken: listNextPageToken })
|
||||
}
|
||||
if (url.endsWith(`/${SPACE_NAME}`)) return jsonResponse(fetchedSpace)
|
||||
return jsonResponse({ error: { message: 'not found' } }, 404)
|
||||
})
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
/** The decoded `filter` the last `spaces.list` request carried, if any. */
|
||||
function listFilter(): string | null {
|
||||
const listUrl = requestedUrls.find((url) => url.includes('/spaces?'))
|
||||
return listUrl ? new URL(listUrl).searchParams.get('filter') : null
|
||||
}
|
||||
|
||||
/** Search params of the first `spaces.messages.list` request. */
|
||||
function messagesParams(): URLSearchParams {
|
||||
const url = requestedUrls.find((requested) => requested.includes('/messages?'))
|
||||
if (!url) throw new Error('no messages request was made')
|
||||
return new URL(url).searchParams
|
||||
}
|
||||
|
||||
describe('google-chat space scope', () => {
|
||||
it('filters to named spaces only when spaceTypes is unset', async () => {
|
||||
await googleChatConnector.listDocuments('token', {})
|
||||
expect(listFilter()).toBe('spaceType = "SPACE"')
|
||||
})
|
||||
|
||||
it('filters to named spaces only when spaceTypes is an empty string', async () => {
|
||||
await googleChatConnector.listDocuments('token', { spaceTypes: '' })
|
||||
expect(listFilter()).toBe('spaceType = "SPACE"')
|
||||
})
|
||||
|
||||
it('widens the filter to group chats without reaching direct messages', async () => {
|
||||
await googleChatConnector.listDocuments('token', { spaceTypes: 'SPACE_AND_GROUP_CHAT' })
|
||||
expect(listFilter()).toBe('spaceType = "SPACE" OR spaceType = "GROUP_CHAT"')
|
||||
})
|
||||
|
||||
it('omits the filter only when every conversation type is requested', async () => {
|
||||
await googleChatConnector.listDocuments('token', { spaceTypes: 'ALL' })
|
||||
expect(listFilter()).toBeNull()
|
||||
})
|
||||
|
||||
it('rejects an unsupported space type selection', async () => {
|
||||
await expect(
|
||||
googleChatConnector.validateConfig('token', { spaceTypes: 'DIRECT_MESSAGE' })
|
||||
).resolves.toEqual({ valid: false, error: 'Unsupported space type selection' })
|
||||
})
|
||||
})
|
||||
|
||||
describe('google-chat change detection', () => {
|
||||
it('hashes on lastActiveTime and keeps the stub and hydrated hash identical', async () => {
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const listed = await googleChatConnector.listDocuments('token', {}, undefined, syncContext)
|
||||
const stub = listed.documents[0]
|
||||
expect(stub.contentDeferred).toBe(true)
|
||||
expect(stub.contentHash).toContain(SPACE.lastActiveTime)
|
||||
|
||||
const hydrated = await googleChatConnector.getDocument('token', {}, SPACE_NAME, syncContext)
|
||||
expect(hydrated?.contentHash).toBe(stub.contentHash)
|
||||
expect(hydrated?.contentDeferred).toBe(false)
|
||||
expect(hydrated?.content).toContain('Ada Lovelace: Morning')
|
||||
expect(hydrated?.content).toContain('Grace Hopper: Shipping today')
|
||||
})
|
||||
|
||||
it('reuses the listed space rather than re-reading it during hydration', async () => {
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
await googleChatConnector.listDocuments('token', {}, undefined, syncContext)
|
||||
requestedUrls.length = 0
|
||||
|
||||
await googleChatConnector.getDocument('token', {}, SPACE_NAME, syncContext)
|
||||
expect(requestedUrls.some((url) => url.endsWith(`/${SPACE_NAME}`))).toBe(false)
|
||||
})
|
||||
|
||||
it('produces a stable hash across syncs while lastActiveTime is unchanged', async () => {
|
||||
const first = await googleChatConnector.listDocuments('token', {}, undefined, {})
|
||||
const second = await googleChatConnector.listDocuments('token', {}, undefined, {})
|
||||
expect(second.documents[0].contentHash).toBe(first.documents[0].contentHash)
|
||||
})
|
||||
|
||||
it('re-hydrates every sync when a space has no lastActiveTime', async () => {
|
||||
listedSpaces = [{ ...SPACE, lastActiveTime: undefined }]
|
||||
|
||||
const runOne: Record<string, unknown> = {}
|
||||
const firstPage = await googleChatConnector.listDocuments('token', {}, undefined, runOne)
|
||||
const runTwo: Record<string, unknown> = {}
|
||||
const secondPage = await googleChatConnector.listDocuments('token', {}, undefined, runTwo)
|
||||
|
||||
// Stable within a run, so the hydrated document keeps the stub's hash …
|
||||
const hydrated = await googleChatConnector.getDocument('token', {}, SPACE_NAME, runOne)
|
||||
expect(hydrated?.contentHash).toBe(firstPage.documents[0].contentHash)
|
||||
|
||||
// … and different on the next run, so content never silently goes stale.
|
||||
expect(secondPage.documents[0].contentHash).not.toBe(firstPage.documents[0].contentHash)
|
||||
})
|
||||
|
||||
it('declares rehydrateOnFullSync so edits and deletions are recoverable', () => {
|
||||
expect(googleChatConnectorMeta.rehydrateOnFullSync).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('google-chat listing caps', () => {
|
||||
it('flags the listing as capped only when the cap truncated a larger source', async () => {
|
||||
listedSpaces = [SPACE, { ...SPACE, name: 'spaces/BBBB2222' }]
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
|
||||
const result = await googleChatConnector.listDocuments(
|
||||
'token',
|
||||
{ maxSpaces: '1' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
expect(result.documents).toHaveLength(1)
|
||||
expect(result.hasMore).toBe(false)
|
||||
expect(syncContext.listingCapped).toBe(true)
|
||||
})
|
||||
|
||||
it('stops paginating at the cap even while the source offers another page', async () => {
|
||||
listNextPageToken = 'page-2'
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
|
||||
const result = await googleChatConnector.listDocuments(
|
||||
'token',
|
||||
{ maxSpaces: '1' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
expect(result.hasMore).toBe(false)
|
||||
expect(result.nextCursor).toBeUndefined()
|
||||
expect(syncContext.listingCapped).toBe(true)
|
||||
})
|
||||
|
||||
it('leaves deletion reconciliation enabled when the source is exhausted at the cap', async () => {
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
await googleChatConnector.listDocuments('token', { maxSpaces: '1' }, undefined, syncContext)
|
||||
expect(syncContext.listingCapped).toBeUndefined()
|
||||
})
|
||||
|
||||
it('paginates with the returned nextPageToken', async () => {
|
||||
listNextPageToken = 'page-2'
|
||||
const result = await googleChatConnector.listDocuments('token', {})
|
||||
expect(result.hasMore).toBe(true)
|
||||
expect(result.nextCursor).toBe('page-2')
|
||||
})
|
||||
})
|
||||
|
||||
describe('google-chat empty windows', () => {
|
||||
it('indexes a space whose only prose is its description when no message has text', async () => {
|
||||
fetchedSpace = { ...SPACE, spaceDetails: { description: 'Release coordination' } }
|
||||
listedSpaces = [fetchedSpace]
|
||||
listedMessages = []
|
||||
|
||||
const doc = await googleChatConnector.getDocument('token', {}, SPACE_NAME)
|
||||
expect(doc).not.toBeNull()
|
||||
expect(doc?.content).toContain('Release coordination')
|
||||
})
|
||||
|
||||
it('indexes a space whose only prose is its guidelines when no message has text', async () => {
|
||||
fetchedSpace = { ...SPACE, spaceDetails: { guidelines: 'Be excellent to each other' } }
|
||||
listedSpaces = [fetchedSpace]
|
||||
listedMessages = []
|
||||
|
||||
const doc = await googleChatConnector.getDocument('token', {}, SPACE_NAME)
|
||||
expect(doc?.content).toContain('Be excellent to each other')
|
||||
})
|
||||
|
||||
it('returns a document rather than null when the window is empty, so a cleared space does not keep a stale transcript', async () => {
|
||||
listedMessages = []
|
||||
const doc = await googleChatConnector.getDocument('token', {}, SPACE_NAME)
|
||||
expect(doc).not.toBeNull()
|
||||
expect(doc?.content).not.toContain('Shipping today')
|
||||
expect(doc?.metadata?.messageCount).toBe(0)
|
||||
})
|
||||
|
||||
it('omits the transcript header entirely when no message contributed text', async () => {
|
||||
listedMessages = []
|
||||
const doc = await googleChatConnector.getDocument('token', {}, SPACE_NAME)
|
||||
expect(doc?.content).not.toContain('--- Messages ---')
|
||||
})
|
||||
|
||||
it('still returns null when the space itself is gone', async () => {
|
||||
const doc = await googleChatConnector.getDocument('token', {}, 'spaces/MISSING')
|
||||
expect(doc).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('google-chat message window', () => {
|
||||
it('requests messages newest-first so the cap keeps the most recent conversation', async () => {
|
||||
await googleChatConnector.getDocument('token', {}, SPACE_NAME)
|
||||
expect(messagesParams().get('orderBy')).toBe('createTime DESC')
|
||||
})
|
||||
|
||||
it('renders the newest-first page back into chronological order', async () => {
|
||||
const doc = await googleChatConnector.getDocument('token', {}, SPACE_NAME)
|
||||
const content = doc?.content ?? ''
|
||||
expect(content.indexOf('Ada Lovelace')).toBeLessThan(content.indexOf('Grace Hopper'))
|
||||
})
|
||||
|
||||
it('bounds the message page to the remaining window rather than the API maximum', async () => {
|
||||
await googleChatConnector.getDocument('token', { maxMessages: '5' }, SPACE_NAME)
|
||||
expect(messagesParams().get('pageSize')).toBe('5')
|
||||
})
|
||||
|
||||
it('sends the lookback cutoff as an RFC-3339 timestamp without fractional seconds', async () => {
|
||||
await googleChatConnector.getDocument('token', { lookbackDays: '30' }, SPACE_NAME)
|
||||
const filter = messagesParams().get('filter')
|
||||
expect(filter).toMatch(/^createTime > "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z"$/)
|
||||
})
|
||||
|
||||
it('omits the filter entirely when no lookback window is configured', async () => {
|
||||
await googleChatConnector.getDocument('token', {}, SPACE_NAME)
|
||||
expect(messagesParams().get('filter')).toBeNull()
|
||||
})
|
||||
|
||||
it('indexes a card-only message through its fallbackText', async () => {
|
||||
fetchMock.mockImplementation(async (input) => {
|
||||
const url = String(input)
|
||||
requestedUrls.push(url)
|
||||
if (url.includes('/messages?')) {
|
||||
return jsonResponse({
|
||||
messages: [
|
||||
{
|
||||
name: `${SPACE_NAME}/messages/card`,
|
||||
sender: { name: 'users/9', type: 'BOT' },
|
||||
createTime: '2026-02-01T09:00:00Z',
|
||||
fallbackText: 'Deploy finished',
|
||||
},
|
||||
],
|
||||
})
|
||||
}
|
||||
if (url.endsWith(`/${SPACE_NAME}`)) return jsonResponse(SPACE)
|
||||
return jsonResponse({}, 404)
|
||||
})
|
||||
|
||||
const doc = await googleChatConnector.getDocument('token', {}, SPACE_NAME)
|
||||
expect(doc?.content).toContain('Deploy finished')
|
||||
expect(doc?.metadata?.messageCount).toBe(1)
|
||||
})
|
||||
|
||||
it('labels a sender by resource name when user auth omits displayName', async () => {
|
||||
fetchMock.mockImplementation(async (input) => {
|
||||
const url = String(input)
|
||||
requestedUrls.push(url)
|
||||
if (url.includes('/messages?')) {
|
||||
return jsonResponse({
|
||||
messages: [
|
||||
{
|
||||
name: `${SPACE_NAME}/messages/m1`,
|
||||
sender: { name: 'users/104512345678', type: 'HUMAN' },
|
||||
createTime: '2026-02-01T10:00:00Z',
|
||||
text: 'Morning',
|
||||
},
|
||||
],
|
||||
})
|
||||
}
|
||||
if (url.endsWith(`/${SPACE_NAME}`)) return jsonResponse(SPACE)
|
||||
return jsonResponse({}, 404)
|
||||
})
|
||||
|
||||
const doc = await googleChatConnector.getDocument('token', {}, SPACE_NAME)
|
||||
expect(doc?.content).toContain('users/104512345678: Morning')
|
||||
})
|
||||
|
||||
it('rehashes when the configured window changes so the stored transcript is refetched', async () => {
|
||||
const base = await googleChatConnector.listDocuments('token', {}, undefined, {})
|
||||
const narrower = await googleChatConnector.listDocuments(
|
||||
'token',
|
||||
{ maxMessages: '50' },
|
||||
undefined,
|
||||
{}
|
||||
)
|
||||
const windowed = await googleChatConnector.listDocuments(
|
||||
'token',
|
||||
{ lookbackDays: '30' },
|
||||
undefined,
|
||||
{}
|
||||
)
|
||||
|
||||
expect(narrower.documents[0].contentHash).not.toBe(base.documents[0].contentHash)
|
||||
expect(windowed.documents[0].contentHash).not.toBe(base.documents[0].contentHash)
|
||||
})
|
||||
})
|
||||
|
||||
describe('google-chat getDocument', () => {
|
||||
it('returns null for a space that no longer exists', async () => {
|
||||
await expect(googleChatConnector.getDocument('token', {}, 'spaces/MISSING')).resolves.toBeNull()
|
||||
})
|
||||
|
||||
it('accepts a bare space id and normalizes it to a resource name', async () => {
|
||||
const doc = await googleChatConnector.getDocument('token', {}, 'AAAA1111')
|
||||
expect(doc?.externalId).toBe(SPACE_NAME)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,559 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import {
|
||||
DEFAULT_MAX_MESSAGES,
|
||||
googleChatConnectorMeta,
|
||||
MESSAGES_PAGE_SIZE,
|
||||
SPACES_PAGE_SIZE,
|
||||
} from '@/connectors/google-chat/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('GoogleChatConnector')
|
||||
|
||||
const CHAT_API_BASE = 'https://chat.googleapis.com/v1'
|
||||
const MS_PER_DAY = 24 * 60 * 60 * 1000
|
||||
|
||||
/** `syncContext` key holding the spaces this run listed, keyed by resource name. */
|
||||
const SPACE_CACHE_KEY = 'googleChatSpaces'
|
||||
|
||||
/** `syncContext` key holding this run's fallback activity token. */
|
||||
const ACTIVITY_TOKEN_KEY = 'googleChatActivityToken'
|
||||
|
||||
/**
|
||||
* A Google Chat space as returned by `spaces.list` / `spaces.get`. Only the
|
||||
* fields this connector reads are modeled.
|
||||
*
|
||||
* https://developers.google.com/workspace/chat/api/reference/rest/v1/spaces
|
||||
*/
|
||||
interface Space {
|
||||
name: string
|
||||
displayName?: string
|
||||
spaceType?: 'SPACE_TYPE_UNSPECIFIED' | 'SPACE' | 'GROUP_CHAT' | 'DIRECT_MESSAGE'
|
||||
spaceUri?: string
|
||||
createTime?: string
|
||||
/** Output only. Timestamp of the last message in the space. */
|
||||
lastActiveTime?: string
|
||||
spaceDetails?: { description?: string; guidelines?: string }
|
||||
}
|
||||
|
||||
interface SpacesListResponse {
|
||||
spaces?: Space[]
|
||||
nextPageToken?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* The author of a message.
|
||||
*
|
||||
* This connector authenticates as a user, and the User reference states that
|
||||
* "if your Chat app authenticates as a user, the output for a User resource
|
||||
* only populates the user's `name` and `type`" — so `displayName` is expected
|
||||
* to be absent here and `name` (a `users/{id}` resource name) is the label that
|
||||
* actually reaches the transcript. Resolving human names would need
|
||||
* `spaces.members.list` and the `chat.memberships.readonly` scope, which this
|
||||
* connector does not request. `displayName` is still read first so the
|
||||
* transcript improves for free if a call ever does carry it.
|
||||
*
|
||||
* https://developers.google.com/workspace/chat/api/reference/rest/v1/User
|
||||
*/
|
||||
interface ChatUser {
|
||||
name?: string
|
||||
displayName?: string
|
||||
type?: 'TYPE_UNSPECIFIED' | 'HUMAN' | 'BOT'
|
||||
}
|
||||
|
||||
/**
|
||||
* A message in a space. `text` is the plain-text body; `fallbackText` is the
|
||||
* plain-text description of a card-only message.
|
||||
*
|
||||
* https://developers.google.com/workspace/chat/api/reference/rest/v1/spaces.messages
|
||||
*/
|
||||
interface ChatMessage {
|
||||
name: string
|
||||
sender?: ChatUser
|
||||
createTime?: string
|
||||
lastUpdateTime?: string
|
||||
text?: string
|
||||
fallbackText?: string
|
||||
}
|
||||
|
||||
interface MessagesListResponse {
|
||||
messages?: ChatMessage[]
|
||||
nextPageToken?: string
|
||||
}
|
||||
|
||||
function chatHeaders(accessToken: string): Record<string, string> {
|
||||
return { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }
|
||||
}
|
||||
|
||||
/** Trailing id segment of a `spaces/{space}` resource name. */
|
||||
function resourceIdSegment(resourceName: string): string {
|
||||
return resourceName.slice(resourceName.lastIndexOf('/') + 1)
|
||||
}
|
||||
|
||||
/** Normalizes a space identifier to its full resource name, tolerating a bare id. */
|
||||
function spaceResourceName(externalId: string): string {
|
||||
const trimmed = externalId.trim()
|
||||
return trimmed.startsWith('spaces/') ? trimmed : `spaces/${trimmed}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the `spaces.list` filter from the configured scope. The API only
|
||||
* supports filtering on `spaceType`, with `OR` between values; `ALL` omits the
|
||||
* filter entirely so every conversation type is listed.
|
||||
*/
|
||||
function buildSpacesFilter(sourceConfig: Record<string, unknown>): string | undefined {
|
||||
const scope = typeof sourceConfig.spaceTypes === 'string' ? sourceConfig.spaceTypes : 'SPACE'
|
||||
if (scope === 'ALL') return undefined
|
||||
if (scope === 'SPACE_AND_GROUP_CHAT') return 'spaceType = "SPACE" OR spaceType = "GROUP_CHAT"'
|
||||
return 'spaceType = "SPACE"'
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the per-space message window, falling back to the default for
|
||||
* missing, non-numeric, or non-positive values. `validateConfig` rejects those
|
||||
* inputs, but a config saved before validation existed would otherwise yield
|
||||
* `NaN` and index nothing.
|
||||
*/
|
||||
function resolveMaxMessages(value: unknown): number {
|
||||
const parsed = Number(value)
|
||||
return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : DEFAULT_MAX_MESSAGES
|
||||
}
|
||||
|
||||
/** Resolves the lookback window in days, or 0 when unset/invalid (no cutoff). */
|
||||
function resolveLookbackDays(value: unknown): number {
|
||||
const parsed = Number(value)
|
||||
return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Human-readable title for a space. `displayName` is optional and documented as
|
||||
* possibly empty for direct messages, so the conversation type plus the space id
|
||||
* is used as a stable fallback rather than an anonymous "Untitled".
|
||||
*/
|
||||
function spaceTitle(space: Space): string {
|
||||
const displayName = space.displayName?.trim()
|
||||
if (displayName) return displayName
|
||||
const id = resourceIdSegment(space.name)
|
||||
if (space.spaceType === 'DIRECT_MESSAGE') return `Direct message ${id}`
|
||||
if (space.spaceType === 'GROUP_CHAT') return `Group chat ${id}`
|
||||
return `Google Chat space ${id}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Change-detection token for a space.
|
||||
*
|
||||
* `lastActiveTime` is documented as the timestamp of the last message in the
|
||||
* space, so it moves whenever new content arrives without any message having to
|
||||
* be fetched. The `spaces.list` reference documents `permissionSettings` as the
|
||||
* only field omitted from list responses, but it does not positively guarantee
|
||||
* `lastActiveTime` is populated for every space — and a space missing it would
|
||||
* hash to a constant, so its content would never be refreshed again.
|
||||
*
|
||||
* A space without the field therefore falls back to a token that is stable for
|
||||
* one sync run and different on the next, so the space re-hydrates every sync
|
||||
* rather than going silently stale. `listDocuments` warns when that branch is
|
||||
* taken, since it re-indexes the space's content on every sync.
|
||||
*/
|
||||
function activityToken(space: Space, syncContext?: Record<string, unknown>): string {
|
||||
const lastActiveTime = space.lastActiveTime?.trim()
|
||||
if (lastActiveTime) return lastActiveTime
|
||||
|
||||
const cached = syncContext?.[ACTIVITY_TOKEN_KEY]
|
||||
if (typeof cached === 'string') return cached
|
||||
|
||||
const token = `unknown-${generateId()}`
|
||||
if (syncContext) syncContext[ACTIVITY_TOKEN_KEY] = token
|
||||
return token
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata-based change-detection hash. The window settings are folded in
|
||||
* because changing them changes which messages the document contains.
|
||||
*
|
||||
* Known limitation: editing or deleting an existing message does not advance
|
||||
* `lastActiveTime`, so such a change is only picked up once a newer message
|
||||
* lands or an explicit full resync runs — see `rehydrateOnFullSync` in `meta.ts`.
|
||||
*/
|
||||
function buildContentHash(
|
||||
space: Space,
|
||||
maxMessages: number,
|
||||
lookbackDays: number,
|
||||
syncContext?: Record<string, unknown>
|
||||
): string {
|
||||
const token = activityToken(space, syncContext)
|
||||
return `gchat:v1:${space.name}:${token}:${maxMessages}:${lookbackDays}`
|
||||
}
|
||||
|
||||
/** Records the spaces a listing page returned so `getDocument` can reuse them. */
|
||||
function cacheSpaces(spaces: Space[], syncContext?: Record<string, unknown>): void {
|
||||
if (!syncContext) return
|
||||
const existing = syncContext[SPACE_CACHE_KEY]
|
||||
const cache = (existing && typeof existing === 'object' ? existing : {}) as Record<string, Space>
|
||||
for (const space of spaces) cache[space.name] = space
|
||||
syncContext[SPACE_CACHE_KEY] = cache
|
||||
}
|
||||
|
||||
/** The listed space for a resource name, when this sync run listed it. */
|
||||
function cachedSpace(name: string, syncContext?: Record<string, unknown>): Space | undefined {
|
||||
const cache = syncContext?.[SPACE_CACHE_KEY]
|
||||
if (!cache || typeof cache !== 'object') return undefined
|
||||
return (cache as Record<string, Space>)[name]
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the deferred listing stub for a space. Messages are fetched lazily in
|
||||
* `getDocument`; only metadata and the change hash are computed here.
|
||||
*/
|
||||
function spaceToStub(
|
||||
space: Space,
|
||||
maxMessages: number,
|
||||
lookbackDays: number,
|
||||
syncContext?: Record<string, unknown>
|
||||
): ExternalDocument {
|
||||
return {
|
||||
externalId: space.name,
|
||||
title: spaceTitle(space),
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: space.spaceUri,
|
||||
contentHash: buildContentHash(space, maxMessages, lookbackDays, syncContext),
|
||||
metadata: {
|
||||
spaceName: spaceTitle(space),
|
||||
spaceType: space.spaceType,
|
||||
lastActivity: space.lastActiveTime,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** Fetches a single space. Returns null on 404 (space deleted or left). */
|
||||
async function fetchSpace(accessToken: string, name: string): Promise<Space | null> {
|
||||
const response = await fetchWithRetry(`${CHAT_API_BASE}/${name}`, {
|
||||
method: 'GET',
|
||||
headers: chatHeaders(accessToken),
|
||||
})
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error(`Failed to fetch Google Chat space: ${response.status}`)
|
||||
}
|
||||
return (await response.json()) as Space
|
||||
}
|
||||
|
||||
/**
|
||||
* An RFC-3339 timestamp without fractional seconds, matching the form every
|
||||
* example in the `spaces.messages.list` filter reference uses. `toISOString()`
|
||||
* alone emits milliseconds, and the API rejects a filter it cannot parse with
|
||||
* `INVALID_ARGUMENT` rather than ignoring it.
|
||||
*
|
||||
* https://developers.google.com/workspace/chat/api/reference/rest/v1/spaces.messages/list
|
||||
*/
|
||||
function rfc3339(date: Date): string {
|
||||
return `${date.toISOString().slice(0, 19)}Z`
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the newest `maxMessages` messages of a space, optionally bounded by a
|
||||
* lookback window. Messages are requested newest-first so the cap keeps the most
|
||||
* recent conversation, then returned in chronological order.
|
||||
*
|
||||
* `orderBy` takes a full ordering expression, not a bare direction: the reference
|
||||
* documents the default as `createTime ASC` and lists ASC/DESC as the ordering
|
||||
* *operations* usable within one. `createTime` is the only orderable field here.
|
||||
*/
|
||||
async function fetchSpaceMessages(
|
||||
accessToken: string,
|
||||
spaceName: string,
|
||||
maxMessages: number,
|
||||
lookbackDays: number
|
||||
): Promise<ChatMessage[]> {
|
||||
const filter =
|
||||
lookbackDays > 0
|
||||
? `createTime > "${rfc3339(new Date(Date.now() - lookbackDays * MS_PER_DAY))}"`
|
||||
: undefined
|
||||
|
||||
const collected: ChatMessage[] = []
|
||||
let pageToken: string | undefined
|
||||
|
||||
while (collected.length < maxMessages) {
|
||||
const params = new URLSearchParams({
|
||||
pageSize: String(Math.min(MESSAGES_PAGE_SIZE, maxMessages - collected.length)),
|
||||
orderBy: 'createTime DESC',
|
||||
})
|
||||
if (filter) params.set('filter', filter)
|
||||
if (pageToken) params.set('pageToken', pageToken)
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
`${CHAT_API_BASE}/${spaceName}/messages?${params.toString()}`,
|
||||
{ method: 'GET', headers: chatHeaders(accessToken) }
|
||||
)
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) break
|
||||
throw new Error(`Failed to list Google Chat messages: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as MessagesListResponse
|
||||
const messages = data.messages ?? []
|
||||
if (messages.length === 0) break
|
||||
collected.push(...messages)
|
||||
|
||||
pageToken = data.nextPageToken?.trim() || undefined
|
||||
if (!pageToken) break
|
||||
}
|
||||
|
||||
return collected.slice(0, maxMessages).reverse()
|
||||
}
|
||||
|
||||
/** Resolves a message author's label, preferring the display name. */
|
||||
function senderLabel(sender: ChatUser | undefined): string {
|
||||
return sender?.displayName?.trim() || sender?.name?.trim() || 'Unknown'
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders a space's messages as one plain-text transcript. Card-only messages
|
||||
* fall back to `fallbackText`, the documented plain-text description of a
|
||||
* message's cards; messages with neither are skipped as they carry no text.
|
||||
*
|
||||
* Message attachments are deliberately not indexed. Downloading them needs the
|
||||
* separate `media.download` endpoint (and, for Drive-hosted attachments, Drive
|
||||
* scopes this connector does not request), which would turn one document per
|
||||
* space into an unbounded per-message download fan-out. A file shared in Chat
|
||||
* that belongs in a knowledge base is synced through the Google Drive connector
|
||||
* instead, which already handles size caps, OCR, and format parsing.
|
||||
*/
|
||||
function formatSpaceContent(space: Space, messages: ChatMessage[]): string {
|
||||
const parts: string[] = [`Space: ${spaceTitle(space)}`]
|
||||
const description = space.spaceDetails?.description?.trim()
|
||||
if (description) parts.push(`Description: ${description}`)
|
||||
const guidelines = space.spaceDetails?.guidelines?.trim()
|
||||
if (guidelines) parts.push(`Guidelines: ${guidelines}`)
|
||||
|
||||
const lines: string[] = []
|
||||
for (const message of messages) {
|
||||
const text = message.text?.trim() || message.fallbackText?.trim()
|
||||
if (!text) continue
|
||||
const timestamp = message.createTime ?? ''
|
||||
lines.push(`[${timestamp}] ${senderLabel(message.sender)}: ${text}`)
|
||||
}
|
||||
|
||||
if (lines.length > 0) {
|
||||
parts.push('')
|
||||
parts.push('--- Messages ---')
|
||||
parts.push(...lines)
|
||||
}
|
||||
|
||||
return parts.join('\n')
|
||||
}
|
||||
|
||||
/** Number of messages that actually contributed text to the transcript. */
|
||||
function countIndexedMessages(messages: ChatMessage[]): number {
|
||||
let count = 0
|
||||
for (const message of messages) {
|
||||
if (message.text?.trim() || message.fallbackText?.trim()) count++
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
export const googleChatConnector: ConnectorConfig = {
|
||||
...googleChatConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const maxSpaces = sourceConfig.maxSpaces ? Number(sourceConfig.maxSpaces) : 0
|
||||
const maxMessages = resolveMaxMessages(sourceConfig.maxMessages)
|
||||
const lookbackDays = resolveLookbackDays(sourceConfig.lookbackDays)
|
||||
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
|
||||
const pageSize =
|
||||
maxSpaces > 0
|
||||
? Math.min(SPACES_PAGE_SIZE, Math.max(1, maxSpaces - prevFetched))
|
||||
: SPACES_PAGE_SIZE
|
||||
const params = new URLSearchParams({ pageSize: String(pageSize) })
|
||||
if (cursor) params.set('pageToken', cursor)
|
||||
const filter = buildSpacesFilter(sourceConfig)
|
||||
if (filter) params.set('filter', filter)
|
||||
|
||||
logger.info('Listing Google Chat spaces', {
|
||||
hasCursor: Boolean(cursor),
|
||||
hasFilter: Boolean(filter),
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(`${CHAT_API_BASE}/spaces?${params.toString()}`, {
|
||||
method: 'GET',
|
||||
headers: chatHeaders(accessToken),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
logger.error('Failed to list Google Chat spaces', {
|
||||
status: response.status,
|
||||
error: errorText.slice(0, 500),
|
||||
})
|
||||
throw new Error(`Failed to list Google Chat spaces: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as SpacesListResponse
|
||||
const spaces = (data.spaces ?? []).filter((space) => Boolean(space.name))
|
||||
const nextPageToken = data.nextPageToken?.trim() || undefined
|
||||
|
||||
const missingActivity = spaces.filter((space) => !space.lastActiveTime?.trim()).length
|
||||
if (missingActivity > 0) {
|
||||
logger.warn(
|
||||
'Google Chat spaces listed without lastActiveTime; their content re-indexes every sync',
|
||||
{ spacesMissingLastActiveTime: missingActivity, spacesInPage: spaces.length }
|
||||
)
|
||||
}
|
||||
|
||||
cacheSpaces(spaces, syncContext)
|
||||
|
||||
const allDocuments = spaces.map((space) =>
|
||||
spaceToStub(space, maxMessages, lookbackDays, syncContext)
|
||||
)
|
||||
|
||||
let documents = allDocuments
|
||||
if (maxSpaces > 0) {
|
||||
const remaining = Math.max(0, maxSpaces - prevFetched)
|
||||
if (allDocuments.length > remaining) documents = allDocuments.slice(0, remaining)
|
||||
}
|
||||
|
||||
const totalFetched = prevFetched + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const reachedCap = maxSpaces > 0 && totalFetched >= maxSpaces
|
||||
|
||||
/**
|
||||
* Only flag the listing as capped when the cap actually truncated a larger
|
||||
* source — either more pages remain, or spaces were dropped from this page.
|
||||
* A source that was fully listed and merely happens to equal the cap stays
|
||||
* unflagged so the sync engine still reconciles deletions.
|
||||
*/
|
||||
const truncated =
|
||||
reachedCap && (Boolean(nextPageToken) || allDocuments.length > documents.length)
|
||||
if (truncated && syncContext) syncContext.listingCapped = true
|
||||
|
||||
const hasMore = !reachedCap && Boolean(nextPageToken)
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? nextPageToken : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocument | null> => {
|
||||
if (!externalId) return null
|
||||
const spaceName = spaceResourceName(externalId)
|
||||
|
||||
/**
|
||||
* Reuses the space the listing already fetched. Beyond saving a `spaces.get`
|
||||
* per document, it is what keeps the hydrated `contentHash` identical to the
|
||||
* stub's: re-reading the space here could observe a `lastActiveTime` that
|
||||
* advanced after the listing, storing a hash for messages this call never
|
||||
* fetched and hiding them until the *next* message arrives.
|
||||
*/
|
||||
const space = cachedSpace(spaceName, syncContext) ?? (await fetchSpace(accessToken, spaceName))
|
||||
if (!space) return null
|
||||
|
||||
const maxMessages = resolveMaxMessages(sourceConfig.maxMessages)
|
||||
const lookbackDays = resolveLookbackDays(sourceConfig.lookbackDays)
|
||||
const messages = await fetchSpaceMessages(accessToken, spaceName, maxMessages, lookbackDays)
|
||||
|
||||
/**
|
||||
* A space with no messages in the window is still a live space, so it is
|
||||
* indexed rather than skipped. `null` is this connector's "document is gone"
|
||||
* signal, and the engine treats it as last-known-good: returning it here would
|
||||
* both drop spaces whose only prose is their description or guidelines, and
|
||||
* leave a previously indexed transcript in place after the space was cleared
|
||||
* or `lookbackDays` was tightened past every message.
|
||||
*/
|
||||
const messageCount = countIndexedMessages(messages)
|
||||
const stub = spaceToStub(space, maxMessages, lookbackDays, syncContext)
|
||||
|
||||
return {
|
||||
...stub,
|
||||
content: formatSpaceContent(space, messages),
|
||||
contentDeferred: false,
|
||||
metadata: { ...stub.metadata, messageCount },
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxMessages = sourceConfig.maxMessages as string | undefined
|
||||
if (maxMessages && (Number.isNaN(Number(maxMessages)) || Number(maxMessages) <= 0)) {
|
||||
return { valid: false, error: 'Max messages per space must be a positive number' }
|
||||
}
|
||||
|
||||
const maxSpaces = sourceConfig.maxSpaces as string | undefined
|
||||
if (maxSpaces && (Number.isNaN(Number(maxSpaces)) || Number(maxSpaces) <= 0)) {
|
||||
return { valid: false, error: 'Max spaces must be a positive number' }
|
||||
}
|
||||
|
||||
const lookbackDays = sourceConfig.lookbackDays as string | undefined
|
||||
if (lookbackDays && (Number.isNaN(Number(lookbackDays)) || Number(lookbackDays) <= 0)) {
|
||||
return { valid: false, error: 'Lookback window must be a positive number of days' }
|
||||
}
|
||||
|
||||
const spaceTypes = sourceConfig.spaceTypes
|
||||
if (
|
||||
spaceTypes != null &&
|
||||
spaceTypes !== '' &&
|
||||
!['SPACE', 'SPACE_AND_GROUP_CHAT', 'ALL'].includes(String(spaceTypes))
|
||||
) {
|
||||
return { valid: false, error: 'Unsupported space type selection' }
|
||||
}
|
||||
|
||||
try {
|
||||
const params = new URLSearchParams({ pageSize: '1' })
|
||||
const filter = buildSpacesFilter(sourceConfig)
|
||||
if (filter) params.set('filter', filter)
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
`${CHAT_API_BASE}/spaces?${params.toString()}`,
|
||||
{ method: 'GET', headers: chatHeaders(accessToken) },
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
return {
|
||||
valid: false,
|
||||
error: `Google Chat access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`,
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
return { valid: false, error: getErrorMessage(error, 'Failed to validate configuration') }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.spaceName === 'string') result.spaceName = metadata.spaceName
|
||||
if (typeof metadata.spaceType === 'string') result.spaceType = metadata.spaceType
|
||||
|
||||
if (metadata.messageCount != null) {
|
||||
const count = Number(metadata.messageCount)
|
||||
if (!Number.isNaN(count)) result.messageCount = count
|
||||
}
|
||||
|
||||
const lastActivity = parseTagDate(metadata.lastActivity)
|
||||
if (lastActivity) result.lastActivity = lastActivity
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { googleChatConnector } from '@/connectors/google-chat/google-chat'
|
||||
@@ -0,0 +1,106 @@
|
||||
import { GoogleChatIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
/**
|
||||
* Default per-space message window. Google Chat spaces are long-lived and a
|
||||
* single document holds one space's history, so the newest N messages are
|
||||
* indexed rather than the entire backlog.
|
||||
*/
|
||||
export const DEFAULT_MAX_MESSAGES = 1000
|
||||
|
||||
/**
|
||||
* Page size for `spaces.list`. The API defaults to 100 and accepts up to 1000;
|
||||
* 100 keeps each listing round-trip small, since a sync walks every page anyway.
|
||||
*
|
||||
* https://developers.google.com/workspace/chat/api/reference/rest/v1/spaces/list
|
||||
*/
|
||||
export const SPACES_PAGE_SIZE = 100
|
||||
|
||||
/**
|
||||
* Page size for `spaces.messages.list`. The API defaults to 25 and accepts up to
|
||||
* 1000; the documented maximum is used so a space's message window is fetched in
|
||||
* as few round-trips as possible.
|
||||
*
|
||||
* https://developers.google.com/workspace/chat/api/reference/rest/v1/spaces.messages/list
|
||||
*/
|
||||
export const MESSAGES_PAGE_SIZE = 1000
|
||||
|
||||
export const googleChatConnectorMeta: ConnectorMeta = {
|
||||
id: 'google_chat',
|
||||
name: 'Google Chat',
|
||||
description: 'Sync space conversations from Google Chat into your knowledge base',
|
||||
version: '1.0.0',
|
||||
icon: GoogleChatIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'google-chat',
|
||||
requiredScopes: [
|
||||
'https://www.googleapis.com/auth/chat.spaces.readonly',
|
||||
'https://www.googleapis.com/auth/chat.messages.readonly',
|
||||
],
|
||||
},
|
||||
|
||||
/**
|
||||
* A space document's `contentHash` is keyed on `lastActiveTime`, documented as
|
||||
* the timestamp of the last message in the space. Editing or deleting an
|
||||
* existing message does not move it, and the Chat API exposes no space-level
|
||||
* revision counter to key on instead, so a routine hash-gated sync cannot see
|
||||
* an edit-only or delete-only change until a newer message lands.
|
||||
*
|
||||
* Routine syncs therefore stay cheap, and an explicit full resync re-hydrates
|
||||
* every space (one messages listing per space) so edits and deletions are
|
||||
* picked up on demand. That cost is paid on every full resync, not once.
|
||||
*/
|
||||
rehydrateOnFullSync: true,
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'spaceTypes',
|
||||
title: 'Space Types',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Named spaces only (default)', id: 'SPACE' },
|
||||
{ label: 'Named spaces and group chats', id: 'SPACE_AND_GROUP_CHAT' },
|
||||
{ label: 'All conversations, including direct messages', id: 'ALL' },
|
||||
],
|
||||
description:
|
||||
'Which conversations to sync. Direct messages are one-to-one conversations of the connected account — indexing them makes their contents searchable by everyone with access to this knowledge base.',
|
||||
},
|
||||
{
|
||||
id: 'maxMessages',
|
||||
title: 'Max Messages Per Space',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: `e.g. 500 (default: ${DEFAULT_MAX_MESSAGES})`,
|
||||
description:
|
||||
'Number of most recent messages indexed per space. Older messages are not included.',
|
||||
},
|
||||
{
|
||||
id: 'maxSpaces',
|
||||
title: 'Max Spaces',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. 100 (default: unlimited)',
|
||||
description: 'Cap the total number of spaces synced. Leave blank to sync all of them.',
|
||||
},
|
||||
{
|
||||
id: 'lookbackDays',
|
||||
title: 'Lookback Window (days)',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. 90 (default: all available)',
|
||||
description: 'Only index messages created in the last N days.',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'spaceName', displayName: 'Space Name', fieldType: 'text' },
|
||||
{ id: 'spaceType', displayName: 'Space Type', fieldType: 'text' },
|
||||
{ id: 'messageCount', displayName: 'Message Count', fieldType: 'number' },
|
||||
{ id: 'lastActivity', displayName: 'Last Activity', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -2,9 +2,11 @@ import { airtableConnector } from '@/connectors/airtable'
|
||||
import { asanaConnector } from '@/connectors/asana'
|
||||
import { ashbyConnector } from '@/connectors/ashby'
|
||||
import { azureDevopsConnector } from '@/connectors/azure-devops'
|
||||
import { bitbucketConnector } from '@/connectors/bitbucket'
|
||||
import { boxConnector } from '@/connectors/box'
|
||||
import { clickupConnector } from '@/connectors/clickup'
|
||||
import { confluenceConnector } from '@/connectors/confluence'
|
||||
import { databricksConnector } from '@/connectors/databricks'
|
||||
import { discordConnector } from '@/connectors/discord'
|
||||
import { docusignConnector } from '@/connectors/docusign'
|
||||
import { dropboxConnector } from '@/connectors/dropbox'
|
||||
@@ -15,6 +17,7 @@ import { gitlabConnector } from '@/connectors/gitlab'
|
||||
import { gmailConnector } from '@/connectors/gmail'
|
||||
import { gongConnector } from '@/connectors/gong'
|
||||
import { googleCalendarConnector } from '@/connectors/google-calendar'
|
||||
import { googleChatConnector } from '@/connectors/google-chat'
|
||||
import { googleDocsConnector } from '@/connectors/google-docs'
|
||||
import { googleDriveConnector } from '@/connectors/google-drive'
|
||||
import { googleFormsConnector } from '@/connectors/google-forms'
|
||||
@@ -54,6 +57,7 @@ import { typeformConnector } from '@/connectors/typeform'
|
||||
import type { ConnectorRegistry } from '@/connectors/types'
|
||||
import { webflowConnector } from '@/connectors/webflow'
|
||||
import { wordpressConnector } from '@/connectors/wordpress'
|
||||
import { workdayConnector } from '@/connectors/workday'
|
||||
import { xConnector } from '@/connectors/x'
|
||||
import { youtubeConnector } from '@/connectors/youtube'
|
||||
import { zendeskConnector } from '@/connectors/zendesk'
|
||||
@@ -72,9 +76,11 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = {
|
||||
asana: asanaConnector,
|
||||
ashby: ashbyConnector,
|
||||
azure_devops: azureDevopsConnector,
|
||||
bitbucket: bitbucketConnector,
|
||||
box: boxConnector,
|
||||
clickup: clickupConnector,
|
||||
confluence: confluenceConnector,
|
||||
databricks: databricksConnector,
|
||||
discord: discordConnector,
|
||||
docusign: docusignConnector,
|
||||
dropbox: dropboxConnector,
|
||||
@@ -85,6 +91,7 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = {
|
||||
gmail: gmailConnector,
|
||||
gong: gongConnector,
|
||||
google_calendar: googleCalendarConnector,
|
||||
google_chat: googleChatConnector,
|
||||
google_docs: googleDocsConnector,
|
||||
google_drive: googleDriveConnector,
|
||||
google_forms: googleFormsConnector,
|
||||
@@ -123,6 +130,7 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = {
|
||||
typeform: typeformConnector,
|
||||
webflow: webflowConnector,
|
||||
wordpress: wordpressConnector,
|
||||
workday: workdayConnector,
|
||||
x: xConnector,
|
||||
youtube: youtubeConnector,
|
||||
zendesk: zendeskConnector,
|
||||
|
||||
@@ -2,9 +2,11 @@ import { airtableConnectorMeta } from '@/connectors/airtable/meta'
|
||||
import { asanaConnectorMeta } from '@/connectors/asana/meta'
|
||||
import { ashbyConnectorMeta } from '@/connectors/ashby/meta'
|
||||
import { azureDevopsConnectorMeta } from '@/connectors/azure-devops/meta'
|
||||
import { bitbucketConnectorMeta } from '@/connectors/bitbucket/meta'
|
||||
import { boxConnectorMeta } from '@/connectors/box/meta'
|
||||
import { clickupConnectorMeta } from '@/connectors/clickup/meta'
|
||||
import { confluenceConnectorMeta } from '@/connectors/confluence/meta'
|
||||
import { databricksConnectorMeta } from '@/connectors/databricks/meta'
|
||||
import { discordConnectorMeta } from '@/connectors/discord/meta'
|
||||
import { docusignConnectorMeta } from '@/connectors/docusign/meta'
|
||||
import { dropboxConnectorMeta } from '@/connectors/dropbox/meta'
|
||||
@@ -15,6 +17,7 @@ import { gitlabConnectorMeta } from '@/connectors/gitlab/meta'
|
||||
import { gmailConnectorMeta } from '@/connectors/gmail/meta'
|
||||
import { gongConnectorMeta } from '@/connectors/gong/meta'
|
||||
import { googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta'
|
||||
import { googleChatConnectorMeta } from '@/connectors/google-chat/meta'
|
||||
import { googleDocsConnectorMeta } from '@/connectors/google-docs/meta'
|
||||
import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta'
|
||||
import { googleFormsConnectorMeta } from '@/connectors/google-forms/meta'
|
||||
@@ -54,6 +57,7 @@ import { typeformConnectorMeta } from '@/connectors/typeform/meta'
|
||||
import type { ConnectorMeta, ConnectorMetaRegistry } from '@/connectors/types'
|
||||
import { webflowConnectorMeta } from '@/connectors/webflow/meta'
|
||||
import { wordpressConnectorMeta } from '@/connectors/wordpress/meta'
|
||||
import { workdayConnectorMeta } from '@/connectors/workday/meta'
|
||||
import { xConnectorMeta } from '@/connectors/x/meta'
|
||||
import { youtubeConnectorMeta } from '@/connectors/youtube/meta'
|
||||
import { zendeskConnectorMeta } from '@/connectors/zendesk/meta'
|
||||
@@ -72,9 +76,11 @@ export const CONNECTOR_META_REGISTRY: ConnectorMetaRegistry = {
|
||||
asana: asanaConnectorMeta,
|
||||
ashby: ashbyConnectorMeta,
|
||||
azure_devops: azureDevopsConnectorMeta,
|
||||
bitbucket: bitbucketConnectorMeta,
|
||||
box: boxConnectorMeta,
|
||||
clickup: clickupConnectorMeta,
|
||||
confluence: confluenceConnectorMeta,
|
||||
databricks: databricksConnectorMeta,
|
||||
discord: discordConnectorMeta,
|
||||
docusign: docusignConnectorMeta,
|
||||
dropbox: dropboxConnectorMeta,
|
||||
@@ -85,6 +91,7 @@ export const CONNECTOR_META_REGISTRY: ConnectorMetaRegistry = {
|
||||
gmail: gmailConnectorMeta,
|
||||
gong: gongConnectorMeta,
|
||||
google_calendar: googleCalendarConnectorMeta,
|
||||
google_chat: googleChatConnectorMeta,
|
||||
google_docs: googleDocsConnectorMeta,
|
||||
google_drive: googleDriveConnectorMeta,
|
||||
google_forms: googleFormsConnectorMeta,
|
||||
@@ -123,6 +130,7 @@ export const CONNECTOR_META_REGISTRY: ConnectorMetaRegistry = {
|
||||
typeform: typeformConnectorMeta,
|
||||
webflow: webflowConnectorMeta,
|
||||
wordpress: wordpressConnectorMeta,
|
||||
workday: workdayConnectorMeta,
|
||||
x: xConnectorMeta,
|
||||
youtube: youtubeConnectorMeta,
|
||||
zendesk: zendeskConnectorMeta,
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export { workdayConnector } from '@/connectors/workday/workday'
|
||||
@@ -0,0 +1,144 @@
|
||||
import { WorkdayIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
/**
|
||||
* Workday Help knowledge articles, read through the public `helpArticle` v1 REST
|
||||
* service (`GET /ccx/api/helpArticle/v1/{tenant}/articleVersions`).
|
||||
*
|
||||
* Scope is deliberately narrow, and the connector is named for it. `helpArticle`
|
||||
* is the only Workday REST service whose documented response carries a title
|
||||
* alongside the article body as plain text, which is what a knowledge base
|
||||
* indexes. Worker, organization, and compensation data — what Sim's Workday
|
||||
* *tools* read over SOAP — are records rather than documents and are out of scope.
|
||||
*
|
||||
* ## Why the credential is a customer API client rather than `mode: 'oauth'`
|
||||
*
|
||||
* Workday REST is OAuth-2.0-only; it does not accept the WS-Security username
|
||||
* and password the SOAP tools use. It still cannot become a Sim OAuth provider:
|
||||
*
|
||||
* - Better Auth's `genericOAuth` takes **static** `authorizationUrl`/`tokenUrl`
|
||||
* strings registered once at module load — see the comment on
|
||||
* `salesforceConnector` in `lib/auth/connectors/providers.ts`, which enumerates
|
||||
* Salesforce's two fixed login hosts for exactly this reason. Workday's
|
||||
* endpoints are `https://{tenantHost}/ccx/oauth2/{tenant}/token`, where both
|
||||
* the host and the tenant segment are per-customer and unbounded, so there is
|
||||
* no finite set to enumerate.
|
||||
* - Every provider in that file supplies Sim's own `env.*_CLIENT_ID/SECRET`.
|
||||
* Workday API clients are registered by the customer inside their own tenant
|
||||
* (the "Register API Client for Integrations" task), so no Sim-owned Workday
|
||||
* application exists whose credentials could be configured there.
|
||||
*
|
||||
* The connector therefore carries the customer's API client itself. The
|
||||
* non-secret client ID is a config field; the client secret and the refresh
|
||||
* token are both secrets and so share the connector's single encrypted key as
|
||||
* `clientSecret:refreshToken` — the packing the Gong connector already uses for
|
||||
* its `accessKey:accessKeySecret` pair. They buy a short-lived bearer token once
|
||||
* per sync run.
|
||||
*
|
||||
* ## Why the article status filter is required
|
||||
*
|
||||
* `/articleVersions` lists article *versions*, not articles: every row carries a
|
||||
* `version` number and a `parentArticle` reference to the article it belongs to,
|
||||
* and the service publishes no "latest version only" filter or sort order — the
|
||||
* whole document has eleven paths and none of them addresses an article. An
|
||||
* unfiltered sync therefore indexes each historical revision as its own
|
||||
* document, so the status filter is required and has no default: the operator
|
||||
* picks the scope explicitly, and the option that indexes every revision says so
|
||||
* on its label.
|
||||
*
|
||||
* The three options are the ones the service's own prose enumerates — "an
|
||||
* article version can have a status of Published, Draft, or Archived". What the
|
||||
* `status` *query parameter* accepts for them is the one thing the published
|
||||
* spec does not settle: it is an untyped `array` of `string` with no enum and no
|
||||
* `x-workday-populated-by`, unlike the sibling `audience` parameter, whose model
|
||||
* names `/values/common/audiences` as its value source. The connector resolves
|
||||
* the chosen name against `/articleStatuses` and sends the Workday ID;
|
||||
* `validateConfig` reads the filtered response back and refuses the
|
||||
* configuration if the tenant answered with a version in another status.
|
||||
*/
|
||||
export const workdayConnectorMeta: ConnectorMeta = {
|
||||
id: 'workday',
|
||||
name: 'Workday Help',
|
||||
description: 'Sync Workday Help knowledge articles into your knowledge base',
|
||||
version: '1.0.0',
|
||||
icon: WorkdayIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'Client Secret & Refresh Token',
|
||||
placeholder: 'clientSecret:refreshToken',
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'tenantUrl',
|
||||
title: 'Tenant Host',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. https://wd5-impl-services1.workday.com',
|
||||
required: true,
|
||||
description:
|
||||
'Host of your Workday instance, without the tenant or any path. Must be a Workday-hosted domain.',
|
||||
},
|
||||
{
|
||||
id: 'tenant',
|
||||
title: 'Tenant',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. acme_pt1',
|
||||
required: true,
|
||||
description: 'Workday tenant name, as it appears in your Workday URLs.',
|
||||
},
|
||||
{
|
||||
id: 'clientId',
|
||||
title: 'Client ID',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. NDdiMGE0ZmQtZjk1YS00...',
|
||||
required: true,
|
||||
description:
|
||||
'Client ID of the Workday API Client for Integrations whose client secret and refresh token you entered above. Register it with a non-expiring refresh token: Sim stores the token you enter and cannot replace one the tenant rotates.',
|
||||
},
|
||||
{
|
||||
id: 'status',
|
||||
title: 'Article Status',
|
||||
type: 'dropdown',
|
||||
required: true,
|
||||
options: [
|
||||
{ label: 'Published', id: 'Published' },
|
||||
{ label: 'Draft', id: 'Draft' },
|
||||
{ label: 'Archived', id: 'Archived' },
|
||||
{ label: 'Every status — indexes every historical revision', id: 'all' },
|
||||
],
|
||||
description:
|
||||
'Which article versions to sync, resolved against the statuses your tenant returns. Workday exposes no "latest version only" filter, so an article that has held the chosen status more than once still contributes one document per revision.',
|
||||
},
|
||||
{
|
||||
id: 'audience',
|
||||
title: 'Audience',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
multi: true,
|
||||
placeholder: 'e.g. All Employees (optional)',
|
||||
description:
|
||||
'Comma-separated audience names to sync, resolved against the audiences your tenant returns. A Workday ID is accepted in place of a name. Leave blank to sync every audience.',
|
||||
},
|
||||
{
|
||||
id: 'maxVersions',
|
||||
title: 'Max Article Versions',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
description: 'Stop syncing after this many article versions.',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'article', displayName: 'Article', fieldType: 'text' },
|
||||
{ id: 'category', displayName: 'Category', fieldType: 'text' },
|
||||
{ id: 'status', displayName: 'Status', fieldType: 'text' },
|
||||
{ id: 'audience', displayName: 'Audience', fieldType: 'text' },
|
||||
{ id: 'articleTags', displayName: 'Tags', fieldType: 'text' },
|
||||
{ id: 'language', displayName: 'Language', fieldType: 'text' },
|
||||
{ id: 'version', displayName: 'Version', fieldType: 'number' },
|
||||
{ id: 'created', displayName: 'Created', fieldType: 'date' },
|
||||
{ id: 'lastUpdated', displayName: 'Last Updated', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*
|
||||
* `GET /articleVersions` returns one row per article *revision* and offers no
|
||||
* latest-version filter, so what this connector indexes is decided entirely by
|
||||
* the required status choice and the version cap. Both are exercised here, along
|
||||
* with the `listingCapped` flag the sync engine reads before hard-deleting the
|
||||
* documents a partial listing left out.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { workdayConnector } from '@/connectors/workday/workday'
|
||||
|
||||
const ACCESS_TOKEN = 'client-secret:refresh-token'
|
||||
const CONFIG = {
|
||||
tenantUrl: 'https://wd5-impl-services1.workday.com',
|
||||
tenant: 'acme_pt1',
|
||||
clientId: 'client-id',
|
||||
status: 'Published',
|
||||
}
|
||||
|
||||
const PUBLISHED_ID = '0d75a5e37a411000167d21b9239f0001'
|
||||
const AUDIENCE_ID = '8ac3f16c1fff10000c53e90920940001'
|
||||
|
||||
const mockFetch = vi.fn()
|
||||
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
})
|
||||
}
|
||||
|
||||
function versionFixture(id: string, version = 1) {
|
||||
return {
|
||||
id,
|
||||
title: `Article ${id}`,
|
||||
content: 'Body text',
|
||||
version,
|
||||
lastUpdatedDate: '2026-01-01T00:00:00.000Z',
|
||||
parentArticle: { id: `parent-${id}`, descriptor: `Article ${id}` },
|
||||
status: { id: PUBLISHED_ID, descriptor: 'Published' },
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Routes by URL rather than call order, because the number of lookups before the
|
||||
* listing depends on which filters the config asks for.
|
||||
*/
|
||||
function mockApi(listing: unknown, options: { audiences?: unknown } = {}) {
|
||||
mockFetch.mockImplementation(async (url: string) => {
|
||||
if (url.includes('/ccx/oauth2/')) {
|
||||
return jsonResponse({ access_token: 'bearer', expires_in: 3600 })
|
||||
}
|
||||
if (url.includes('/articleStatuses')) {
|
||||
return jsonResponse({
|
||||
total: 3,
|
||||
data: [
|
||||
{ id: PUBLISHED_ID, descriptor: 'Published' },
|
||||
{ id: 'aaaa5a5e37a411000167d21b9239f002', descriptor: 'Draft' },
|
||||
{ id: 'bbbb5a5e37a411000167d21b9239f003', descriptor: 'Archived' },
|
||||
],
|
||||
})
|
||||
}
|
||||
if (url.includes('/values/common/audiences')) {
|
||||
return jsonResponse(
|
||||
options.audiences ?? {
|
||||
total: 1,
|
||||
data: [{ id: AUDIENCE_ID, descriptor: 'All Employees' }],
|
||||
}
|
||||
)
|
||||
}
|
||||
return jsonResponse(listing)
|
||||
})
|
||||
}
|
||||
|
||||
function tokenExchanges(): string[] {
|
||||
return mockFetch.mock.calls
|
||||
.map(([url]) => url as string)
|
||||
.filter((url) => url.includes('/ccx/oauth2/'))
|
||||
}
|
||||
|
||||
function listingUrls(): string[] {
|
||||
return mockFetch.mock.calls
|
||||
.map(([url]) => url as string)
|
||||
.filter((url) => url.includes('/articleVersions'))
|
||||
}
|
||||
|
||||
describe('workday listDocuments', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
it('resolves the status name to the tenant Workday ID and filters the listing by it', async () => {
|
||||
mockApi({ total: 1, data: [versionFixture('a')] })
|
||||
|
||||
const result = await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {})
|
||||
|
||||
expect(listingUrls()[0]).toContain(`status=${PUBLISHED_ID}`)
|
||||
expect(result.documents).toHaveLength(1)
|
||||
expect(result.documents[0].externalId).toBe('a')
|
||||
})
|
||||
|
||||
it('omits the status filter only when the operator explicitly chose every status', async () => {
|
||||
mockApi({ total: 1, data: [versionFixture('a')] })
|
||||
|
||||
await workdayConnector.listDocuments(ACCESS_TOKEN, { ...CONFIG, status: 'all' }, undefined, {})
|
||||
|
||||
expect(listingUrls()[0]).not.toContain('status=')
|
||||
expect(mockFetch.mock.calls.some(([url]) => (url as string).includes('/articleStatuses'))).toBe(
|
||||
false
|
||||
)
|
||||
})
|
||||
|
||||
it('refuses to list when no status was chosen, rather than indexing every revision', async () => {
|
||||
mockApi({ total: 1, data: [versionFixture('a')] })
|
||||
|
||||
await expect(
|
||||
workdayConnector.listDocuments(ACCESS_TOKEN, { ...CONFIG, status: '' }, undefined, {})
|
||||
).rejects.toThrow(/Article Status is required/)
|
||||
})
|
||||
|
||||
it('resolves audience names to Workday IDs', async () => {
|
||||
mockApi({ total: 1, data: [versionFixture('a')] })
|
||||
|
||||
await workdayConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, audience: 'All Employees' },
|
||||
undefined,
|
||||
{}
|
||||
)
|
||||
|
||||
expect(listingUrls()[0]).toContain(`audience=${AUDIENCE_ID}`)
|
||||
})
|
||||
|
||||
it('names the available audiences when one cannot be resolved', async () => {
|
||||
mockApi({ total: 1, data: [versionFixture('a')] })
|
||||
|
||||
await expect(
|
||||
workdayConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, audience: 'Contractors' },
|
||||
undefined,
|
||||
{}
|
||||
)
|
||||
).rejects.toThrow(/no audience named "Contractors". Available: All Employees/)
|
||||
})
|
||||
|
||||
it('leaves listingCapped unset when the cap lands exactly on source exhaustion', async () => {
|
||||
mockApi({ total: 1, data: [versionFixture('a')] })
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
await workdayConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, maxVersions: '1' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(syncContext.listingCapped).toBeUndefined()
|
||||
})
|
||||
|
||||
it('flags listingCapped when the cap stops short of the tenant total', async () => {
|
||||
mockApi({ total: 10, data: [versionFixture('a')] })
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
await workdayConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, maxVersions: '1' },
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(syncContext.listingCapped).toBe(true)
|
||||
})
|
||||
|
||||
it('leaves listingCapped unset when an uncapped page exhausts the source', async () => {
|
||||
mockApi({ total: 2, data: [versionFixture('a'), versionFixture('b')] })
|
||||
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
const result = await workdayConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
CONFIG,
|
||||
undefined,
|
||||
syncContext
|
||||
)
|
||||
|
||||
expect(syncContext.listingCapped).toBeUndefined()
|
||||
expect(result.hasMore).toBe(false)
|
||||
})
|
||||
|
||||
it('requests only the versions the cap still allows on the final page', async () => {
|
||||
mockApi({ total: 500, data: [versionFixture('a')] })
|
||||
|
||||
await workdayConnector.listDocuments(ACCESS_TOKEN, { ...CONFIG, maxVersions: '101' }, '100', {})
|
||||
|
||||
expect(listingUrls()[0]).toContain('limit=1')
|
||||
expect(listingUrls()[0]).toContain('offset=100')
|
||||
})
|
||||
|
||||
it('strips a path and trailing slash from the configured tenant host', async () => {
|
||||
mockApi({ total: 1, data: [versionFixture('a')] })
|
||||
|
||||
await workdayConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, tenantUrl: 'https://wd5-impl-services1.workday.com/acme_pt1/d/home.htmld' },
|
||||
undefined,
|
||||
{}
|
||||
)
|
||||
|
||||
expect(listingUrls()[0]).toContain(
|
||||
'https://wd5-impl-services1.workday.com/ccx/api/helpArticle/v1/acme_pt1/articleVersions?'
|
||||
)
|
||||
})
|
||||
|
||||
it('accepts a tenant host written without a scheme', async () => {
|
||||
mockApi({ total: 1, data: [versionFixture('a')] })
|
||||
|
||||
await workdayConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, tenantUrl: 'wd5-impl-services1.workday.com' },
|
||||
undefined,
|
||||
{}
|
||||
)
|
||||
|
||||
expect(listingUrls()[0]).toContain(
|
||||
'https://wd5-impl-services1.workday.com/ccx/api/helpArticle/v1/acme_pt1/articleVersions?'
|
||||
)
|
||||
})
|
||||
|
||||
it('passes a Workday ID through instead of looking it up as a display name', async () => {
|
||||
mockApi({ total: 1, data: [versionFixture('a')] })
|
||||
|
||||
await workdayConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, audience: AUDIENCE_ID },
|
||||
undefined,
|
||||
{}
|
||||
)
|
||||
|
||||
expect(listingUrls()[0]).toContain(`audience=${AUDIENCE_ID}`)
|
||||
expect(
|
||||
mockFetch.mock.calls.some(([url]) => (url as string).includes('/values/common/audiences'))
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('caps how many available values an unresolved-name error names', async () => {
|
||||
const many = Array.from({ length: 30 }, (_, index) => ({
|
||||
id: `${index}`.padStart(32, '0'),
|
||||
descriptor: `Audience ${index}`,
|
||||
}))
|
||||
mockApi({ total: 1, data: [versionFixture('a')] }, { audiences: { total: 30, data: many } })
|
||||
|
||||
const error = await workdayConnector
|
||||
.listDocuments(ACCESS_TOKEN, { ...CONFIG, audience: 'Nope' }, undefined, {})
|
||||
.catch((thrown: Error) => thrown)
|
||||
|
||||
expect((error as Error).message).toContain('and 10 more')
|
||||
expect((error as Error).message).not.toContain('Audience 25')
|
||||
})
|
||||
|
||||
it('reads a version cap persisted as a number rather than a string', async () => {
|
||||
mockApi({ total: 500, data: [versionFixture('a')] })
|
||||
|
||||
await workdayConnector.listDocuments(ACCESS_TOKEN, { ...CONFIG, maxVersions: 1 }, undefined, {})
|
||||
|
||||
expect(listingUrls()[0]).toContain('limit=1')
|
||||
})
|
||||
|
||||
it('rejects a tenant host that is not a Workday domain', async () => {
|
||||
mockApi({ total: 0, data: [] })
|
||||
|
||||
await expect(
|
||||
workdayConnector.listDocuments(
|
||||
ACCESS_TOKEN,
|
||||
{ ...CONFIG, tenantUrl: 'https://evil.example.com' },
|
||||
undefined,
|
||||
{}
|
||||
)
|
||||
).rejects.toThrow(/Workday-hosted domain/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('workday credential failures', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
it('explains that a rejected refresh token has to be re-entered', async () => {
|
||||
mockFetch.mockResolvedValue(jsonResponse({ error: 'invalid_grant' }, 400))
|
||||
|
||||
const result = await workdayConnector.validateConfig(ACCESS_TOKEN, CONFIG)
|
||||
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.error).toMatch(/Manage Refresh Tokens for Integrations/)
|
||||
})
|
||||
|
||||
it('buys one bearer token for a whole sync run and reuses it across pages', async () => {
|
||||
mockApi({ total: 500, data: [versionFixture('a')] })
|
||||
const syncContext: Record<string, unknown> = {}
|
||||
|
||||
await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, syncContext)
|
||||
await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, '100', syncContext)
|
||||
|
||||
expect(tokenExchanges()).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('re-authenticates once and replays the request when a cached token has expired', async () => {
|
||||
let listingCalls = 0
|
||||
mockFetch.mockImplementation(async (url: string) => {
|
||||
if (url.includes('/ccx/oauth2/')) {
|
||||
return jsonResponse({ access_token: 'bearer', expires_in: 3600 })
|
||||
}
|
||||
if (url.includes('/articleStatuses')) {
|
||||
return jsonResponse({ total: 1, data: [{ id: PUBLISHED_ID, descriptor: 'Published' }] })
|
||||
}
|
||||
listingCalls += 1
|
||||
return listingCalls === 1
|
||||
? jsonResponse({ error: 'expired token' }, 401)
|
||||
: jsonResponse({ total: 1, data: [versionFixture('a')] })
|
||||
})
|
||||
|
||||
const result = await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {})
|
||||
|
||||
expect(result.documents).toHaveLength(1)
|
||||
expect(listingCalls).toBe(2)
|
||||
expect(tokenExchanges()).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('surfaces a persistent 401 instead of retrying the exchange forever', async () => {
|
||||
mockFetch.mockImplementation(async (url: string) => {
|
||||
if (url.includes('/ccx/oauth2/')) {
|
||||
return jsonResponse({ access_token: 'bearer', expires_in: 3600 })
|
||||
}
|
||||
if (url.includes('/articleStatuses')) {
|
||||
return jsonResponse({ total: 1, data: [{ id: PUBLISHED_ID, descriptor: 'Published' }] })
|
||||
}
|
||||
return jsonResponse({ error: 'still unauthorized' }, 401)
|
||||
})
|
||||
|
||||
await expect(
|
||||
workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {})
|
||||
).rejects.toThrow(/still unauthorized/)
|
||||
expect(tokenExchanges()).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('rejects a version cap that is not a positive number', async () => {
|
||||
mockApi({ total: 0, data: [] })
|
||||
|
||||
const result = await workdayConnector.validateConfig(ACCESS_TOKEN, {
|
||||
...CONFIG,
|
||||
maxVersions: '0',
|
||||
})
|
||||
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.error).toMatch(/positive number/)
|
||||
})
|
||||
|
||||
it('refuses a tenant that answered the status filter with another status', async () => {
|
||||
mockApi({
|
||||
total: 1,
|
||||
data: [{ ...versionFixture('a'), status: { id: 'other', descriptor: 'Draft' } }],
|
||||
})
|
||||
|
||||
const result = await workdayConnector.validateConfig(ACCESS_TOKEN, CONFIG)
|
||||
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.error).toMatch(/ignored the article status filter/)
|
||||
})
|
||||
|
||||
it('rejects a credential that is not a clientSecret:refreshToken pair', async () => {
|
||||
mockApi({ total: 0, data: [] })
|
||||
|
||||
const result = await workdayConnector.validateConfig('only-one-secret', CONFIG)
|
||||
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.error).toMatch(/clientSecret:refreshToken/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('workday document mapping', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
it('produces the same contentHash from the listing and from getDocument', async () => {
|
||||
mockApi({ total: 1, data: [versionFixture('a', 4)] })
|
||||
const listed = await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {})
|
||||
|
||||
vi.clearAllMocks()
|
||||
mockFetch.mockImplementation(async (url: string) =>
|
||||
url.includes('/ccx/oauth2/')
|
||||
? jsonResponse({ access_token: 'bearer', expires_in: 3600 })
|
||||
: jsonResponse(versionFixture('a', 4))
|
||||
)
|
||||
const fetched = await workdayConnector.getDocument(ACCESS_TOKEN, CONFIG, 'a', {})
|
||||
|
||||
expect(fetched?.contentHash).toBe(listed.documents[0].contentHash)
|
||||
expect(fetched?.contentHash).toBe('workday:a:4:2026-01-01T00:00:00.000Z')
|
||||
})
|
||||
|
||||
it('returns null for an article version the tenant no longer has', async () => {
|
||||
mockFetch.mockImplementation(async (url: string) =>
|
||||
url.includes('/ccx/oauth2/')
|
||||
? jsonResponse({ access_token: 'bearer', expires_in: 3600 })
|
||||
: jsonResponse({ error: 'not found' }, 404)
|
||||
)
|
||||
|
||||
expect(await workdayConnector.getDocument(ACCESS_TOKEN, CONFIG, 'gone', {})).toBeNull()
|
||||
})
|
||||
|
||||
it('indexes documented plain-text content verbatim', async () => {
|
||||
mockApi({
|
||||
total: 1,
|
||||
data: [{ ...versionFixture('a'), content: 'Email support <help@acme.com> for 5 < 10 cases' }],
|
||||
})
|
||||
|
||||
const result = await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {})
|
||||
|
||||
expect(result.documents[0].content).toBe('Email support <help@acme.com> for 5 < 10 cases')
|
||||
})
|
||||
|
||||
it('reduces a tenant that emits real markup to plain text', async () => {
|
||||
mockApi({
|
||||
total: 1,
|
||||
data: [{ ...versionFixture('a'), content: '<p>Reset your <strong>password</strong>.</p>' }],
|
||||
})
|
||||
|
||||
const result = await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {})
|
||||
|
||||
expect(result.documents[0].content).toBe('Reset your password .')
|
||||
})
|
||||
|
||||
it('maps every declared tag from the metadata the listing stores', async () => {
|
||||
mockApi({
|
||||
total: 1,
|
||||
data: [
|
||||
{
|
||||
...versionFixture('a', 2),
|
||||
createdDate: '2025-06-01T00:00:00.000Z',
|
||||
category: { id: 'c1', descriptor: 'Benefits' },
|
||||
language: { id: 'l1', descriptor: 'English' },
|
||||
audience: [{ id: AUDIENCE_ID, descriptor: 'All Employees' }],
|
||||
tags: [
|
||||
{ id: 't1', descriptor: 'payroll' },
|
||||
{ id: 't2', descriptor: 'benefits' },
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const result = await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {})
|
||||
const tags = workdayConnector.mapTags?.(result.documents[0].metadata ?? {})
|
||||
|
||||
expect(tags).toEqual({
|
||||
article: 'Article a',
|
||||
category: 'Benefits',
|
||||
status: 'Published',
|
||||
language: 'English',
|
||||
audience: 'All Employees',
|
||||
articleTags: 'payroll, benefits',
|
||||
version: 2,
|
||||
created: new Date('2025-06-01T00:00:00.000Z'),
|
||||
lastUpdated: new Date('2026-01-01T00:00:00.000Z'),
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,776 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { validatePathSegment, validateWorkdayTenantUrl } from '@/lib/core/security/input-validation'
|
||||
import type { RetryOptions } from '@/lib/knowledge/documents/utils'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import {
|
||||
htmlToPlainText,
|
||||
joinTagArray,
|
||||
looksLikeHtml,
|
||||
parseMultiValue,
|
||||
parseTagDate,
|
||||
} from '@/connectors/utils'
|
||||
import { workdayConnectorMeta } from '@/connectors/workday/meta'
|
||||
|
||||
const logger = createLogger('WorkdayConnector')
|
||||
|
||||
/**
|
||||
* Maximum `limit` the `helpArticle` v1 service accepts on `GET /articleVersions`
|
||||
* and `GET /articleStatuses`; both document a default of 20 and a maximum of 100.
|
||||
*/
|
||||
const PAGE_SIZE = 100
|
||||
|
||||
/** `GET /values/common/audiences/` documents a default *and* maximum of 1000. */
|
||||
const VALUES_PAGE_SIZE = 1000
|
||||
|
||||
/**
|
||||
* Hard stop on the value-lookup pagination that turns audience and status names
|
||||
* into Workday IDs, so a tenant with an unexpectedly large prompt-value list
|
||||
* cannot pull an unbounded number of rows into memory.
|
||||
*/
|
||||
const MAX_VALUE_LOOKUP_ROWS = 5000
|
||||
|
||||
/** How many available values an unresolved-name error is allowed to name. */
|
||||
const MAX_ERROR_DESCRIPTORS = 20
|
||||
|
||||
/**
|
||||
* Workday resolves REST services as `/ccx/api/{service}/{version}/{tenant}/...`,
|
||||
* where `{service}/{version}` is the `basePath` the service's OpenAPI document
|
||||
* declares — `/helpArticle/v1` here.
|
||||
*/
|
||||
const API_BASE_PATH = '/ccx/api/helpArticle/v1'
|
||||
|
||||
/** Sentinel status choice meaning "do not filter by status at all". */
|
||||
const ALL_STATUSES = 'all'
|
||||
|
||||
/**
|
||||
* A Workday instance identifier as `INSTANCE_MODEL_REFERENCE` declares it: a
|
||||
* 32-character lowercase Workday ID, or a `Type=Value` reference ID. Config
|
||||
* values matching it are passed to the API untouched instead of being looked up
|
||||
* as display names.
|
||||
*/
|
||||
const WORKDAY_INSTANCE_ID = /^(?:[0-9a-f]{32}|\S+=\S+)$/
|
||||
|
||||
/** Refresh a bearer token this far before it actually expires. */
|
||||
const TOKEN_EXPIRY_SKEW_MS = 60_000
|
||||
|
||||
/** Workday documents its access tokens as valid for 3600 seconds. */
|
||||
const DEFAULT_TOKEN_LIFETIME_SECONDS = 3600
|
||||
|
||||
/**
|
||||
* A Workday instance reference (`INSTANCE_MODEL_REFERENCE`): a Workday ID or
|
||||
* reference ID, plus the display name Workday renders for it.
|
||||
*/
|
||||
interface WorkdayInstance {
|
||||
id?: string
|
||||
descriptor?: string
|
||||
href?: string
|
||||
}
|
||||
|
||||
/** The `{ total, data }` envelope every collection in this service returns. */
|
||||
interface WorkdayCollection<T> {
|
||||
total?: number
|
||||
data?: T[]
|
||||
}
|
||||
|
||||
/**
|
||||
* One article version from `GET /articleVersions` and `GET /articleVersions/{ID}`,
|
||||
* which share the `articleVersionDetails` schema. Every field is optional because
|
||||
* the schema marks none of them required and Workday omits what the caller is not
|
||||
* permitted to see.
|
||||
*/
|
||||
interface WorkdayArticleVersion {
|
||||
id?: string
|
||||
title?: string
|
||||
content?: string
|
||||
version?: number
|
||||
createdDate?: string
|
||||
lastUpdatedDate?: string
|
||||
viewLink?: string
|
||||
latestPublishedVersionViewURL?: string
|
||||
status?: WorkdayInstance
|
||||
category?: WorkdayInstance
|
||||
language?: WorkdayInstance
|
||||
location?: WorkdayInstance
|
||||
parentArticle?: WorkdayInstance
|
||||
tags?: WorkdayInstance[]
|
||||
audience?: WorkdayInstance[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Workday's error body: a message in `error`, plus per-field detail in `errors`
|
||||
* for validation failures.
|
||||
*/
|
||||
interface WorkdayErrorBody {
|
||||
error?: string
|
||||
errors?: { error?: string; field?: string }[]
|
||||
}
|
||||
|
||||
/** A bearer token cached for the remainder of a sync run, with its expiry. */
|
||||
interface WorkdayBearerToken {
|
||||
value: string
|
||||
expiresAt: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Tenant coordinates resolved from `sourceConfig`, with the host already checked
|
||||
* against the Workday domain allowlist and reduced to a bare origin.
|
||||
*/
|
||||
interface WorkdayTenant {
|
||||
origin: string
|
||||
tenant: string
|
||||
clientId: string
|
||||
}
|
||||
|
||||
/** The status and audience filters, resolved from display names to Workday IDs. */
|
||||
interface WorkdayFilters {
|
||||
status: string[]
|
||||
audience: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Reduces the configured tenant host to an origin the API paths can be appended
|
||||
* to. `validateWorkdayTenantUrl` only reports whether a URL is well-formed and
|
||||
* on a Workday domain — it returns the string as written — so a value carrying a
|
||||
* path, a query, or a trailing slash would otherwise be spliced into every
|
||||
* request URL. A scheme-less host is accepted and assumed HTTPS, which is the
|
||||
* only scheme the service's OpenAPI document declares.
|
||||
*/
|
||||
function normalizeTenantOrigin(raw: unknown): string {
|
||||
const trimmed = typeof raw === 'string' ? raw.trim() : ''
|
||||
const candidate = trimmed && !/^https?:\/\//i.test(trimmed) ? `https://${trimmed}` : trimmed
|
||||
const validation = validateWorkdayTenantUrl(candidate || undefined)
|
||||
if (!validation.isValid) {
|
||||
throw new Error(validation.error ?? 'Invalid Workday tenant host')
|
||||
}
|
||||
return new URL(validation.sanitized ?? candidate).origin
|
||||
}
|
||||
|
||||
function resolveTenant(sourceConfig: Record<string, unknown>): WorkdayTenant {
|
||||
const origin = normalizeTenantOrigin(sourceConfig.tenantUrl)
|
||||
|
||||
const tenant = (sourceConfig.tenant as string | undefined)?.trim()
|
||||
const tenantValidation = validatePathSegment(tenant, { paramName: 'tenant', maxLength: 128 })
|
||||
if (!tenantValidation.isValid) {
|
||||
throw new Error(tenantValidation.error ?? 'Invalid Workday tenant')
|
||||
}
|
||||
|
||||
const clientId = (sourceConfig.clientId as string | undefined)?.trim()
|
||||
if (!clientId) {
|
||||
throw new Error('Client ID is required')
|
||||
}
|
||||
|
||||
return { origin, tenant: tenant as string, clientId }
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits the stored key into the client secret and refresh token. Both are
|
||||
* secrets, so they share the connector's single encrypted field rather than one
|
||||
* of them sitting in plaintext `sourceConfig`. Only the first colon separates
|
||||
* them — a secret may legitimately contain further colons.
|
||||
*/
|
||||
function splitCredentials(accessToken: string): { clientSecret: string; refreshToken: string } {
|
||||
const separator = accessToken.indexOf(':')
|
||||
if (separator <= 0 || separator === accessToken.length - 1) {
|
||||
throw new Error('Credential must be in the form clientSecret:refreshToken')
|
||||
}
|
||||
return {
|
||||
clientSecret: accessToken.slice(0, separator),
|
||||
refreshToken: accessToken.slice(separator + 1),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Exchanges the refresh token for a bearer access token at the tenant's token
|
||||
* endpoint (`POST /ccx/oauth2/{tenant}/token`), authenticating the API client
|
||||
* with HTTP Basic. Workday access tokens live an hour, so the result is cached
|
||||
* on `syncContext` with its expiry and reused across the pages of a sync run.
|
||||
*/
|
||||
async function getBearerToken(
|
||||
accessToken: string,
|
||||
wd: WorkdayTenant,
|
||||
syncContext?: Record<string, unknown>,
|
||||
retryOptions?: RetryOptions,
|
||||
forceRefresh = false
|
||||
): Promise<string> {
|
||||
if (!forceRefresh) {
|
||||
const cached = syncContext?.workdayBearerToken as WorkdayBearerToken | undefined
|
||||
if (cached?.value && cached.expiresAt > Date.now()) return cached.value
|
||||
}
|
||||
|
||||
const { clientSecret, refreshToken } = splitCredentials(accessToken)
|
||||
const basic = Buffer.from(`${wd.clientId}:${clientSecret}`, 'utf8').toString('base64')
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
`${wd.origin}/ccx/oauth2/${encodeURIComponent(wd.tenant)}/token`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Basic ${basic}`,
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
Accept: 'application/json',
|
||||
},
|
||||
body: new URLSearchParams({
|
||||
grant_type: 'refresh_token',
|
||||
refresh_token: refreshToken,
|
||||
}).toString(),
|
||||
},
|
||||
retryOptions
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const detail = (await response.text().catch(() => '')).slice(0, 300)
|
||||
/**
|
||||
* A refresh token is the one credential this connector cannot repair on its
|
||||
* own: it lives in the encrypted key the user typed, and a sync has nowhere
|
||||
* to write a replacement. Say so, rather than reporting a bare 400 that
|
||||
* looks like a transient outage.
|
||||
*/
|
||||
if (response.status === 400 || response.status === 401) {
|
||||
throw new Error(
|
||||
`Workday rejected the stored refresh token (HTTP ${response.status}). Sim cannot replace a refresh token that the tenant rotated, expired, or revoked — reissue one with the "Manage Refresh Tokens for Integrations" task and re-enter the credential as clientSecret:refreshToken.${detail ? ` Workday said: ${detail}` : ''}`
|
||||
)
|
||||
}
|
||||
throw new Error(
|
||||
`Workday token exchange failed (HTTP ${response.status})${detail ? `: ${detail}` : ''}`
|
||||
)
|
||||
}
|
||||
|
||||
const body = (await response.json()) as {
|
||||
access_token?: string
|
||||
expires_in?: number
|
||||
refresh_token?: string
|
||||
}
|
||||
if (!body.access_token) {
|
||||
throw new Error('Workday token response did not include an access token')
|
||||
}
|
||||
|
||||
/**
|
||||
* A tenant configured to rotate refresh tokens invalidates the stored one the
|
||||
* first time it is redeemed, so the next sync fails no matter what this run
|
||||
* does. Nothing here can persist the replacement, so record the cause up front
|
||||
* — the failure it produces an hour later reads as an unexplained 400.
|
||||
*/
|
||||
if (body.refresh_token && body.refresh_token !== refreshToken) {
|
||||
logger.warn(
|
||||
'Workday returned a rotated refresh token that Sim cannot persist; register the API client with a non-expiring refresh token or this connector will stop syncing',
|
||||
{ tenant: wd.tenant }
|
||||
)
|
||||
}
|
||||
|
||||
const lifetimeSeconds =
|
||||
typeof body.expires_in === 'number' && body.expires_in > 0
|
||||
? body.expires_in
|
||||
: DEFAULT_TOKEN_LIFETIME_SECONDS
|
||||
|
||||
if (syncContext) {
|
||||
syncContext.workdayBearerToken = {
|
||||
value: body.access_token,
|
||||
expiresAt: Date.now() + lifetimeSeconds * 1000 - TOKEN_EXPIRY_SKEW_MS,
|
||||
} satisfies WorkdayBearerToken
|
||||
}
|
||||
return body.access_token
|
||||
}
|
||||
|
||||
/**
|
||||
* Issues an authenticated GET, retrying once with a fresh bearer token on a 401.
|
||||
* A sync run can outlive the hour-long token, and a cached token that expires
|
||||
* between pages would otherwise fail the run rather than the request.
|
||||
*/
|
||||
async function workdayGet(
|
||||
url: string,
|
||||
accessToken: string,
|
||||
wd: WorkdayTenant,
|
||||
syncContext?: Record<string, unknown>,
|
||||
retryOptions?: RetryOptions
|
||||
): Promise<Response> {
|
||||
const send = async (bearer: string) =>
|
||||
fetchWithRetry(
|
||||
url,
|
||||
{ method: 'GET', headers: { Authorization: `Bearer ${bearer}`, Accept: 'application/json' } },
|
||||
retryOptions
|
||||
)
|
||||
|
||||
const response = await send(await getBearerToken(accessToken, wd, syncContext, retryOptions))
|
||||
if (response.status !== 401) return response
|
||||
|
||||
return send(await getBearerToken(accessToken, wd, syncContext, retryOptions, true))
|
||||
}
|
||||
|
||||
async function readErrorMessage(response: Response): Promise<string> {
|
||||
const text = await response.text().catch(() => '')
|
||||
if (!text) return `HTTP ${response.status}`
|
||||
try {
|
||||
const body = JSON.parse(text) as WorkdayErrorBody
|
||||
if (body.error) {
|
||||
const details = (body.errors ?? [])
|
||||
.map((entry) => (entry.field ? `${entry.field}: ${entry.error}` : entry.error))
|
||||
.filter((detail): detail is string => Boolean(detail))
|
||||
return details.length > 0 ? `${body.error} (${details.join('; ')})` : body.error
|
||||
}
|
||||
} catch {
|
||||
/* Workday returns HTML for some gateway failures; fall through to the raw body. */
|
||||
}
|
||||
return text.slice(0, 300)
|
||||
}
|
||||
|
||||
function tenantResourceUrl(wd: WorkdayTenant, resource: string, query?: URLSearchParams): string {
|
||||
const suffix = query ? `?${query.toString()}` : ''
|
||||
return `${wd.origin}${API_BASE_PATH}/${encodeURIComponent(wd.tenant)}${resource}${suffix}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a whole prompt-value collection so display names can be resolved to
|
||||
* Workday IDs. Both `/articleStatuses` and `/values/common/audiences/` use the
|
||||
* service's `limit`/`offset` paging and `{ total, data }` envelope.
|
||||
*/
|
||||
async function fetchInstances(
|
||||
resource: string,
|
||||
pageSize: number,
|
||||
accessToken: string,
|
||||
wd: WorkdayTenant,
|
||||
syncContext?: Record<string, unknown>,
|
||||
retryOptions?: RetryOptions
|
||||
): Promise<WorkdayInstance[]> {
|
||||
const instances: WorkdayInstance[] = []
|
||||
|
||||
for (let offset = 0; offset < MAX_VALUE_LOOKUP_ROWS; offset += pageSize) {
|
||||
const query = new URLSearchParams({ limit: String(pageSize), offset: String(offset) })
|
||||
const response = await workdayGet(
|
||||
tenantResourceUrl(wd, resource, query),
|
||||
accessToken,
|
||||
wd,
|
||||
syncContext,
|
||||
retryOptions
|
||||
)
|
||||
if (!response.ok) {
|
||||
throw new Error(`Workday ${resource} lookup failed: ${await readErrorMessage(response)}`)
|
||||
}
|
||||
|
||||
const body = (await response.json()) as WorkdayCollection<WorkdayInstance>
|
||||
const page = body.data ?? []
|
||||
instances.push(...page)
|
||||
|
||||
if (page.length < pageSize) break
|
||||
if (typeof body.total === 'number' && instances.length >= body.total) break
|
||||
}
|
||||
|
||||
if (instances.length >= MAX_VALUE_LOOKUP_ROWS) {
|
||||
logger.warn('Workday value lookup hit its row cap; names beyond it cannot be resolved', {
|
||||
resource,
|
||||
rows: instances.length,
|
||||
})
|
||||
}
|
||||
|
||||
return instances
|
||||
}
|
||||
|
||||
/**
|
||||
* Names the values the lookup did return, capped. An audience list runs to as
|
||||
* many rows as {@link MAX_VALUE_LOOKUP_ROWS} allows, and the whole list would
|
||||
* otherwise be interpolated into an error string the knowledge base UI renders.
|
||||
*/
|
||||
function availableSuffix(descriptors: string[]): string {
|
||||
if (descriptors.length === 0) return ''
|
||||
const shown = descriptors.slice(0, MAX_ERROR_DESCRIPTORS)
|
||||
const omitted = descriptors.length - shown.length
|
||||
return `. Available: ${shown.join(', ')}${omitted > 0 ? `, and ${omitted} more` : ''}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns configured display names into the Workday IDs the `status` and
|
||||
* `audience` query parameters take. A value already shaped like a Workday ID or
|
||||
* a `Type=Value` reference ID is passed through, so an operator who has the ID
|
||||
* to hand never has to match a descriptor exactly.
|
||||
*/
|
||||
async function resolveInstanceIds(
|
||||
values: string[],
|
||||
loadInstances: () => Promise<WorkdayInstance[]>,
|
||||
label: string
|
||||
): Promise<string[]> {
|
||||
/**
|
||||
* Every value already being an ID is the common case for an operator who
|
||||
* pasted Workday IDs, and the lookup it would otherwise run reads the tenant's
|
||||
* whole prompt-value collection.
|
||||
*/
|
||||
if (values.every((value) => WORKDAY_INSTANCE_ID.test(value))) return values
|
||||
|
||||
const instances = await loadInstances()
|
||||
const idByDescriptor = new Map<string, string>()
|
||||
const descriptors: string[] = []
|
||||
for (const instance of instances) {
|
||||
const descriptor = instance.descriptor?.trim()
|
||||
if (!descriptor || !instance.id) continue
|
||||
const key = descriptor.toLowerCase()
|
||||
if (idByDescriptor.has(key)) continue
|
||||
idByDescriptor.set(key, instance.id)
|
||||
descriptors.push(descriptor)
|
||||
}
|
||||
|
||||
return values.map((value) => {
|
||||
if (WORKDAY_INSTANCE_ID.test(value)) return value
|
||||
const resolved = idByDescriptor.get(value.toLowerCase())
|
||||
if (resolved) return resolved
|
||||
throw new Error(
|
||||
`Your Workday tenant has no ${label} named "${value}"${availableSuffix(descriptors)}`
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the configured status and audience filters once per sync run and
|
||||
* caches them on `syncContext`, so later pages reuse the lookup.
|
||||
*
|
||||
* Status is required and has no default: `/articleVersions` returns one row per
|
||||
* article *revision*, and the service offers no latest-version filter, so the
|
||||
* scope of a sync has to be a decision the operator made rather than one this
|
||||
* code made silently.
|
||||
*/
|
||||
async function resolveFilters(
|
||||
accessToken: string,
|
||||
wd: WorkdayTenant,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
syncContext?: Record<string, unknown>,
|
||||
retryOptions?: RetryOptions
|
||||
): Promise<WorkdayFilters> {
|
||||
const cached = syncContext?.workdayFilters as WorkdayFilters | undefined
|
||||
if (cached) return cached
|
||||
|
||||
const statusChoice = typeof sourceConfig.status === 'string' ? sourceConfig.status.trim() : ''
|
||||
if (!statusChoice) {
|
||||
throw new Error(
|
||||
'Article Status is required: choose which article versions to sync, or "Every status" to index every historical revision.'
|
||||
)
|
||||
}
|
||||
|
||||
const status =
|
||||
statusChoice === ALL_STATUSES
|
||||
? []
|
||||
: await resolveInstanceIds(
|
||||
[statusChoice],
|
||||
() =>
|
||||
fetchInstances(
|
||||
'/articleStatuses',
|
||||
PAGE_SIZE,
|
||||
accessToken,
|
||||
wd,
|
||||
syncContext,
|
||||
retryOptions
|
||||
),
|
||||
'article status'
|
||||
)
|
||||
|
||||
const audienceNames = parseMultiValue(sourceConfig.audience)
|
||||
const audience =
|
||||
audienceNames.length === 0
|
||||
? []
|
||||
: await resolveInstanceIds(
|
||||
audienceNames,
|
||||
() =>
|
||||
fetchInstances(
|
||||
'/values/common/audiences/',
|
||||
VALUES_PAGE_SIZE,
|
||||
accessToken,
|
||||
wd,
|
||||
syncContext,
|
||||
retryOptions
|
||||
),
|
||||
'audience'
|
||||
)
|
||||
|
||||
const filters: WorkdayFilters = { status, audience }
|
||||
if (syncContext) syncContext.workdayFilters = filters
|
||||
return filters
|
||||
}
|
||||
|
||||
function descriptorsOf(instances: WorkdayInstance[] | undefined): string[] {
|
||||
return (instances ?? [])
|
||||
.map((instance) => instance.descriptor)
|
||||
.filter((descriptor): descriptor is string => Boolean(descriptor))
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the indexable document for one article version.
|
||||
*
|
||||
* `contentHash` is derived only from fields the list response already carries —
|
||||
* the version number Workday increments on edit, and `lastUpdatedDate` — and
|
||||
* `GET /articleVersions/{ID}` returns the same `articleVersionDetails` schema, so
|
||||
* the hash is identical by construction whichever call produced the row.
|
||||
*/
|
||||
function articleToDocument(article: WorkdayArticleVersion): ExternalDocument | null {
|
||||
const externalId = article.id
|
||||
if (!externalId) return null
|
||||
|
||||
/**
|
||||
* The service documents `content` as the article body "displayed in plain
|
||||
* text", so it is indexed as written. `looksLikeHtml` is the shared markup
|
||||
* test rather than an unconditional strip: a tenant that emits real markup has
|
||||
* it reduced, while plain text that merely contains angle brackets — an email
|
||||
* address, a bare autolink — survives untouched.
|
||||
*/
|
||||
const rawContent = article.content ?? ''
|
||||
const content = looksLikeHtml(rawContent) ? htmlToPlainText(rawContent) : rawContent
|
||||
|
||||
/**
|
||||
* `viewLink` addresses this version and is populated only once the article is
|
||||
* published; `latestPublishedVersionViewURL` points at the article's newest
|
||||
* published version, which is the closest link a draft or archived revision has.
|
||||
*/
|
||||
const sourceUrl = article.viewLink || article.latestPublishedVersionViewURL || undefined
|
||||
|
||||
return {
|
||||
externalId,
|
||||
title: article.title || 'Untitled article',
|
||||
content,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl,
|
||||
contentHash: `workday:${externalId}:${article.version ?? ''}:${article.lastUpdatedDate ?? ''}`,
|
||||
metadata: {
|
||||
article: article.parentArticle?.descriptor,
|
||||
category: article.category?.descriptor,
|
||||
status: article.status?.descriptor,
|
||||
language: article.language?.descriptor,
|
||||
audience: descriptorsOf(article.audience),
|
||||
articleTags: descriptorsOf(article.tags),
|
||||
version: article.version,
|
||||
created: article.createdDate,
|
||||
lastUpdated: article.lastUpdatedDate,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function buildListUrl(
|
||||
wd: WorkdayTenant,
|
||||
filters: WorkdayFilters,
|
||||
offset: number,
|
||||
limit: number
|
||||
): string {
|
||||
const query = new URLSearchParams({ limit: String(limit), offset: String(offset) })
|
||||
for (const status of filters.status) query.append('status', status)
|
||||
for (const audience of filters.audience) query.append('audience', audience)
|
||||
return tenantResourceUrl(wd, '/articleVersions', query)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the optional version cap: `0` when none was configured, the floored
|
||||
* count when one was, and `-1` when a value was supplied that is not a positive
|
||||
* number, which {@link workdayConnector.validateConfig} rejects.
|
||||
*
|
||||
* The field is a short-input so the value normally arrives as a string, but a
|
||||
* `sourceConfig` persisted with a JSON number must not crash the sync — reading
|
||||
* it as a string and calling `.trim()` on it would.
|
||||
*/
|
||||
function parseMaxVersions(value: unknown): number {
|
||||
const raw = typeof value === 'string' ? value.trim() : value
|
||||
if (raw === '' || raw === null || raw === undefined) return 0
|
||||
const parsed = Number(raw)
|
||||
return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : -1
|
||||
}
|
||||
|
||||
export const workdayConnector: ConnectorConfig = {
|
||||
...workdayConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const wd = resolveTenant(sourceConfig)
|
||||
const filters = await resolveFilters(accessToken, wd, sourceConfig, syncContext)
|
||||
|
||||
const offset = cursor ? Number(cursor) : 0
|
||||
if (!Number.isFinite(offset) || offset < 0) {
|
||||
throw new Error(`Invalid pagination cursor: ${cursor}`)
|
||||
}
|
||||
|
||||
const maxVersions = Math.max(parseMaxVersions(sourceConfig.maxVersions), 0)
|
||||
const alreadyFetched = offset
|
||||
const remaining = maxVersions > 0 ? maxVersions - alreadyFetched : PAGE_SIZE
|
||||
const limit = Math.min(PAGE_SIZE, Math.max(remaining, 0))
|
||||
|
||||
if (limit === 0) {
|
||||
if (syncContext) syncContext.listingCapped = true
|
||||
return { documents: [], hasMore: false }
|
||||
}
|
||||
|
||||
const response = await workdayGet(
|
||||
buildListUrl(wd, filters, offset, limit),
|
||||
accessToken,
|
||||
wd,
|
||||
syncContext
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Workday article listing failed: ${await readErrorMessage(response)}`)
|
||||
}
|
||||
|
||||
const body = (await response.json()) as WorkdayCollection<WorkdayArticleVersion>
|
||||
const page = body.data ?? []
|
||||
|
||||
const documents: ExternalDocument[] = []
|
||||
for (const article of page) {
|
||||
const document = articleToDocument(article)
|
||||
if (document) documents.push(document)
|
||||
}
|
||||
|
||||
const fetched = alreadyFetched + page.length
|
||||
const total = typeof body.total === 'number' ? body.total : undefined
|
||||
const sourceExhausted = page.length < limit || (total !== undefined && fetched >= total)
|
||||
const hitCap = maxVersions > 0 && fetched >= maxVersions
|
||||
|
||||
/**
|
||||
* The sync engine hard-deletes any stored document the listing did not return,
|
||||
* so a listing the cap truncated while Workday still has more article versions
|
||||
* must say so. Reaching the cap exactly at source exhaustion is a complete
|
||||
* listing and must still reconcile deletions.
|
||||
*/
|
||||
if (syncContext && hitCap && !sourceExhausted) {
|
||||
syncContext.listingCapped = true
|
||||
}
|
||||
|
||||
const hasMore = !sourceExhausted && !hitCap
|
||||
|
||||
logger.info('Listed Workday article versions', {
|
||||
tenant: wd.tenant,
|
||||
offset,
|
||||
returned: page.length,
|
||||
/** Rows Workday returned without an `id`, which cannot be addressed or indexed. */
|
||||
unidentified: page.length - documents.length,
|
||||
total,
|
||||
hasMore,
|
||||
})
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? String(fetched) : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocument | null> => {
|
||||
const wd = resolveTenant(sourceConfig)
|
||||
|
||||
const response = await workdayGet(
|
||||
tenantResourceUrl(wd, `/articleVersions/${encodeURIComponent(externalId)}`),
|
||||
accessToken,
|
||||
wd,
|
||||
syncContext
|
||||
)
|
||||
|
||||
if (response.status === 404) return null
|
||||
if (!response.ok) {
|
||||
throw new Error(`Workday article fetch failed: ${await readErrorMessage(response)}`)
|
||||
}
|
||||
|
||||
const article = (await response.json()) as WorkdayArticleVersion
|
||||
return articleToDocument(article)
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
if (parseMaxVersions(sourceConfig.maxVersions) < 0) {
|
||||
return { valid: false, error: 'Max article versions must be a positive number' }
|
||||
}
|
||||
|
||||
const statusChoice = typeof sourceConfig.status === 'string' ? sourceConfig.status.trim() : ''
|
||||
|
||||
try {
|
||||
const wd = resolveTenant(sourceConfig)
|
||||
|
||||
/** Scratch context so validation buys one bearer token, not one per call. */
|
||||
const validationContext: Record<string, unknown> = {}
|
||||
|
||||
/**
|
||||
* Resolving the filters exercises the token exchange and the status and
|
||||
* audience lookups, so a mistyped audience name or a credential without the
|
||||
* Help Article REST API domain fails here rather than mid-sync.
|
||||
*/
|
||||
const filters = await resolveFilters(
|
||||
accessToken,
|
||||
wd,
|
||||
sourceConfig,
|
||||
validationContext,
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
const response = await workdayGet(
|
||||
buildListUrl(wd, filters, 0, 1),
|
||||
accessToken,
|
||||
wd,
|
||||
validationContext,
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Workday returned ${response.status}: ${await readErrorMessage(response)}`,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The `helpArticle` OpenAPI document declares `status` as an untyped
|
||||
* `array` of `string` with no enum, no `$ref` and — unlike the sibling
|
||||
* `audience` parameter, whose model carries
|
||||
* `x-workday-populated-by: /values/common/audiences` — no declared value
|
||||
* source. Whether it binds to an `/articleStatuses` Workday ID cannot be
|
||||
* settled from the published spec, so the one request validation already
|
||||
* makes is read back: a tenant that ignored the filter answers with a
|
||||
* version in some other status, and the connector says so at configuration
|
||||
* time instead of silently indexing the wrong scope on every sync.
|
||||
*/
|
||||
const sample = ((await response.json()) as WorkdayCollection<WorkdayArticleVersion>).data?.[0]
|
||||
const returnedStatus = sample?.status?.descriptor
|
||||
if (
|
||||
statusChoice !== ALL_STATUSES &&
|
||||
returnedStatus &&
|
||||
returnedStatus.toLowerCase() !== statusChoice.toLowerCase()
|
||||
) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Workday ignored the article status filter: the first article version it returned is "${returnedStatus}", not "${statusChoice}". Choose "Every status" to sync every revision, or ask your Workday administrator whether the Help Article REST API accepts a status filter in this tenant.`,
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
logger.warn('Workday connector validation failed', { error: getErrorMessage(error) })
|
||||
return { valid: false, error: getErrorMessage(error, 'Failed to reach Workday') }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
for (const key of ['article', 'category', 'status', 'language'] as const) {
|
||||
const value = metadata[key]
|
||||
if (typeof value === 'string' && value) result[key] = value
|
||||
}
|
||||
|
||||
for (const key of ['audience', 'articleTags'] as const) {
|
||||
const value = joinTagArray(metadata[key])
|
||||
if (value) result[key] = value
|
||||
}
|
||||
|
||||
if (metadata.version != null) {
|
||||
const version = Number(metadata.version)
|
||||
if (!Number.isNaN(version)) result.version = version
|
||||
}
|
||||
|
||||
for (const key of ['created', 'lastUpdated'] as const) {
|
||||
const value = parseTagDate(metadata[key])
|
||||
if (value) result[key] = value
|
||||
}
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -567,6 +567,43 @@ export function buildConnectorProviders(): GenericOAuthConfig[] {
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
providerId: 'google-chat',
|
||||
clientId: env.GOOGLE_CLIENT_ID as string,
|
||||
clientSecret: env.GOOGLE_CLIENT_SECRET as string,
|
||||
discoveryUrl: 'https://accounts.google.com/.well-known/openid-configuration',
|
||||
accessType: 'offline',
|
||||
scopes: getCanonicalScopesForProvider('google-chat'),
|
||||
prompt: 'consent',
|
||||
redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-chat`,
|
||||
getUserInfo: async (tokens) => {
|
||||
try {
|
||||
const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', {
|
||||
headers: { Authorization: `Bearer ${tokens.accessToken}` },
|
||||
})
|
||||
if (!response.ok) {
|
||||
await response.text().catch(() => {})
|
||||
logger.error('Failed to fetch Google user info', { status: response.status })
|
||||
throw new Error(`Failed to fetch Google user info: ${response.statusText}`)
|
||||
}
|
||||
const profile = await response.json()
|
||||
const now = new Date()
|
||||
return {
|
||||
id: `${profile.sub}-${generateId()}`,
|
||||
name: profile.name || 'Google User',
|
||||
email: profile.email,
|
||||
image: profile.picture || undefined,
|
||||
emailVerified: profile.email_verified || false,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Error in Google getUserInfo', { error })
|
||||
throw error
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
providerId: 'google-meet',
|
||||
clientId: env.GOOGLE_CLIENT_ID as string,
|
||||
|
||||
@@ -1695,6 +1695,89 @@ export function validateWorkdayTenantUrl(
|
||||
return { isValid: true, sanitized: url as string }
|
||||
}
|
||||
|
||||
/**
|
||||
* Every production Databricks control-plane DNS zone, mirroring `ALL_ENVS` in the
|
||||
* Databricks SDK (`databricks/sdk/environments.py`). The SDK's `.dev.*`/`.staging.*`
|
||||
* zones are internal and deliberately omitted; the ones that are subdomains of a
|
||||
* zone listed here (e.g. `.staging.cloud.databricks.com`) match by suffix anyway.
|
||||
*/
|
||||
const DATABRICKS_ALLOWED_HOST_SUFFIXES = [
|
||||
'.cloud.databricks.com',
|
||||
'.cloud.databricks.us',
|
||||
'.gcp.databricks.com',
|
||||
'.azuredatabricks.net',
|
||||
'.databricks.azure.us',
|
||||
'.databricks.azure.cn',
|
||||
] as const
|
||||
|
||||
/**
|
||||
* Validates a Databricks workspace host to prevent SSRF attacks.
|
||||
*
|
||||
* Databricks is host-scoped: every workspace has its own per-workspace URL, and
|
||||
* every REST call is made against it. Example valid hosts:
|
||||
* - dbc-1234abcd-5678.cloud.databricks.com (AWS)
|
||||
* - dbc-1234abcd-5678.cloud.databricks.us (AWS GovCloud)
|
||||
* - adb-1234567890123456.7.azuredatabricks.net (Azure)
|
||||
* - adb-1234567890123456.7.databricks.azure.us (Azure US Government)
|
||||
* - adb-1234567890123456.7.databricks.azure.cn (Azure China)
|
||||
* - 1234567890123456.7.gcp.databricks.com (GCP)
|
||||
*
|
||||
* The value is user-supplied and fetched server-side, so it is normalized to an
|
||||
* https origin and then checked against a Databricks-owned domain allowlist.
|
||||
* Users routinely paste a full console URL (a `#notebook/123` deep link, a
|
||||
* trailing slash, a `?o=<workspace-id>` query); every API path is built by
|
||||
* appending `/api/2.0/...`, so any surviving path, query or fragment would
|
||||
* produce a 404. `URL.origin` also lower-cases the host and drops the default
|
||||
* port.
|
||||
*
|
||||
* Note: legacy regional URLs (`https://oregon.cloud.databricks.com`) match the
|
||||
* allowlist but are not recommended by Databricks — point the connector at the
|
||||
* per-workspace URL instead.
|
||||
*
|
||||
* @param host - The workspace host or URL to validate, with or without a scheme
|
||||
* @param paramName - Name of the parameter for error messages
|
||||
* @returns ValidationResult whose `sanitized` value is the https origin
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const result = validateDatabricksWorkspaceHost(workspaceHost)
|
||||
* if (!result.isValid) {
|
||||
* throw new Error(result.error)
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
export function validateDatabricksWorkspaceHost(
|
||||
host: string | null | undefined,
|
||||
paramName = 'workspaceHost'
|
||||
): ValidationResult {
|
||||
const raw = typeof host === 'string' ? host.trim() : ''
|
||||
if (!raw) {
|
||||
return { isValid: false, error: `${paramName} is required` }
|
||||
}
|
||||
|
||||
const withScheme = /^https?:\/\//i.test(raw) ? raw : `https://${raw}`
|
||||
|
||||
const urlResult = validateExternalUrl(withScheme, paramName)
|
||||
if (!urlResult.isValid) return urlResult
|
||||
|
||||
const parsed = new URL(withScheme)
|
||||
const hostname = parsed.hostname.toLowerCase()
|
||||
const isAllowedHost = DATABRICKS_ALLOWED_HOST_SUFFIXES.some((suffix) => hostname.endsWith(suffix))
|
||||
|
||||
if (!isAllowedHost) {
|
||||
logger.warn('Databricks workspace host not on allowlist', {
|
||||
paramName,
|
||||
hostname: hostname.substring(0, 100),
|
||||
})
|
||||
return {
|
||||
isValid: false,
|
||||
error: `${paramName} must be a Databricks-hosted domain (e.g., *.cloud.databricks.com, *.azuredatabricks.net, or *.gcp.databricks.com)`,
|
||||
}
|
||||
}
|
||||
|
||||
return { isValid: true, sanitized: parsed.origin }
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates a database identifier (table or column name) to prevent SQL injection.
|
||||
*
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
GoogleAdsIcon,
|
||||
GoogleBigQueryIcon,
|
||||
GoogleCalendarIcon,
|
||||
GoogleChatIcon,
|
||||
GoogleContactsIcon,
|
||||
GoogleDocsIcon,
|
||||
GoogleDriveIcon,
|
||||
@@ -286,6 +287,27 @@ export const OAUTH_PROVIDERS: Record<string, OAuthProviderConfig> = {
|
||||
],
|
||||
serviceAccountProviderId: 'google-service-account',
|
||||
},
|
||||
/**
|
||||
* Deliberately declares no `serviceAccountProviderId`, unlike every sibling
|
||||
* Google service. A Google service-account JWT cannot reach user-scoped Chat
|
||||
* data without domain-wide delegation, so offering service-account auth here
|
||||
* would surface a credential path that always fails. Enterprises that
|
||||
* authenticate other Google connectors through a delegated service account must
|
||||
* attach a per-user OAuth credential for Chat.
|
||||
*/
|
||||
'google-chat': {
|
||||
name: 'Google Chat',
|
||||
description: 'Read Google Chat spaces and messages the signed-in user can access.',
|
||||
providerId: 'google-chat',
|
||||
icon: GoogleChatIcon,
|
||||
baseProviderIcon: GoogleIcon,
|
||||
scopes: [
|
||||
'https://www.googleapis.com/auth/userinfo.email',
|
||||
'https://www.googleapis.com/auth/userinfo.profile',
|
||||
'https://www.googleapis.com/auth/chat.spaces.readonly',
|
||||
'https://www.googleapis.com/auth/chat.messages.readonly',
|
||||
],
|
||||
},
|
||||
'google-meet': {
|
||||
name: 'Google Meet',
|
||||
description: 'Create and manage Google Meet meeting spaces and conferences.',
|
||||
|
||||
@@ -46,6 +46,7 @@ export type OAuthProvider =
|
||||
| 'google-forms'
|
||||
| 'google-groups'
|
||||
| 'google-meet'
|
||||
| 'google-chat'
|
||||
| 'vertex-ai'
|
||||
| 'x'
|
||||
| 'tiktok'
|
||||
@@ -103,6 +104,7 @@ export type OAuthService =
|
||||
| 'google-forms'
|
||||
| 'google-groups'
|
||||
| 'google-meet'
|
||||
| 'google-chat'
|
||||
| 'vertex-ai'
|
||||
| 'x'
|
||||
| 'tiktok'
|
||||
|
||||
@@ -47,6 +47,10 @@ export const SCOPE_DESCRIPTIONS: Record<string, string> = {
|
||||
'https://www.googleapis.com/auth/admin.directory.group.readonly': 'View Google Workspace groups',
|
||||
'https://www.googleapis.com/auth/admin.directory.group.member.readonly':
|
||||
'View Google Workspace group memberships',
|
||||
'https://www.googleapis.com/auth/chat.spaces.readonly':
|
||||
'View Google Chat spaces you are a member of',
|
||||
'https://www.googleapis.com/auth/chat.messages.readonly':
|
||||
'View messages in Google Chat spaces you are a member of',
|
||||
'https://www.googleapis.com/auth/meetings.space.created':
|
||||
'Create and manage Google Meet meeting spaces',
|
||||
'https://www.googleapis.com/auth/meetings.space.readonly':
|
||||
|
||||
@@ -1376,6 +1376,7 @@ const GOOGLE_OAUTH_SERVICES = new Set([
|
||||
'google-forms',
|
||||
'google-groups',
|
||||
'google-meet',
|
||||
'google-chat',
|
||||
'vertex-ai',
|
||||
])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user