mirror of
https://github.com/langgenius/dify.git
synced 2026-09-24 23:22:26 +08:00
feat(dataset): add website crawl preview (#39324)
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
Stephen Zhou
co-authored by
autofix-ci[bot]
autofix-ci[bot]
parent
922ed4d67d
commit
ba5b26be03
@@ -2,7 +2,7 @@ import type {
|
||||
GetKnowledgeSpacesByIdSourceConnectionsResponse,
|
||||
GetSourceProvidersResponse,
|
||||
} from '@dify/contracts/knowledge-fs/types.gen'
|
||||
import { screen, waitFor } from '@testing-library/react'
|
||||
import { act, screen, waitFor, within } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { render } from '@/test/console/render'
|
||||
import { AddSourcePage } from '../add-source-page'
|
||||
@@ -45,11 +45,22 @@ const queryState = vi.hoisted(() => ({
|
||||
}))
|
||||
|
||||
const clientMock = vi.hoisted(() => ({
|
||||
cancelRun: vi.fn(),
|
||||
createConnection: vi.fn(),
|
||||
createSource: vi.fn(),
|
||||
deleteConnection: vi.fn(),
|
||||
deleteSource: vi.fn(),
|
||||
getPages: vi.fn(),
|
||||
getRun: vi.fn(),
|
||||
getSource: vi.fn(),
|
||||
listSources: vi.fn(),
|
||||
refreshConnection: vi.fn(),
|
||||
retryRun: vi.fn(),
|
||||
startPreview: vi.fn(),
|
||||
}))
|
||||
|
||||
const routerMock = vi.hoisted(() => ({ replace: vi.fn() }))
|
||||
|
||||
const queryClientMock = vi.hoisted(() => ({
|
||||
invalidateQueries: vi.fn(),
|
||||
}))
|
||||
@@ -73,8 +84,17 @@ vi.mock('@/service/client', () => ({
|
||||
consoleClient: {
|
||||
knowledgeFs: {
|
||||
deleteKnowledgeSpacesByIdSourceConnectionsByConnectionId: clientMock.deleteConnection,
|
||||
deleteKnowledgeSpacesByIdSourcesBySourceId: clientMock.deleteSource,
|
||||
getKnowledgeSpacesByIdSources: clientMock.listSources,
|
||||
getKnowledgeSpacesByIdSourcesBySourceId: clientMock.getSource,
|
||||
getKnowledgeSpacesByIdSourceWorkflowsByRunId: clientMock.getRun,
|
||||
getKnowledgeSpacesByIdSourceWorkflowsByRunIdPages: clientMock.getPages,
|
||||
postKnowledgeSpacesByIdSourceConnections: clientMock.createConnection,
|
||||
postKnowledgeSpacesByIdSourceConnectionsByConnectionIdRefresh: clientMock.refreshConnection,
|
||||
postKnowledgeSpacesByIdSources: clientMock.createSource,
|
||||
postKnowledgeSpacesByIdSourcesBySourceIdCrawlPreview: clientMock.startPreview,
|
||||
postKnowledgeSpacesByIdSourceWorkflowsByRunIdCancel: clientMock.cancelRun,
|
||||
postKnowledgeSpacesByIdSourceWorkflowsByRunIdRetry: clientMock.retryRun,
|
||||
},
|
||||
},
|
||||
consoleQuery: {
|
||||
@@ -90,6 +110,10 @@ vi.mock('@/service/client', () => ({
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@/next/navigation', () => ({
|
||||
useRouter: () => routerMock,
|
||||
}))
|
||||
|
||||
const firecrawlProvider: GetSourceProvidersResponse['items'][number] = {
|
||||
authKinds: ['api-key', 'endpoint'],
|
||||
available: true,
|
||||
@@ -186,12 +210,48 @@ const connection = (
|
||||
...overrides,
|
||||
})
|
||||
|
||||
const provisionalSource = {
|
||||
connectionId: 'connection-1',
|
||||
createdAt: '2026-07-20T10:00:00Z',
|
||||
id: 'source-1',
|
||||
knowledgeSpaceId: 'space-1',
|
||||
metadata: { preview: true },
|
||||
name: 'Dify docs',
|
||||
status: 'disabled' as const,
|
||||
type: 'web' as const,
|
||||
updatedAt: '2026-07-20T10:00:00Z',
|
||||
uri: 'https://docs.dify.ai/',
|
||||
version: 1,
|
||||
}
|
||||
|
||||
const previewRun = {
|
||||
checkpoint: 'queued',
|
||||
createdAt: '2026-07-20T10:00:00Z',
|
||||
executionAttempts: 1,
|
||||
id: 'run-1',
|
||||
knowledgeSpaceId: 'space-1',
|
||||
kind: 'crawl-preview',
|
||||
maxExecutionAttempts: 3,
|
||||
progressCompleted: 0,
|
||||
progressFailed: 0,
|
||||
progressSkipped: 0,
|
||||
sourceId: 'source-1',
|
||||
state: 'running',
|
||||
updatedAt: '2026-07-20T10:00:00Z',
|
||||
}
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void
|
||||
const promise = new Promise<T>((nextResolve) => {
|
||||
resolve = nextResolve
|
||||
})
|
||||
return { promise, resolve }
|
||||
}
|
||||
|
||||
describe('AddSourcePage', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
clientMock.createConnection.mockReset()
|
||||
clientMock.deleteConnection.mockReset()
|
||||
clientMock.refreshConnection.mockReset()
|
||||
for (const mock of Object.values(clientMock)) mock.mockReset()
|
||||
queryState.connections.refetch.mockReset()
|
||||
queryState.providers.refetch.mockReset()
|
||||
queryState.providers.data = { items: [firecrawlProvider, jinaProvider] }
|
||||
@@ -203,6 +263,13 @@ describe('AddSourcePage', () => {
|
||||
queryState.connections.isFetchNextPageError = false
|
||||
queryState.connections.isFetchingNextPage = false
|
||||
queryState.connections.isPending = false
|
||||
clientMock.createSource.mockResolvedValue(provisionalSource)
|
||||
clientMock.deleteSource.mockResolvedValue({ id: 'deletion-1' })
|
||||
clientMock.getPages.mockResolvedValue({ items: [] })
|
||||
clientMock.getRun.mockResolvedValue({ ...previewRun, state: 'preview_ready' })
|
||||
clientMock.getSource.mockResolvedValue(provisionalSource)
|
||||
clientMock.listSources.mockResolvedValue({ items: [] })
|
||||
clientMock.startPreview.mockResolvedValue(previewRun)
|
||||
})
|
||||
|
||||
it('loads the provider catalog and every scoped connection cursor page', () => {
|
||||
@@ -254,13 +321,10 @@ describe('AddSourcePage', () => {
|
||||
|
||||
expect(screen.getByText(/dataset\.newKnowledge\.providerConnected/)).toBeInTheDocument()
|
||||
expect(screen.getByRole('radio', { name: 'Firecrawl' })).toBeChecked()
|
||||
expect(screen.getByRole('textbox', { name: /dataset\.newKnowledge\.rootUrl/ })).toBeEnabled()
|
||||
expect(
|
||||
screen.getByText('dataset.newKnowledge.crawlSetupUnavailableDescription'),
|
||||
).toBeInTheDocument()
|
||||
expect(screen.queryByRole('textbox')).not.toBeInTheDocument()
|
||||
expect(
|
||||
screen.queryByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }),
|
||||
).not.toBeInTheDocument()
|
||||
screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }),
|
||||
).toBeEnabled()
|
||||
})
|
||||
|
||||
it('creates the exact Firecrawl provider connection without leaking credentials', async () => {
|
||||
@@ -760,6 +824,265 @@ describe('AddSourcePage', () => {
|
||||
expect(toastInfoMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('asks before discarding an edited source draft', async () => {
|
||||
const user = userEvent.setup()
|
||||
queryState.connections.data = { pages: [{ items: [connection('active')] }] }
|
||||
|
||||
render(<AddSourcePage knowledgeSpaceId="space-1" />)
|
||||
await user.type(
|
||||
screen.getByRole('textbox', { name: /dataset\.newKnowledge\.rootUrl/ }),
|
||||
'https://docs.dify.ai',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.cancelAddSource' }))
|
||||
|
||||
const dialog = await screen.findByRole('alertdialog', {
|
||||
name: 'dataset.newKnowledge.discardSourceDraftTitle',
|
||||
})
|
||||
expect(dialog).toHaveTextContent('dataset.newKnowledge.discardSourceDraftDescription')
|
||||
await user.click(within(dialog).getByRole('button', { name: 'common.operation.cancel' }))
|
||||
expect(routerMock.replace).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('asks before discarding provider connection credentials', async () => {
|
||||
const user = userEvent.setup()
|
||||
|
||||
render(<AddSourcePage knowledgeSpaceId="space-1" />)
|
||||
await user.click(
|
||||
screen.getByRole('button', { name: /dataset\.newKnowledge\.configureProvider/ }),
|
||||
)
|
||||
await user.type(screen.getByLabelText(/Api Key/), 'unsaved-secret')
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.cancelAddSource' }))
|
||||
|
||||
expect(
|
||||
await screen.findByRole('alertdialog', {
|
||||
name: 'dataset.newKnowledge.discardSourceDraftTitle',
|
||||
}),
|
||||
).toBeInTheDocument()
|
||||
expect(routerMock.replace).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('guards client-side marketplace navigation while the source draft is dirty', async () => {
|
||||
const user = userEvent.setup()
|
||||
const historyBack = vi.spyOn(window.history, 'back').mockImplementation(() => undefined)
|
||||
queryState.connections.data = { pages: [{ items: [connection('active')] }] }
|
||||
|
||||
render(<AddSourcePage knowledgeSpaceId="space-1" />)
|
||||
await user.type(
|
||||
screen.getByRole('textbox', { name: /dataset\.newKnowledge\.rootUrl/ }),
|
||||
'https://docs.dify.ai',
|
||||
)
|
||||
await user.click(screen.getByRole('link', { name: /dataset\.newKnowledge\.moreProviders/ }))
|
||||
|
||||
const dialog = await screen.findByRole('alertdialog', {
|
||||
name: 'dataset.newKnowledge.discardSourceDraftTitle',
|
||||
})
|
||||
expect(routerMock.replace).not.toHaveBeenCalled()
|
||||
await user.click(
|
||||
within(dialog).getByRole('button', { name: 'dataset.newKnowledge.discardDraftConfirm' }),
|
||||
)
|
||||
await waitFor(() => expect(historyBack).toHaveBeenCalledOnce())
|
||||
|
||||
act(() => window.dispatchEvent(new PopStateEvent('popstate')))
|
||||
expect(routerMock.replace).toHaveBeenCalledWith('/marketplace?category=datasource')
|
||||
})
|
||||
|
||||
it('guards client-side navigation from links outside the add source form', async () => {
|
||||
const user = userEvent.setup()
|
||||
queryState.connections.data = { pages: [{ items: [connection('active')] }] }
|
||||
|
||||
render(
|
||||
<>
|
||||
<a href="/datasets/space-1/documents">Documents</a>
|
||||
<AddSourcePage knowledgeSpaceId="space-1" />
|
||||
</>,
|
||||
)
|
||||
await user.type(
|
||||
screen.getByRole('textbox', { name: /dataset\.newKnowledge\.rootUrl/ }),
|
||||
'https://docs.dify.ai',
|
||||
)
|
||||
await user.click(screen.getByRole('link', { name: 'Documents' }))
|
||||
|
||||
expect(
|
||||
await screen.findByRole('alertdialog', {
|
||||
name: 'dataset.newKnowledge.discardSourceDraftTitle',
|
||||
}),
|
||||
).toBeInTheDocument()
|
||||
expect(routerMock.replace).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('waits for source creation before deleting the provisional source on discard', async () => {
|
||||
const user = userEvent.setup()
|
||||
const sourceRequest = deferred<typeof provisionalSource>()
|
||||
queryState.connections.data = { pages: [{ items: [connection('active')] }] }
|
||||
clientMock.createSource.mockReturnValue(sourceRequest.promise)
|
||||
|
||||
render(<AddSourcePage knowledgeSpaceId="space-1" />)
|
||||
await user.type(
|
||||
screen.getByRole('textbox', { name: /dataset\.newKnowledge\.rootUrl/ }),
|
||||
'https://docs.dify.ai',
|
||||
)
|
||||
await user.type(
|
||||
screen.getByRole('textbox', { name: /dataset\.newKnowledge\.sourceName/ }),
|
||||
'Dify docs',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await waitFor(() => expect(clientMock.createSource).toHaveBeenCalledOnce())
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.cancelAddSource' }))
|
||||
const dialog = await screen.findByRole('alertdialog', {
|
||||
name: 'dataset.newKnowledge.discardSourceDraftTitle',
|
||||
})
|
||||
await user.click(
|
||||
within(dialog).getByRole('button', { name: 'dataset.newKnowledge.discardDraftConfirm' }),
|
||||
)
|
||||
expect(clientMock.deleteSource).not.toHaveBeenCalled()
|
||||
|
||||
await act(async () => sourceRequest.resolve(provisionalSource))
|
||||
|
||||
await waitFor(() => expect(clientMock.deleteSource).toHaveBeenCalledOnce())
|
||||
})
|
||||
|
||||
it('starts with a fresh preview draft after switching active providers', async () => {
|
||||
const user = userEvent.setup()
|
||||
const jinaConnection = connection('active', 2, {
|
||||
configuration: { provider: 'jina' },
|
||||
id: 'jina-connection',
|
||||
name: 'Jina Reader',
|
||||
providerId: 'jina-reader',
|
||||
})
|
||||
const jinaSource = {
|
||||
...provisionalSource,
|
||||
connectionId: jinaConnection.id,
|
||||
id: 'source-2',
|
||||
}
|
||||
queryState.connections.data = {
|
||||
pages: [{ items: [connection('active'), jinaConnection] }],
|
||||
}
|
||||
clientMock.createSource
|
||||
.mockResolvedValueOnce(provisionalSource)
|
||||
.mockResolvedValueOnce(jinaSource)
|
||||
|
||||
render(<AddSourcePage knowledgeSpaceId="space-1" />)
|
||||
await user.type(
|
||||
screen.getByRole('textbox', { name: /dataset\.newKnowledge\.rootUrl/ }),
|
||||
'https://docs.dify.ai',
|
||||
)
|
||||
await user.type(
|
||||
screen.getByRole('textbox', { name: /dataset\.newKnowledge\.sourceName/ }),
|
||||
'Dify docs',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await screen.findByText(/^dataset\.newKnowledge\.noPagesFound:/)
|
||||
|
||||
await user.click(screen.getByRole('radio', { name: 'Jina Reader' }))
|
||||
|
||||
expect(screen.getByRole('textbox', { name: /dataset\.newKnowledge\.rootUrl/ })).toHaveValue('')
|
||||
await user.type(
|
||||
screen.getByRole('textbox', { name: /dataset\.newKnowledge\.rootUrl/ }),
|
||||
'https://jina.ai',
|
||||
)
|
||||
await user.type(
|
||||
screen.getByRole('textbox', { name: /dataset\.newKnowledge\.sourceName/ }),
|
||||
'Jina docs',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.createSource).toHaveBeenCalledTimes(2))
|
||||
expect(clientMock.createSource.mock.calls[1]?.[0].body.connectionId).toBe('jina-connection')
|
||||
})
|
||||
|
||||
it('reconciles a response-lost provisional source deletion before leaving', async () => {
|
||||
const user = userEvent.setup()
|
||||
const historyBack = vi.spyOn(window.history, 'back').mockImplementation(() => undefined)
|
||||
queryState.connections.data = { pages: [{ items: [connection('active')] }] }
|
||||
clientMock.deleteSource.mockRejectedValue(new Error('response lost'))
|
||||
clientMock.getSource.mockRejectedValue(
|
||||
Object.assign(new Error('source not found'), { status: 404 }),
|
||||
)
|
||||
|
||||
render(<AddSourcePage knowledgeSpaceId="space-1" />)
|
||||
await user.type(
|
||||
screen.getByRole('textbox', { name: /dataset\.newKnowledge\.rootUrl/ }),
|
||||
'https://docs.dify.ai',
|
||||
)
|
||||
await user.type(
|
||||
screen.getByRole('textbox', { name: /dataset\.newKnowledge\.sourceName/ }),
|
||||
'Dify docs',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await waitFor(() => expect(clientMock.createSource).toHaveBeenCalledOnce())
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.cancelAddSource' }))
|
||||
const dialog = await screen.findByRole('alertdialog', {
|
||||
name: 'dataset.newKnowledge.discardSourceDraftTitle',
|
||||
})
|
||||
await user.click(
|
||||
within(dialog).getByRole('button', { name: 'dataset.newKnowledge.discardDraftConfirm' }),
|
||||
)
|
||||
|
||||
await waitFor(() =>
|
||||
expect(clientMock.deleteSource).toHaveBeenCalledWith({
|
||||
body: { expectedRevision: 1 },
|
||||
headers: { 'idempotency-key': expect.any(String) },
|
||||
params: { id: 'space-1', sourceId: 'source-1' },
|
||||
query: { documents: 'cascade' },
|
||||
}),
|
||||
)
|
||||
expect(clientMock.getSource).toHaveBeenCalledWith({
|
||||
params: { id: 'space-1', sourceId: 'source-1' },
|
||||
})
|
||||
expect(historyBack).toHaveBeenCalledOnce()
|
||||
act(() => window.dispatchEvent(new PopStateEvent('popstate')))
|
||||
expect(routerMock.replace).toHaveBeenCalledWith('/datasets/new/space-1/sources')
|
||||
})
|
||||
|
||||
it('protects an edited source draft from browser unload', async () => {
|
||||
const user = userEvent.setup()
|
||||
queryState.connections.data = { pages: [{ items: [connection('active')] }] }
|
||||
|
||||
render(<AddSourcePage knowledgeSpaceId="space-1" />)
|
||||
await user.type(
|
||||
screen.getByRole('textbox', { name: /dataset\.newKnowledge\.rootUrl/ }),
|
||||
'https://docs.dify.ai',
|
||||
)
|
||||
const event = new Event('beforeunload', { cancelable: true })
|
||||
|
||||
act(() => window.dispatchEvent(event))
|
||||
|
||||
expect(event.defaultPrevented).toBe(true)
|
||||
})
|
||||
|
||||
it('deletes a known provisional source with keepalive when the page unloads', async () => {
|
||||
const user = userEvent.setup()
|
||||
queryState.connections.data = { pages: [{ items: [connection('active')] }] }
|
||||
|
||||
render(<AddSourcePage knowledgeSpaceId="space-1" />)
|
||||
await user.type(
|
||||
screen.getByRole('textbox', { name: /dataset\.newKnowledge\.rootUrl/ }),
|
||||
'https://docs.dify.ai',
|
||||
)
|
||||
await user.type(
|
||||
screen.getByRole('textbox', { name: /dataset\.newKnowledge\.sourceName/ }),
|
||||
'Dify docs',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await waitFor(() => expect(clientMock.createSource).toHaveBeenCalledOnce())
|
||||
|
||||
act(() => window.dispatchEvent(new Event('pagehide')))
|
||||
|
||||
await waitFor(() =>
|
||||
expect(clientMock.deleteSource).toHaveBeenCalledWith(
|
||||
{
|
||||
body: { expectedRevision: 1 },
|
||||
headers: { 'idempotency-key': expect.any(String) },
|
||||
params: { id: 'space-1', sourceId: 'source-1' },
|
||||
query: { documents: 'cascade' },
|
||||
},
|
||||
{ context: { keepalive: true, silent: true } },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
it('shows catalog unavailability instead of offering a fake connection', () => {
|
||||
queryState.providers.data = {
|
||||
items: [{ ...firecrawlProvider, available: false, unavailableReason: 'Disabled by admin' }],
|
||||
|
||||
@@ -0,0 +1,877 @@
|
||||
import type { Source, SourceWorkflowRun } from '@dify/contracts/knowledge-fs/types.gen'
|
||||
import { act, screen, waitFor } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { render } from '@/test/console/render'
|
||||
import { WebsiteCrawlPreview } from '../website-crawl-preview'
|
||||
|
||||
const clientMock = vi.hoisted(() => ({
|
||||
cancel: vi.fn(),
|
||||
createSource: vi.fn(),
|
||||
getPages: vi.fn(),
|
||||
getRun: vi.fn(),
|
||||
listSources: vi.fn(),
|
||||
retry: vi.fn(),
|
||||
startPreview: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/service/client', () => ({
|
||||
consoleClient: {
|
||||
knowledgeFs: {
|
||||
getKnowledgeSpacesByIdSources: clientMock.listSources,
|
||||
getKnowledgeSpacesByIdSourceWorkflowsByRunId: clientMock.getRun,
|
||||
getKnowledgeSpacesByIdSourceWorkflowsByRunIdPages: clientMock.getPages,
|
||||
postKnowledgeSpacesByIdSources: clientMock.createSource,
|
||||
postKnowledgeSpacesByIdSourcesBySourceIdCrawlPreview: clientMock.startPreview,
|
||||
postKnowledgeSpacesByIdSourceWorkflowsByRunIdCancel: clientMock.cancel,
|
||||
postKnowledgeSpacesByIdSourceWorkflowsByRunIdRetry: clientMock.retry,
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
const connection = {
|
||||
id: 'connection-1',
|
||||
providerId: 'plugin-daemon-website',
|
||||
}
|
||||
|
||||
const source = (metadata: Source['metadata'] = {}): Source => ({
|
||||
connectionId: connection.id,
|
||||
createdAt: '2026-07-20T10:00:00Z',
|
||||
id: 'source-1',
|
||||
knowledgeSpaceId: 'space-1',
|
||||
metadata,
|
||||
name: 'Dify docs',
|
||||
status: 'disabled',
|
||||
type: 'web',
|
||||
updatedAt: '2026-07-20T10:00:00Z',
|
||||
uri: 'https://docs.dify.ai/',
|
||||
})
|
||||
|
||||
const run = (state: string, overrides: Partial<SourceWorkflowRun> = {}): SourceWorkflowRun => ({
|
||||
checkpoint: 'crawl',
|
||||
createdAt: '2026-07-20T10:00:00Z',
|
||||
executionAttempts: 1,
|
||||
id: 'run-1',
|
||||
knowledgeSpaceId: 'space-1',
|
||||
kind: 'crawl-preview',
|
||||
maxExecutionAttempts: 3,
|
||||
progressCompleted: 0,
|
||||
progressFailed: 0,
|
||||
progressSkipped: 0,
|
||||
sourceId: 'source-1',
|
||||
state,
|
||||
updatedAt: '2026-07-20T10:00:00Z',
|
||||
...overrides,
|
||||
})
|
||||
|
||||
async function fillValidForm() {
|
||||
const user = userEvent.setup()
|
||||
await user.type(screen.getByLabelText(/^dataset\.newKnowledge\.rootUrl/), 'https://docs.dify.ai')
|
||||
await user.type(screen.getByLabelText(/^dataset\.newKnowledge\.sourceName/), 'Dify docs')
|
||||
return user
|
||||
}
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void
|
||||
const promise = new Promise<T>((nextResolve) => {
|
||||
resolve = nextResolve
|
||||
})
|
||||
return { promise, resolve }
|
||||
}
|
||||
|
||||
describe('WebsiteCrawlPreview', () => {
|
||||
beforeEach(() => {
|
||||
vi.useRealTimers()
|
||||
for (const mock of Object.values(clientMock)) mock.mockReset()
|
||||
clientMock.createSource.mockResolvedValue(source())
|
||||
clientMock.startPreview.mockResolvedValue(run('running'))
|
||||
clientMock.getRun.mockResolvedValue(
|
||||
run('preview_ready', {
|
||||
checkpoint: 'preview-staged',
|
||||
progressCompleted: 1,
|
||||
progressTotal: 1,
|
||||
}),
|
||||
)
|
||||
clientMock.getPages.mockResolvedValue({
|
||||
items: [
|
||||
{
|
||||
pageId: 'page-1',
|
||||
sourceUrl: 'https://docs.dify.ai/getting-started',
|
||||
title: 'Getting started',
|
||||
},
|
||||
],
|
||||
})
|
||||
clientMock.listSources.mockResolvedValue({ items: [] })
|
||||
})
|
||||
|
||||
afterEach(() => vi.useRealTimers())
|
||||
|
||||
it('validates the URL and starts a real crawl preview with a provisional source', async () => {
|
||||
const user = userEvent.setup()
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
|
||||
const start = screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' })
|
||||
const rootUrl = screen.getByLabelText(/^dataset\.newKnowledge\.rootUrl/)
|
||||
expect(start).toBeEnabled()
|
||||
await user.click(start)
|
||||
expect(rootUrl).toHaveFocus()
|
||||
expect(rootUrl).toHaveAttribute('aria-invalid', 'true')
|
||||
await user.type(rootUrl, 'ftp://docs.dify.ai')
|
||||
await user.tab()
|
||||
expect(rootUrl).toHaveAttribute('aria-invalid', 'true')
|
||||
await user.clear(rootUrl)
|
||||
await user.type(rootUrl, 'https://docs.dify.ai')
|
||||
await user.type(screen.getByLabelText(/^dataset\.newKnowledge\.sourceName/), 'Dify docs')
|
||||
await user.click(start)
|
||||
|
||||
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledOnce())
|
||||
expect(clientMock.createSource).toHaveBeenCalledWith({
|
||||
body: {
|
||||
connectionId: 'connection-1',
|
||||
metadata: {
|
||||
clientRequestId: expect.any(String),
|
||||
crawlOptions: { includeSubpages: true, limit: 100 },
|
||||
preview: true,
|
||||
providerId: 'plugin-daemon-website',
|
||||
},
|
||||
name: 'Dify docs',
|
||||
status: 'disabled',
|
||||
type: 'web',
|
||||
uri: 'https://docs.dify.ai/',
|
||||
},
|
||||
params: { id: 'space-1' },
|
||||
})
|
||||
expect(clientMock.startPreview).toHaveBeenCalledWith({
|
||||
headers: { 'Idempotency-Key': expect.any(String) },
|
||||
params: { id: 'space-1', sourceId: 'source-1' },
|
||||
})
|
||||
expect(await screen.findByText('Getting started')).toBeInTheDocument()
|
||||
expect(screen.getByText(/^dataset\.newKnowledge\.pagesCrawled/)).toHaveAttribute(
|
||||
'role',
|
||||
'status',
|
||||
)
|
||||
expect(screen.getByRole('button', { name: 'dataset.newKnowledge.reCrawl' })).toBeInTheDocument()
|
||||
expect(
|
||||
screen.queryByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }),
|
||||
).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('uses the shared form controls with stable field names and autocomplete metadata', async () => {
|
||||
const user = userEvent.setup()
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
|
||||
expect(screen.getByLabelText(/^dataset\.newKnowledge\.rootUrl/)).toHaveAttribute(
|
||||
'name',
|
||||
'root-url',
|
||||
)
|
||||
expect(screen.getByLabelText(/^dataset\.newKnowledge\.rootUrl/)).toHaveAttribute(
|
||||
'autocomplete',
|
||||
'off',
|
||||
)
|
||||
expect(screen.getByLabelText(/^dataset\.newKnowledge\.sourceName/)).toHaveAttribute(
|
||||
'name',
|
||||
'source-name',
|
||||
)
|
||||
expect(screen.getByLabelText(/^dataset\.newKnowledge\.sourceName/)).toHaveAttribute(
|
||||
'autocomplete',
|
||||
'off',
|
||||
)
|
||||
|
||||
await user.click(screen.getByRole('button', { name: /dataset\.newKnowledge\.crawlOptions/ }))
|
||||
|
||||
expect(
|
||||
screen.getByRole('checkbox', { name: 'dataset.newKnowledge.includeSubpages' }),
|
||||
).toBeInTheDocument()
|
||||
expect(document.querySelector('input[name="include-subpages"]')).toBeInTheDocument()
|
||||
expect(screen.getByRole('textbox', { name: 'dataset.newKnowledge.maxPages' })).toHaveAttribute(
|
||||
'name',
|
||||
'page-limit',
|
||||
)
|
||||
})
|
||||
|
||||
it('only labels collapsed crawl options as defaults when they are unchanged', async () => {
|
||||
const user = userEvent.setup()
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
|
||||
expect(screen.getByText('dataset.newKnowledge.usingDefaults')).toBeInTheDocument()
|
||||
await user.click(screen.getByRole('button', { name: /dataset\.newKnowledge\.crawlOptions/ }))
|
||||
await user.click(screen.getByRole('checkbox', { name: 'dataset.newKnowledge.includeSubpages' }))
|
||||
await user.click(screen.getByRole('button', { name: /dataset\.newKnowledge\.crawlOptions/ }))
|
||||
|
||||
expect(screen.queryByText('dataset.newKnowledge.usingDefaults')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('keeps completed results associated with the submitted URL while the draft is edited', async () => {
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await screen.findByText('Getting started')
|
||||
|
||||
const rootUrl = screen.getByLabelText(/^dataset\.newKnowledge\.rootUrl/)
|
||||
await user.clear(rootUrl)
|
||||
await user.type(rootUrl, 'https://example.com')
|
||||
|
||||
const completedStatus = screen.getByText(/^dataset\.newKnowledge\.pagesCrawled/)
|
||||
expect(completedStatus).toHaveTextContent('docs.dify.ai')
|
||||
expect(completedStatus).not.toHaveTextContent('example.com')
|
||||
})
|
||||
|
||||
it('submits the crawl form with Enter and enforces the source name contract limit', async () => {
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
const name = screen.getByLabelText(/^dataset\.newKnowledge\.sourceName/)
|
||||
expect(name).toHaveAttribute('maxlength', '200')
|
||||
await user.clear(name)
|
||||
await user.type(name, 'x'.repeat(201))
|
||||
expect(name).toHaveValue('x'.repeat(200))
|
||||
await user.type(name, '{enter}')
|
||||
|
||||
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledOnce())
|
||||
})
|
||||
|
||||
it('shows pending feedback while a completed crawl is being restarted', async () => {
|
||||
const restartRequest = deferred<SourceWorkflowRun>()
|
||||
clientMock.startPreview
|
||||
.mockResolvedValueOnce(run('running'))
|
||||
.mockReturnValueOnce(restartRequest.promise)
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
const reCrawl = await screen.findByRole('button', { name: 'dataset.newKnowledge.reCrawl' })
|
||||
await user.click(reCrawl)
|
||||
|
||||
expect(screen.getByRole('button', { name: 'dataset.newKnowledge.crawling' })).toHaveAttribute(
|
||||
'aria-disabled',
|
||||
'true',
|
||||
)
|
||||
await act(async () =>
|
||||
restartRequest.resolve(
|
||||
run('running', {
|
||||
id: 'run-2',
|
||||
updatedAt: '2026-07-20T10:01:00Z',
|
||||
}),
|
||||
),
|
||||
)
|
||||
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledTimes(2))
|
||||
const firstKey = clientMock.startPreview.mock.calls[0]?.[0].headers['Idempotency-Key']
|
||||
const secondKey = clientMock.startPreview.mock.calls[1]?.[0].headers['Idempotency-Key']
|
||||
expect(secondKey).not.toBe(firstKey)
|
||||
expect(clientMock.createSource).toHaveBeenCalledOnce()
|
||||
expect(clientMock.retry).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('streams page cursors while running and replaces them with the final snapshot', async () => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true })
|
||||
clientMock.getRun
|
||||
.mockResolvedValueOnce(run('running', { progressCompleted: 2, progressTotal: 4 }))
|
||||
.mockResolvedValueOnce(run('succeeded', { progressCompleted: 4, progressTotal: 4 }))
|
||||
clientMock.getPages
|
||||
.mockResolvedValueOnce({
|
||||
items: [{ pageId: 'page-1', sourceUrl: 'https://docs.dify.ai/one', title: 'Old one' }],
|
||||
nextCursor: 'page-2',
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
items: [
|
||||
{ pageId: 'page-2', sourceUrl: 'https://docs.dify.ai/two', title: 'Two' },
|
||||
{
|
||||
pageId: 'page-deleted',
|
||||
sourceUrl: 'https://docs.dify.ai/deleted',
|
||||
title: 'Deleted page',
|
||||
},
|
||||
],
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
items: [
|
||||
{ pageId: 'page-0', sourceUrl: 'https://docs.dify.ai/new', title: 'New first' },
|
||||
{ pageId: 'page-2', sourceUrl: 'https://docs.dify.ai/two', title: 'Two' },
|
||||
],
|
||||
nextCursor: 'final-page-2',
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
items: [
|
||||
{ pageId: 'page-1', sourceUrl: 'https://docs.dify.ai/one', title: 'Updated one' },
|
||||
{ pageId: 'page-3', sourceUrl: 'https://docs.dify.ai/three', title: 'Three' },
|
||||
],
|
||||
})
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
|
||||
expect(await screen.findByText('Two')).toBeInTheDocument()
|
||||
expect(clientMock.getRun).toHaveBeenNthCalledWith(1, {
|
||||
params: { id: 'space-1', runId: 'run-1' },
|
||||
})
|
||||
expect(clientMock.getPages).toHaveBeenNthCalledWith(1, {
|
||||
params: { id: 'space-1', runId: 'run-1' },
|
||||
query: { limit: 200 },
|
||||
})
|
||||
expect(clientMock.getPages).toHaveBeenNthCalledWith(2, {
|
||||
params: { id: 'space-1', runId: 'run-1' },
|
||||
query: { cursor: 'page-2', limit: 200 },
|
||||
})
|
||||
expect(
|
||||
screen.getByRole('progressbar', {
|
||||
name: 'dataset.newKnowledge.crawlProgress:{"host":"docs.dify.ai"}',
|
||||
}),
|
||||
).toHaveValue(3)
|
||||
const status = screen.getByText(/^dataset\.newKnowledge\.crawlingPages/)
|
||||
expect(status).toHaveAttribute('role', 'status')
|
||||
expect(status.querySelector('button')).not.toBeInTheDocument()
|
||||
const skeletons = screen.getAllByTestId('crawl-page-skeleton')
|
||||
expect(skeletons).toHaveLength(2)
|
||||
expect(skeletons[0]?.closest('ul')).toBe(screen.getByRole('list'))
|
||||
await act(async () => vi.advanceTimersByTime(1500))
|
||||
expect(await screen.findByText('Three')).toBeInTheDocument()
|
||||
expect(screen.getByText('New first')).toBeInTheDocument()
|
||||
expect(screen.getByText('Updated one')).toBeInTheDocument()
|
||||
expect(screen.queryByText('Old one')).not.toBeInTheDocument()
|
||||
expect(screen.queryByText('Deleted page')).not.toBeInTheDocument()
|
||||
expect(clientMock.getPages).toHaveBeenNthCalledWith(3, {
|
||||
params: { id: 'space-1', runId: 'run-1' },
|
||||
query: { limit: 200 },
|
||||
})
|
||||
expect(clientMock.getPages).toHaveBeenNthCalledWith(4, {
|
||||
params: { id: 'space-1', runId: 'run-1' },
|
||||
query: { cursor: 'final-page-2', limit: 200 },
|
||||
})
|
||||
expect(
|
||||
screen.getAllByRole('checkbox').map((checkbox) => checkbox.getAttribute('aria-label')),
|
||||
).toEqual(['New first', 'Two', 'Updated one', 'Three'])
|
||||
expect(
|
||||
screen.queryByRole('button', { name: 'dataset.newKnowledge.stopCrawl' }),
|
||||
).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('stops the active run once and keeps pages already discovered', async () => {
|
||||
clientMock.getRun.mockResolvedValue(run('running', { progressCompleted: 1 }))
|
||||
clientMock.cancel.mockResolvedValue(run('canceled', { progressCompleted: 1 }))
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await screen.findByText('Getting started')
|
||||
const stop = screen.getByRole('button', { name: 'dataset.newKnowledge.stopCrawl' })
|
||||
await user.dblClick(stop)
|
||||
|
||||
await waitFor(() => expect(clientMock.cancel).toHaveBeenCalledOnce())
|
||||
expect(clientMock.cancel).toHaveBeenCalledWith({
|
||||
body: { reason: 'user_requested' },
|
||||
params: { id: 'space-1', runId: 'run-1' },
|
||||
})
|
||||
expect(screen.getByText('Getting started')).toBeInTheDocument()
|
||||
expect(await screen.findByText('dataset.newKnowledge.crawlStopped')).toHaveAttribute(
|
||||
'role',
|
||||
'status',
|
||||
)
|
||||
})
|
||||
|
||||
it('reconciles a lost Stop response without sending cancel twice', async () => {
|
||||
clientMock.getRun
|
||||
.mockResolvedValueOnce(run('running', { progressCompleted: 1 }))
|
||||
.mockResolvedValueOnce(run('canceled', { progressCompleted: 1 }))
|
||||
clientMock.cancel.mockRejectedValue(new Error('response lost'))
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await screen.findByText('Getting started')
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.stopCrawl' }))
|
||||
|
||||
expect(await screen.findByText('dataset.newKnowledge.crawlStopped')).toHaveAttribute(
|
||||
'role',
|
||||
'status',
|
||||
)
|
||||
expect(clientMock.cancel).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('only reconciles Stop while an ambiguous cancel response remains stale', async () => {
|
||||
clientMock.getRun.mockResolvedValue(run('running', { progressCompleted: 1 }))
|
||||
clientMock.cancel.mockRejectedValue(
|
||||
Object.assign(new Error('proxy connection lost'), { status: 502 }),
|
||||
)
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await screen.findByText('Getting started')
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.stopCrawl' }))
|
||||
clientMock.getRun.mockRejectedValueOnce(
|
||||
Object.assign(new Error('reconciliation unavailable'), { status: 503 }),
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.stopCrawl' }))
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.stopCrawl' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.getRun.mock.calls.length).toBeGreaterThanOrEqual(3))
|
||||
expect(clientMock.cancel).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('allows Stop to be retried after a definitive HTTP rejection', async () => {
|
||||
clientMock.getRun.mockResolvedValue(run('running', { progressCompleted: 1 }))
|
||||
clientMock.cancel
|
||||
.mockRejectedValueOnce(Object.assign(new Error('rate limited'), { status: 429 }))
|
||||
.mockResolvedValueOnce(run('canceled', { progressCompleted: 1 }))
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await screen.findByText('Getting started')
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.stopCrawl' }))
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.stopCrawl' }))
|
||||
|
||||
expect(await screen.findByText('dataset.newKnowledge.crawlStopped')).toBeInTheDocument()
|
||||
expect(clientMock.cancel).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('shows provider failures and retries the same workflow without duplicate requests', async () => {
|
||||
clientMock.getRun.mockResolvedValue(
|
||||
run('failed', { lastErrorCode: 'PROVIDER_403', progressFailed: 1 }),
|
||||
)
|
||||
clientMock.getPages.mockResolvedValue({ items: [] })
|
||||
clientMock.retry.mockResolvedValue(run('running'))
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent(
|
||||
'dataset.newKnowledge.crawlFailed403',
|
||||
)
|
||||
const retry = screen.getByRole('button', { name: 'dataset.newKnowledge.retryCrawl' })
|
||||
await user.dblClick(retry)
|
||||
|
||||
await waitFor(() => expect(clientMock.retry).toHaveBeenCalledOnce())
|
||||
expect(clientMock.retry).toHaveBeenCalledWith({
|
||||
params: { id: 'space-1', runId: 'run-1' },
|
||||
})
|
||||
expect(clientMock.createSource).toHaveBeenCalledOnce()
|
||||
expect(clientMock.startPreview).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('reconciles a lost Retry response without sending retry twice', async () => {
|
||||
clientMock.getRun
|
||||
.mockResolvedValueOnce(run('failed', { lastErrorCode: 'PROVIDER_403' }))
|
||||
.mockResolvedValueOnce(
|
||||
run('running', {
|
||||
executionAttempts: 2,
|
||||
updatedAt: '2026-07-20T10:01:00Z',
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
run('succeeded', {
|
||||
executionAttempts: 2,
|
||||
progressCompleted: 1,
|
||||
updatedAt: '2026-07-20T10:02:00Z',
|
||||
}),
|
||||
)
|
||||
clientMock.retry.mockRejectedValue(new Error('response lost'))
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await screen.findByRole('alert')
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.retryCrawl' }))
|
||||
|
||||
expect(
|
||||
await screen.findByRole('button', { name: 'dataset.newKnowledge.reCrawl' }),
|
||||
).toBeInTheDocument()
|
||||
expect(clientMock.retry).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('only reconciles Retry while an ambiguous retry response remains stale', async () => {
|
||||
const failedRun = run('failed', { lastErrorCode: 'PROVIDER_403', progressFailed: 1 })
|
||||
clientMock.getRun
|
||||
.mockResolvedValueOnce(failedRun)
|
||||
.mockResolvedValueOnce(failedRun)
|
||||
.mockRejectedValueOnce(
|
||||
Object.assign(new Error('reconciliation unavailable'), { status: 503 }),
|
||||
)
|
||||
.mockResolvedValue(failedRun)
|
||||
clientMock.getPages.mockResolvedValue({ items: [] })
|
||||
clientMock.retry.mockRejectedValue(Object.assign(new Error('proxy timeout'), { status: 504 }))
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await screen.findByRole('alert')
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.retryCrawl' }))
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.retryCrawl' }))
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.retryCrawl' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.getRun).toHaveBeenCalledTimes(4))
|
||||
expect(clientMock.retry).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('allows Retry to be repeated after a definitive HTTP rejection', async () => {
|
||||
clientMock.getRun.mockResolvedValue(
|
||||
run('failed', { lastErrorCode: 'PROVIDER_403', progressFailed: 1 }),
|
||||
)
|
||||
clientMock.getPages.mockResolvedValue({ items: [] })
|
||||
clientMock.retry
|
||||
.mockRejectedValueOnce(Object.assign(new Error('rate limited'), { status: 429 }))
|
||||
.mockResolvedValueOnce(run('running', { executionAttempts: 2 }))
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await screen.findByRole('alert')
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.retryCrawl' }))
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.retryCrawl' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.retry).toHaveBeenCalledTimes(2))
|
||||
})
|
||||
|
||||
it.each([
|
||||
['failed', 'PROVIDER_TIMEOUT', 'dataset.newKnowledge.crawlFailedTimeout'],
|
||||
['failed', 'PROVIDER_UNAVAILABLE', 'dataset.newKnowledge.crawlFailedProvider'],
|
||||
['timed_out', undefined, 'dataset.newKnowledge.crawlFailedTimeout'],
|
||||
])('shows a recovery message for %s / %s', async (state, lastErrorCode, message) => {
|
||||
clientMock.getRun.mockResolvedValue(run(state, { lastErrorCode }))
|
||||
clientMock.getPages.mockResolvedValue({ items: [] })
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Jina Reader"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
|
||||
const alert = await screen.findByRole('alert')
|
||||
expect(alert).toHaveTextContent(message)
|
||||
if (lastErrorCode === 'PROVIDER_UNAVAILABLE')
|
||||
expect(alert).toHaveTextContent('"provider":"Jina Reader"')
|
||||
})
|
||||
|
||||
it('offers an adjust-and-recrawl path after a successful zero-result crawl', async () => {
|
||||
clientMock.getRun.mockResolvedValue(run('succeeded'))
|
||||
clientMock.getPages.mockResolvedValue({ items: [] })
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
|
||||
const noPages = await screen.findByText(/^dataset\.newKnowledge\.noPagesFound:/)
|
||||
expect(noPages.closest('[role="status"]')).toHaveAttribute('aria-live', 'polite')
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.adjustAndRecrawl' }))
|
||||
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledTimes(2))
|
||||
const firstKey = clientMock.startPreview.mock.calls[0]?.[0].headers['Idempotency-Key']
|
||||
const secondKey = clientMock.startPreview.mock.calls[1]?.[0].headers['Idempotency-Key']
|
||||
expect(secondKey).not.toBe(firstKey)
|
||||
expect(clientMock.createSource).toHaveBeenCalledOnce()
|
||||
expect(clientMock.retry).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reports draft and provisional source lifecycle to the parent flow', async () => {
|
||||
const onDraftChange = vi.fn()
|
||||
const onProvisionalSource = vi.fn()
|
||||
const view = render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
onDraftChange={onDraftChange}
|
||||
onProvisionalSource={onProvisionalSource}
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
|
||||
const user = await fillValidForm()
|
||||
expect(onDraftChange).toHaveBeenLastCalledWith(true)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await waitFor(() => expect(onProvisionalSource).toHaveBeenCalledWith(source()))
|
||||
|
||||
view.unmount()
|
||||
expect(onDraftChange).toHaveBeenLastCalledWith(false)
|
||||
})
|
||||
|
||||
it('treats a superseded workflow as terminal and stops polling', async () => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true })
|
||||
clientMock.getRun.mockResolvedValue(run('superseded'))
|
||||
clientMock.getPages.mockResolvedValue({ items: [] })
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
expect(await screen.findByText('dataset.newKnowledge.crawlStopped')).toHaveAttribute(
|
||||
'role',
|
||||
'status',
|
||||
)
|
||||
await act(async () => vi.advanceTimersByTime(6000))
|
||||
|
||||
expect(clientMock.getRun).toHaveBeenCalledOnce()
|
||||
expect(
|
||||
screen.queryByRole('button', { name: 'dataset.newKnowledge.stopCrawl' }),
|
||||
).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('resumes polling after a transient status request failure without retrying the workflow', async () => {
|
||||
clientMock.getRun
|
||||
.mockRejectedValueOnce(new Error('temporary network failure'))
|
||||
.mockResolvedValueOnce(run('succeeded', { progressCompleted: 1 }))
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent(
|
||||
'dataset.newKnowledge.crawlFailedDescription',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.retryCrawl' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.getRun).toHaveBeenCalledTimes(2))
|
||||
expect(clientMock.retry).not.toHaveBeenCalled()
|
||||
expect(await screen.findByText('Getting started')).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('reuses the idempotency key when the preview start response is lost', async () => {
|
||||
clientMock.startPreview
|
||||
.mockRejectedValueOnce(new Error('response lost'))
|
||||
.mockResolvedValueOnce(run('running'))
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await screen.findByRole('alert')
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.retryCrawl' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledTimes(2))
|
||||
expect(clientMock.createSource).toHaveBeenCalledOnce()
|
||||
expect(clientMock.startPreview.mock.calls[0]?.[0].headers).toEqual(
|
||||
clientMock.startPreview.mock.calls[1]?.[0].headers,
|
||||
)
|
||||
})
|
||||
|
||||
it('reconciles a response-lost provisional source by its client request id', async () => {
|
||||
clientMock.createSource.mockRejectedValue(new Error('response lost'))
|
||||
clientMock.listSources.mockImplementation(
|
||||
async ({ query }: { query?: { cursor?: string } }) => {
|
||||
const clientRequestId =
|
||||
clientMock.createSource.mock.calls[0]?.[0].body.metadata.clientRequestId
|
||||
return query?.cursor
|
||||
? { items: [source({ clientRequestId })] }
|
||||
: { items: [], nextCursor: 'next' }
|
||||
},
|
||||
)
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.listSources).toHaveBeenCalledTimes(2))
|
||||
expect(clientMock.startPreview).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('checks for a response-lost source before repeating its create request', async () => {
|
||||
clientMock.createSource.mockRejectedValue(new Error('response lost'))
|
||||
clientMock.listSources.mockResolvedValueOnce({ items: [] }).mockImplementationOnce(async () => {
|
||||
const clientRequestId =
|
||||
clientMock.createSource.mock.calls[0]?.[0].body.metadata.clientRequestId
|
||||
return { items: [source({ clientRequestId })] }
|
||||
})
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await screen.findByRole('alert')
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.retryCrawl' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledOnce())
|
||||
expect(clientMock.createSource).toHaveBeenCalledOnce()
|
||||
expect(clientMock.listSources).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('retries source creation after bounded reconciliation of an ambiguous result', async () => {
|
||||
clientMock.createSource
|
||||
.mockRejectedValueOnce(Object.assign(new Error('proxy timeout'), { status: 504 }))
|
||||
.mockResolvedValueOnce(source())
|
||||
clientMock.listSources.mockResolvedValue({ items: [] })
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await screen.findByRole('alert')
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.retryCrawl' }))
|
||||
await waitFor(() => expect(clientMock.listSources).toHaveBeenCalledTimes(2))
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.retryCrawl' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledOnce())
|
||||
expect(clientMock.listSources).toHaveBeenCalledTimes(3)
|
||||
expect(clientMock.createSource).toHaveBeenCalledTimes(2)
|
||||
expect(clientMock.createSource.mock.calls[0]?.[0].body.metadata.clientRequestId).toBe(
|
||||
clientMock.createSource.mock.calls[1]?.[0].body.metadata.clientRequestId,
|
||||
)
|
||||
})
|
||||
|
||||
it('retries source creation after a definitive HTTP rejection', async () => {
|
||||
clientMock.createSource
|
||||
.mockRejectedValueOnce(Object.assign(new Error('rate limited'), { status: 429 }))
|
||||
.mockResolvedValueOnce(source())
|
||||
|
||||
render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await screen.findByRole('alert')
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.retryCrawl' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledOnce())
|
||||
expect(clientMock.createSource).toHaveBeenCalledTimes(2)
|
||||
expect(clientMock.listSources).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('cleans up polling when the component unmounts', async () => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true })
|
||||
clientMock.getRun.mockResolvedValue(run('running'))
|
||||
const view = render(
|
||||
<WebsiteCrawlPreview
|
||||
connection={connection}
|
||||
knowledgeSpaceId="space-1"
|
||||
providerName="Firecrawl"
|
||||
/>,
|
||||
)
|
||||
const user = await fillValidForm()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
|
||||
await waitFor(() => expect(clientMock.getRun).toHaveBeenCalledOnce())
|
||||
view.unmount()
|
||||
await act(async () => vi.advanceTimersByTime(6000))
|
||||
|
||||
expect(clientMock.getRun).toHaveBeenCalledOnce()
|
||||
})
|
||||
})
|
||||
@@ -3,16 +3,20 @@
|
||||
import type {
|
||||
GetKnowledgeSpacesByIdSourceConnectionsResponse,
|
||||
GetSourceProvidersResponse,
|
||||
Source,
|
||||
} from '@dify/contracts/knowledge-fs/types.gen'
|
||||
import { Button } from '@langgenius/dify-ui/button'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { useInfiniteQuery, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { useCallback, useEffect, useId, useMemo, useState } from 'react'
|
||||
import { useCallback, useEffect, useId, useMemo, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Loading from '@/app/components/base/loading'
|
||||
import Link from '@/next/link'
|
||||
import { useRouter } from '@/next/navigation'
|
||||
import { consoleClient, consoleQuery } from '@/service/client'
|
||||
import { AddSourceExitDialog } from './components/add-source-exit-dialog'
|
||||
import { newKnowledgeDetailPath } from './routes'
|
||||
import { WebsiteCrawlPreview } from './website-crawl-preview'
|
||||
|
||||
type Provider = GetSourceProvidersResponse['items'][number]
|
||||
type ProviderField = Provider['configuration'][number]
|
||||
@@ -23,6 +27,7 @@ type SourceType = 'onlineDocuments' | 'onlineDrive' | 'websiteCrawl'
|
||||
const CONNECTION_PAGE_SIZE = 200
|
||||
const CONNECTION_RECONCILIATION_RETRY_DELAYS = [0, 100, 300]
|
||||
const FIRECRAWL_PROVIDER_ID = 'plugin-daemon-website'
|
||||
const SOURCE_MARKETPLACE_PATH = '/marketplace?category=datasource'
|
||||
const FIRECRAWL_CONFIGURATION = {
|
||||
datasource: 'crawl',
|
||||
pluginId: 'langgenius/firecrawl_datasource',
|
||||
@@ -81,6 +86,15 @@ function findConnectionById(connections: Connection[], connectionId: string) {
|
||||
)[0]
|
||||
}
|
||||
|
||||
function responseStatus(error: unknown) {
|
||||
if (error instanceof Response) return error.status
|
||||
if (!error || typeof error !== 'object') return undefined
|
||||
if ('status' in error && typeof error.status === 'number') return error.status
|
||||
if ('data' in error && error.data && typeof error.data === 'object' && 'status' in error.data) {
|
||||
return typeof error.data.status === 'number' ? error.data.status : undefined
|
||||
}
|
||||
}
|
||||
|
||||
function getAuthFields(provider: Provider, authKind: ConnectionAuthKind) {
|
||||
const configurableFields = provider.configuration.filter(
|
||||
(field) =>
|
||||
@@ -153,10 +167,12 @@ function SourceTypeSelector({
|
||||
}
|
||||
|
||||
function ProviderSelector({
|
||||
onNavigate,
|
||||
onChange,
|
||||
providers,
|
||||
value,
|
||||
}: {
|
||||
onNavigate: (path: string) => void
|
||||
onChange: (providerId: string) => void
|
||||
providers: Provider[]
|
||||
value?: string
|
||||
@@ -171,8 +187,21 @@ function ProviderSelector({
|
||||
{t(($) => $['newKnowledge.providerLabel'])}
|
||||
</span>
|
||||
<Link
|
||||
href="/marketplace?category=datasource"
|
||||
href={SOURCE_MARKETPLACE_PATH}
|
||||
className="inline-flex items-center gap-0.5 rounded-sm system-xs-medium text-text-accent outline-hidden hover:underline focus-visible:ring-2 focus-visible:ring-state-accent-solid"
|
||||
onClick={(event) => {
|
||||
if (
|
||||
event.defaultPrevented ||
|
||||
event.button !== 0 ||
|
||||
event.metaKey ||
|
||||
event.ctrlKey ||
|
||||
event.shiftKey ||
|
||||
event.altKey
|
||||
)
|
||||
return
|
||||
event.preventDefault()
|
||||
onNavigate(SOURCE_MARKETPLACE_PATH)
|
||||
}}
|
||||
>
|
||||
{t(($) => $['newKnowledge.moreProviders'])}
|
||||
<span aria-hidden className="i-ri-arrow-right-up-line size-3.5" />
|
||||
@@ -270,48 +299,43 @@ function ProviderFieldControl({
|
||||
)
|
||||
}
|
||||
|
||||
function ProviderConnected({ provider }: { provider: Provider }) {
|
||||
const { t } = useTranslation('dataset')
|
||||
|
||||
return (
|
||||
<div role="status" className="rounded-xl bg-background-section p-4">
|
||||
<span className="flex size-9 items-center justify-center rounded-lg border border-divider-subtle bg-background-default">
|
||||
<span
|
||||
aria-hidden
|
||||
className={cn(providerIconClass(provider), 'size-[18px] text-text-tertiary')}
|
||||
/>
|
||||
</span>
|
||||
<h3 className="mt-3 system-sm-semibold text-text-primary">
|
||||
{t(($) => $['newKnowledge.providerConnected'], {
|
||||
provider: provider.displayName,
|
||||
})}
|
||||
</h3>
|
||||
<p className="mt-1 system-xs-regular text-text-tertiary">
|
||||
{t(($) => $['newKnowledge.crawlSetupUnavailableDescription'])}
|
||||
</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function ConnectionForm({
|
||||
knowledgeSpaceId,
|
||||
onConnected,
|
||||
onDraftChange,
|
||||
onReconcile,
|
||||
provider,
|
||||
}: {
|
||||
knowledgeSpaceId: string
|
||||
onConnected: (connection: Connection) => void
|
||||
onDraftChange?: (dirty: boolean) => void
|
||||
onReconcile: () => Promise<Connection | undefined>
|
||||
provider: Provider
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
const supportedAuthKinds = getSupportedAuthKinds(provider)
|
||||
const [authKind, setAuthKind] = useState<ConnectionAuthKind>(supportedAuthKinds[0] ?? 'api-key')
|
||||
const defaultAuthKind = supportedAuthKinds[0] ?? 'api-key'
|
||||
const [authKind, setAuthKind] = useState<ConnectionAuthKind>(defaultAuthKind)
|
||||
const [configuration, setConfiguration] = useState<Record<string, string>>({})
|
||||
const [credentials, setCredentials] = useState<Record<string, string>>({})
|
||||
const [error, setError] = useState(false)
|
||||
const [pending, setPending] = useState(false)
|
||||
const visibleFields = getAuthFields(provider, authKind)
|
||||
const hasDraftChanges =
|
||||
authKind !== defaultAuthKind ||
|
||||
Object.values(configuration).some((value) => Boolean(value.trim())) ||
|
||||
Object.values(credentials).some((value) => Boolean(value.trim()))
|
||||
|
||||
useEffect(() => {
|
||||
onDraftChange?.(hasDraftChanges)
|
||||
}, [hasDraftChanges, onDraftChange])
|
||||
|
||||
useEffect(
|
||||
() => () => {
|
||||
onDraftChange?.(false)
|
||||
},
|
||||
[onDraftChange],
|
||||
)
|
||||
|
||||
const changeAuthKind = (nextAuthKind: ConnectionAuthKind) => {
|
||||
if (nextAuthKind !== authKind) setCredentials({})
|
||||
@@ -437,11 +461,13 @@ function ConnectionForm({
|
||||
function UnconfiguredProvider({
|
||||
knowledgeSpaceId,
|
||||
onConnected,
|
||||
onDraftChange,
|
||||
onReconcile,
|
||||
provider,
|
||||
}: {
|
||||
knowledgeSpaceId: string
|
||||
onConnected: (connection: Connection) => void
|
||||
onDraftChange?: (dirty: boolean) => void
|
||||
onReconcile: () => Promise<Connection | undefined>
|
||||
provider: Provider
|
||||
}) {
|
||||
@@ -453,6 +479,7 @@ function UnconfiguredProvider({
|
||||
<ConnectionForm
|
||||
knowledgeSpaceId={knowledgeSpaceId}
|
||||
onConnected={onConnected}
|
||||
onDraftChange={onDraftChange}
|
||||
onReconcile={onReconcile}
|
||||
provider={provider}
|
||||
/>
|
||||
@@ -626,10 +653,30 @@ function ProvisioningConnection({
|
||||
|
||||
export function AddSourcePage({ knowledgeSpaceId }: { knowledgeSpaceId: string }) {
|
||||
const { t } = useTranslation('dataset')
|
||||
const router = useRouter()
|
||||
const queryClient = useQueryClient()
|
||||
const [sourceType, setSourceType] = useState<SourceType>('websiteCrawl')
|
||||
const [selectedProviderId, setSelectedProviderId] = useState<string>()
|
||||
const [ignoredConnectionIds, setIgnoredConnectionIds] = useState<Set<string>>(() => new Set())
|
||||
const [crawlDraftDirty, setCrawlDraftDirty] = useState(false)
|
||||
const [provisionalSourceCount, setProvisionalSourceCount] = useState(0)
|
||||
const [exitOpen, setExitOpen] = useState(false)
|
||||
const [discarding, setDiscarding] = useState(false)
|
||||
const [discardError, setDiscardError] = useState(false)
|
||||
const detailPath = newKnowledgeDetailPath(knowledgeSpaceId)
|
||||
const provisionalSourcesRef = useRef(new Map<string, Source>())
|
||||
const pendingPreviewOperationsRef = useRef(new Set<Promise<void>>())
|
||||
const deletionKeysRef = useRef(new Map<string, string>())
|
||||
const historyGuardArmedRef = useRef(false)
|
||||
const browserBackExitRef = useRef(false)
|
||||
const pendingNavigationRef = useRef<string | undefined>(undefined)
|
||||
const exitDestinationRef = useRef(detailPath)
|
||||
const hasUnsavedChanges = Boolean(
|
||||
crawlDraftDirty ||
|
||||
provisionalSourceCount ||
|
||||
sourceType !== 'websiteCrawl' ||
|
||||
selectedProviderId,
|
||||
)
|
||||
const providersQuery = useQuery(
|
||||
consoleQuery.knowledgeFs.getSourceProviders.queryOptions({
|
||||
input: {},
|
||||
@@ -699,6 +746,229 @@ export function AddSourcePage({ knowledgeSpaceId }: { knowledgeSpaceId: string }
|
||||
refetch: refetchConnections,
|
||||
} = connectionsQuery
|
||||
|
||||
const armHistoryGuard = useCallback(() => {
|
||||
globalThis.history.pushState(globalThis.history.state, '', globalThis.location.href)
|
||||
historyGuardArmedRef.current = true
|
||||
}, [])
|
||||
|
||||
const replaceAfterHistoryGuard = useCallback(
|
||||
(path: string) => {
|
||||
if (!historyGuardArmedRef.current) {
|
||||
router.replace(path)
|
||||
return
|
||||
}
|
||||
|
||||
pendingNavigationRef.current = path
|
||||
globalThis.history.back()
|
||||
},
|
||||
[router],
|
||||
)
|
||||
|
||||
const requestNavigation = useCallback(
|
||||
(path: string) => {
|
||||
if (discarding) return
|
||||
if (hasUnsavedChanges) {
|
||||
exitDestinationRef.current = path
|
||||
browserBackExitRef.current = false
|
||||
setDiscardError(false)
|
||||
setExitOpen(true)
|
||||
return
|
||||
}
|
||||
replaceAfterHistoryGuard(path)
|
||||
},
|
||||
[discarding, hasUnsavedChanges, replaceAfterHistoryGuard],
|
||||
)
|
||||
|
||||
useEffect(() => {
|
||||
if (
|
||||
!hasUnsavedChanges ||
|
||||
historyGuardArmedRef.current ||
|
||||
browserBackExitRef.current ||
|
||||
pendingNavigationRef.current
|
||||
)
|
||||
return
|
||||
|
||||
armHistoryGuard()
|
||||
}, [armHistoryGuard, hasUnsavedChanges])
|
||||
|
||||
useEffect(() => {
|
||||
const handlePopState = () => {
|
||||
if (!historyGuardArmedRef.current) return
|
||||
|
||||
historyGuardArmedRef.current = false
|
||||
const pendingNavigation = pendingNavigationRef.current
|
||||
if (pendingNavigation) {
|
||||
pendingNavigationRef.current = undefined
|
||||
router.replace(pendingNavigation)
|
||||
return
|
||||
}
|
||||
if (!hasUnsavedChanges) {
|
||||
router.replace(detailPath)
|
||||
return
|
||||
}
|
||||
|
||||
browserBackExitRef.current = true
|
||||
exitDestinationRef.current = detailPath
|
||||
setDiscardError(false)
|
||||
setExitOpen(true)
|
||||
}
|
||||
|
||||
globalThis.addEventListener('popstate', handlePopState)
|
||||
return () => globalThis.removeEventListener('popstate', handlePopState)
|
||||
}, [detailPath, hasUnsavedChanges, router])
|
||||
|
||||
useEffect(() => {
|
||||
if (!hasUnsavedChanges) return
|
||||
|
||||
const handleBeforeUnload = (event: BeforeUnloadEvent) => {
|
||||
event.preventDefault()
|
||||
event.returnValue = ''
|
||||
}
|
||||
|
||||
globalThis.addEventListener('beforeunload', handleBeforeUnload)
|
||||
return () => globalThis.removeEventListener('beforeunload', handleBeforeUnload)
|
||||
}, [hasUnsavedChanges])
|
||||
|
||||
useEffect(() => {
|
||||
if (!hasUnsavedChanges) return
|
||||
|
||||
const handleDocumentClick = (event: MouseEvent) => {
|
||||
if (
|
||||
event.defaultPrevented ||
|
||||
event.button !== 0 ||
|
||||
event.metaKey ||
|
||||
event.ctrlKey ||
|
||||
event.shiftKey ||
|
||||
event.altKey
|
||||
)
|
||||
return
|
||||
|
||||
const anchor = event
|
||||
.composedPath()
|
||||
.find((target): target is HTMLAnchorElement => target instanceof HTMLAnchorElement)
|
||||
if (
|
||||
!anchor ||
|
||||
anchor.hasAttribute('download') ||
|
||||
(anchor.target && anchor.target !== '_self')
|
||||
)
|
||||
return
|
||||
|
||||
const destination = new URL(anchor.href, globalThis.location.href)
|
||||
if (destination.origin !== globalThis.location.origin) return
|
||||
|
||||
const current = new URL(globalThis.location.href)
|
||||
if (destination.pathname === current.pathname && destination.search === current.search) return
|
||||
|
||||
event.preventDefault()
|
||||
requestNavigation(`${destination.pathname}${destination.search}${destination.hash}`)
|
||||
}
|
||||
|
||||
document.addEventListener('click', handleDocumentClick, true)
|
||||
return () => document.removeEventListener('click', handleDocumentClick, true)
|
||||
}, [hasUnsavedChanges, requestNavigation])
|
||||
|
||||
const rememberProvisionalSource = useCallback((source: Source) => {
|
||||
provisionalSourcesRef.current.set(source.id, source)
|
||||
setProvisionalSourceCount(provisionalSourcesRef.current.size)
|
||||
}, [])
|
||||
|
||||
const getDeletionKey = useCallback((sourceId: string) => {
|
||||
let idempotencyKey = deletionKeysRef.current.get(sourceId)
|
||||
if (!idempotencyKey) {
|
||||
idempotencyKey = globalThis.crypto.randomUUID()
|
||||
deletionKeysRef.current.set(sourceId, idempotencyKey)
|
||||
}
|
||||
return idempotencyKey
|
||||
}, [])
|
||||
|
||||
const rememberPendingPreviewOperation = useCallback((operation: Promise<void>) => {
|
||||
pendingPreviewOperationsRef.current.add(operation)
|
||||
void operation.then(
|
||||
() => pendingPreviewOperationsRef.current.delete(operation),
|
||||
() => pendingPreviewOperationsRef.current.delete(operation),
|
||||
)
|
||||
}, [])
|
||||
|
||||
const requestExit = () => requestNavigation(detailPath)
|
||||
|
||||
const cancelExit = () => {
|
||||
setExitOpen(false)
|
||||
setDiscardError(false)
|
||||
exitDestinationRef.current = detailPath
|
||||
if (!browserBackExitRef.current) return
|
||||
|
||||
browserBackExitRef.current = false
|
||||
armHistoryGuard()
|
||||
}
|
||||
|
||||
const confirmExit = async () => {
|
||||
if (discarding) return
|
||||
setDiscarding(true)
|
||||
setDiscardError(false)
|
||||
try {
|
||||
await Promise.allSettled([...pendingPreviewOperationsRef.current])
|
||||
for (const source of provisionalSourcesRef.current.values()) {
|
||||
try {
|
||||
await consoleClient.knowledgeFs.deleteKnowledgeSpacesByIdSourcesBySourceId({
|
||||
body: { expectedRevision: source.version ?? 1 },
|
||||
headers: { 'idempotency-key': getDeletionKey(source.id) },
|
||||
params: { id: knowledgeSpaceId, sourceId: source.id },
|
||||
query: { documents: 'cascade' },
|
||||
})
|
||||
} catch (deletionError) {
|
||||
let deletionConfirmed = false
|
||||
try {
|
||||
await consoleClient.knowledgeFs.getKnowledgeSpacesByIdSourcesBySourceId({
|
||||
params: { id: knowledgeSpaceId, sourceId: source.id },
|
||||
})
|
||||
} catch (reconciliationError) {
|
||||
deletionConfirmed = responseStatus(reconciliationError) === 404
|
||||
}
|
||||
if (!deletionConfirmed) throw deletionError
|
||||
}
|
||||
provisionalSourcesRef.current.delete(source.id)
|
||||
deletionKeysRef.current.delete(source.id)
|
||||
setProvisionalSourceCount(provisionalSourcesRef.current.size)
|
||||
}
|
||||
setCrawlDraftDirty(false)
|
||||
setExitOpen(false)
|
||||
browserBackExitRef.current = false
|
||||
const exitDestination = exitDestinationRef.current
|
||||
exitDestinationRef.current = detailPath
|
||||
replaceAfterHistoryGuard(exitDestination)
|
||||
} catch {
|
||||
setDiscardError(true)
|
||||
} finally {
|
||||
setDiscarding(false)
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
const handlePageHide = () => {
|
||||
for (const source of provisionalSourcesRef.current.values()) {
|
||||
void consoleClient.knowledgeFs
|
||||
.deleteKnowledgeSpacesByIdSourcesBySourceId(
|
||||
{
|
||||
body: { expectedRevision: source.version ?? 1 },
|
||||
headers: { 'idempotency-key': getDeletionKey(source.id) },
|
||||
params: { id: knowledgeSpaceId, sourceId: source.id },
|
||||
query: { documents: 'cascade' },
|
||||
},
|
||||
{
|
||||
context: {
|
||||
keepalive: true,
|
||||
silent: true,
|
||||
},
|
||||
},
|
||||
)
|
||||
.catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
globalThis.addEventListener('pagehide', handlePageHide)
|
||||
return () => globalThis.removeEventListener('pagehide', handlePageHide)
|
||||
}, [getDeletionKey, knowledgeSpaceId])
|
||||
|
||||
useEffect(() => {
|
||||
if (
|
||||
hasNextConnectionPage &&
|
||||
@@ -814,6 +1084,7 @@ export function AddSourcePage({ knowledgeSpaceId }: { knowledgeSpaceId: string }
|
||||
{sourceType === 'websiteCrawl' ? (
|
||||
<>
|
||||
<ProviderSelector
|
||||
onNavigate={requestNavigation}
|
||||
providers={providers}
|
||||
value={provider?.id}
|
||||
onChange={setSelectedProviderId}
|
||||
@@ -844,7 +1115,15 @@ export function AddSourcePage({ knowledgeSpaceId }: { knowledgeSpaceId: string }
|
||||
</p>
|
||||
</div>
|
||||
) : connection?.status === 'active' ? (
|
||||
<ProviderConnected key={provider.id} provider={provider} />
|
||||
<WebsiteCrawlPreview
|
||||
key={connection.id}
|
||||
connection={connection}
|
||||
knowledgeSpaceId={knowledgeSpaceId}
|
||||
onDraftChange={setCrawlDraftDirty}
|
||||
onPendingOperation={rememberPendingPreviewOperation}
|
||||
onProvisionalSource={rememberProvisionalSource}
|
||||
providerName={provider.displayName}
|
||||
/>
|
||||
) : connection?.status === 'provisioning' ? (
|
||||
<ProvisioningConnection
|
||||
key={provider.id}
|
||||
@@ -866,6 +1145,7 @@ export function AddSourcePage({ knowledgeSpaceId }: { knowledgeSpaceId: string }
|
||||
key={provider.id}
|
||||
knowledgeSpaceId={knowledgeSpaceId}
|
||||
onConnected={rememberConnection}
|
||||
onDraftChange={setCrawlDraftDirty}
|
||||
onReconcile={reconcileConnection}
|
||||
provider={provider}
|
||||
/>
|
||||
@@ -882,17 +1162,36 @@ export function AddSourcePage({ knowledgeSpaceId }: { knowledgeSpaceId: string }
|
||||
</div>
|
||||
)}
|
||||
<div className="flex justify-end gap-2 border-t border-divider-subtle pt-5">
|
||||
<Link
|
||||
href={newKnowledgeDetailPath(knowledgeSpaceId)}
|
||||
className="inline-flex h-8 items-center justify-center rounded-lg border-[0.5px] border-components-button-secondary-border bg-components-button-secondary-bg px-3.5 system-sm-medium text-components-button-secondary-text shadow-xs outline-hidden hover:bg-components-button-secondary-bg-hover focus-visible:ring-2 focus-visible:ring-state-accent-solid"
|
||||
>
|
||||
<Button type="button" onClick={requestExit}>
|
||||
{t(($) => $['newKnowledge.cancelAddSource'])}
|
||||
</Link>
|
||||
<Button variant="primary" disabled>
|
||||
{t(($) => $['newKnowledge.addSource'])}
|
||||
</Button>
|
||||
{sourceType === 'websiteCrawl' ? (
|
||||
<>
|
||||
<span id="add-source-selection-requirement" className="sr-only">
|
||||
{t(($) => $['newKnowledge.addSourceRequiresSelection'])}
|
||||
</span>
|
||||
<Button
|
||||
variant="primary"
|
||||
disabled
|
||||
aria-describedby="add-source-selection-requirement"
|
||||
>
|
||||
{t(($) => $['newKnowledge.addSource'])}
|
||||
</Button>
|
||||
</>
|
||||
) : (
|
||||
<Button variant="primary" disabled>
|
||||
{t(($) => $['newKnowledge.addSource'])}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<AddSourceExitDialog
|
||||
discarding={discarding}
|
||||
error={discardError}
|
||||
onCancel={cancelExit}
|
||||
onConfirm={() => void confirmExit()}
|
||||
open={exitOpen}
|
||||
/>
|
||||
</main>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
'use client'
|
||||
|
||||
import {
|
||||
AlertDialog,
|
||||
AlertDialogActions,
|
||||
AlertDialogCancelButton,
|
||||
AlertDialogConfirmButton,
|
||||
AlertDialogContent,
|
||||
AlertDialogDescription,
|
||||
AlertDialogTitle,
|
||||
} from '@langgenius/dify-ui/alert-dialog'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
export function AddSourceExitDialog({
|
||||
discarding,
|
||||
error,
|
||||
onCancel,
|
||||
onConfirm,
|
||||
open,
|
||||
}: {
|
||||
discarding: boolean
|
||||
error: boolean
|
||||
onCancel: () => void
|
||||
onConfirm: () => void
|
||||
open: boolean
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
const { t: tCommon } = useTranslation('common')
|
||||
|
||||
return (
|
||||
<AlertDialog
|
||||
open={open}
|
||||
onOpenChange={(nextOpen) => {
|
||||
if (!nextOpen && !discarding) onCancel()
|
||||
}}
|
||||
>
|
||||
<AlertDialogContent className="w-120 rounded-2xl border-[0.5px] border-components-panel-border bg-components-panel-bg p-0! text-left align-middle shadow-lg">
|
||||
<div className="flex flex-col items-start gap-2 self-stretch p-6 pb-4">
|
||||
<AlertDialogTitle className="title-2xl-semi-bold text-text-primary">
|
||||
{t(($) => $['newKnowledge.discardSourceDraftTitle'])}
|
||||
</AlertDialogTitle>
|
||||
<AlertDialogDescription
|
||||
render={<div />}
|
||||
className="system-md-regular text-text-secondary"
|
||||
>
|
||||
{t(($) => $['newKnowledge.discardSourceDraftDescription'])}
|
||||
</AlertDialogDescription>
|
||||
{error && (
|
||||
<p role="alert" className="system-sm-regular text-text-destructive">
|
||||
{t(($) => $['newKnowledge.discardSourceDraftFailed'])}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
<AlertDialogActions className="gap-2 p-6">
|
||||
<AlertDialogCancelButton variant="secondary" disabled={discarding}>
|
||||
{tCommon(($) => $['operation.cancel'])}
|
||||
</AlertDialogCancelButton>
|
||||
<AlertDialogConfirmButton loading={discarding} disabled={discarding} onClick={onConfirm}>
|
||||
{t(($) => $['newKnowledge.discardDraftConfirm'])}
|
||||
</AlertDialogConfirmButton>
|
||||
</AlertDialogActions>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,971 @@
|
||||
'use client'
|
||||
|
||||
import type {
|
||||
GetKnowledgeSpacesByIdSourceWorkflowsByRunIdPagesResponse,
|
||||
Source,
|
||||
SourceWorkflowRun,
|
||||
} from '@dify/contracts/knowledge-fs/types.gen'
|
||||
import { Button } from '@langgenius/dify-ui/button'
|
||||
import { Checkbox } from '@langgenius/dify-ui/checkbox'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { Field, FieldError, FieldLabel } from '@langgenius/dify-ui/field'
|
||||
import { Form } from '@langgenius/dify-ui/form'
|
||||
import { Input } from '@langgenius/dify-ui/input'
|
||||
import { NumberField, NumberFieldGroup, NumberFieldInput } from '@langgenius/dify-ui/number-field'
|
||||
import { memo, useCallback, useEffect, useId, useMemo, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { consoleClient } from '@/service/client'
|
||||
|
||||
type ConnectionReference = {
|
||||
id: string
|
||||
providerId: string
|
||||
}
|
||||
|
||||
type PreviewPage = GetKnowledgeSpacesByIdSourceWorkflowsByRunIdPagesResponse['items'][number]
|
||||
|
||||
type CrawlConfiguration = {
|
||||
includeSubpages: boolean
|
||||
limit: number
|
||||
name: string
|
||||
url: string
|
||||
}
|
||||
|
||||
type PreviewDraft = {
|
||||
clientRequestId: string
|
||||
configurationKey: string
|
||||
creationAttempted?: boolean
|
||||
previewRequestId: string
|
||||
reconciliationAttempts?: number
|
||||
source?: Source
|
||||
}
|
||||
|
||||
function previewPagesEqual(left: PreviewPage, right: PreviewPage) {
|
||||
return (
|
||||
left.pageId === right.pageId &&
|
||||
left.description === right.description &&
|
||||
left.etag === right.etag &&
|
||||
left.sourceUrl === right.sourceUrl &&
|
||||
left.title === right.title
|
||||
)
|
||||
}
|
||||
|
||||
const PAGE_SIZE = 200
|
||||
const MAX_CURSOR_PAGES = 100
|
||||
const POLL_INTERVAL_MS = 1500
|
||||
const DEFAULT_PAGE_LIMIT = 100
|
||||
const MAX_PAGE_LIMIT = 1000
|
||||
const MAX_SOURCE_NAME_LENGTH = 200
|
||||
const MAX_SOURCE_RECONCILIATION_ATTEMPTS = 3
|
||||
const SUCCESS_STATES = new Set(['complete', 'completed', 'preview_ready', 'success', 'succeeded'])
|
||||
const FAILURE_STATES = new Set(['error', 'exhausted', 'failed', 'timed_out', 'timeout'])
|
||||
const CANCELED_STATES = new Set(['canceled', 'cancelled', 'superseded'])
|
||||
|
||||
function normalizedState(state: string) {
|
||||
return state.trim().toLowerCase().replaceAll('-', '_').replaceAll(' ', '_')
|
||||
}
|
||||
|
||||
function isSuccessful(state: string) {
|
||||
return SUCCESS_STATES.has(normalizedState(state))
|
||||
}
|
||||
|
||||
function isFailed(state: string) {
|
||||
return FAILURE_STATES.has(normalizedState(state))
|
||||
}
|
||||
|
||||
function isCanceled(state: string) {
|
||||
return CANCELED_STATES.has(normalizedState(state))
|
||||
}
|
||||
|
||||
function isTerminal(state: string) {
|
||||
return isSuccessful(state) || isFailed(state) || isCanceled(state)
|
||||
}
|
||||
|
||||
function normalizeURL(value: string) {
|
||||
try {
|
||||
const url = new URL(value.trim())
|
||||
if (
|
||||
!['http:', 'https:'].includes(url.protocol) ||
|
||||
!url.hostname ||
|
||||
url.username ||
|
||||
url.password
|
||||
)
|
||||
return undefined
|
||||
url.hash = ''
|
||||
return url
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
function configurationKey(configuration: CrawlConfiguration) {
|
||||
return JSON.stringify(configuration)
|
||||
}
|
||||
|
||||
function workflowAttemptKey(run: SourceWorkflowRun) {
|
||||
return `${run.id}:${run.executionAttempts}`
|
||||
}
|
||||
|
||||
function responseStatus(error: unknown) {
|
||||
if (error instanceof Response) return error.status
|
||||
if (!error || typeof error !== 'object') return undefined
|
||||
if ('status' in error && typeof error.status === 'number') return error.status
|
||||
if ('data' in error && error.data && typeof error.data === 'object' && 'status' in error.data) {
|
||||
return typeof error.data.status === 'number' ? error.data.status : undefined
|
||||
}
|
||||
}
|
||||
|
||||
function isDefinitiveRequestFailure(error: unknown) {
|
||||
const status = responseStatus(error)
|
||||
return status !== undefined && [400, 401, 403, 404, 409, 422, 429].includes(status)
|
||||
}
|
||||
|
||||
function isRetryConfirmed(previous: SourceWorkflowRun, current: SourceWorkflowRun) {
|
||||
return (
|
||||
previous.id === current.id &&
|
||||
(current.executionAttempts > previous.executionAttempts ||
|
||||
(isTerminal(previous.state) && !isTerminal(current.state)))
|
||||
)
|
||||
}
|
||||
|
||||
function isCancelConfirmed(current: SourceWorkflowRun) {
|
||||
return isTerminal(current.state)
|
||||
}
|
||||
|
||||
function createRequestId() {
|
||||
return globalThis.crypto.randomUUID()
|
||||
}
|
||||
|
||||
async function listWorkflowPageUpdates(
|
||||
knowledgeSpaceId: string,
|
||||
runId: string,
|
||||
initialCursor?: string,
|
||||
) {
|
||||
const pages = new Map<string, PreviewPage>()
|
||||
const seenCursors = new Set(initialCursor ? [initialCursor] : [])
|
||||
let cursor = initialCursor
|
||||
let resumeCursor = initialCursor
|
||||
let pageCount = 0
|
||||
|
||||
do {
|
||||
pageCount += 1
|
||||
if (pageCount > MAX_CURSOR_PAGES) throw new Error('Workflow page cursor limit exceeded')
|
||||
const response =
|
||||
await consoleClient.knowledgeFs.getKnowledgeSpacesByIdSourceWorkflowsByRunIdPages({
|
||||
params: { id: knowledgeSpaceId, runId },
|
||||
query: { ...(cursor ? { cursor } : {}), limit: PAGE_SIZE },
|
||||
})
|
||||
for (const page of response.items) pages.set(page.pageId, page)
|
||||
const nextCursor = response.nextCursor
|
||||
if (!nextCursor || seenCursors.has(nextCursor)) break
|
||||
seenCursors.add(nextCursor)
|
||||
cursor = nextCursor
|
||||
resumeCursor = nextCursor
|
||||
} while (cursor)
|
||||
|
||||
return { items: [...pages.values()], resumeCursor }
|
||||
}
|
||||
|
||||
async function findProvisionalSource(knowledgeSpaceId: string, clientRequestId: string) {
|
||||
const seenCursors = new Set<string>()
|
||||
let cursor: string | undefined
|
||||
let pageCount = 0
|
||||
|
||||
do {
|
||||
pageCount += 1
|
||||
if (pageCount > MAX_CURSOR_PAGES) throw new Error('Source cursor limit exceeded')
|
||||
const response = await consoleClient.knowledgeFs.getKnowledgeSpacesByIdSources({
|
||||
params: { id: knowledgeSpaceId },
|
||||
query: { ...(cursor ? { cursor } : {}), limit: PAGE_SIZE },
|
||||
})
|
||||
const source = response.items.find(
|
||||
(candidate) => candidate.metadata.clientRequestId === clientRequestId,
|
||||
)
|
||||
if (source) return source
|
||||
const nextCursor = response.nextCursor
|
||||
if (!nextCursor || seenCursors.has(nextCursor)) return undefined
|
||||
seenCursors.add(nextCursor)
|
||||
cursor = nextCursor
|
||||
} while (cursor)
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
const CrawlPageList = memo(
|
||||
({ loading = false, pages }: { loading?: boolean; pages: PreviewPage[] }) => {
|
||||
if (!pages.length && !loading) return null
|
||||
|
||||
return (
|
||||
<ul className="max-h-64 divide-y divide-divider-subtle overflow-y-auto" aria-live="polite">
|
||||
{pages.map((page) => (
|
||||
<li
|
||||
key={page.pageId}
|
||||
className="flex items-start gap-2.5 px-4 py-2.5 [contain-intrinsic-size:auto_40px] [content-visibility:auto]"
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
disabled
|
||||
aria-label={page.title || page.sourceUrl}
|
||||
className="mt-0.5 size-4"
|
||||
/>
|
||||
<span className="min-w-0">
|
||||
<span className="block truncate system-xs-medium text-text-primary">
|
||||
{page.title || page.sourceUrl}
|
||||
</span>
|
||||
<span className="block truncate system-2xs-regular text-text-tertiary">
|
||||
{page.sourceUrl}
|
||||
</span>
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
{loading &&
|
||||
[0, 1].map((placeholder) => (
|
||||
<li
|
||||
key={`placeholder-${placeholder}`}
|
||||
data-testid="crawl-page-skeleton"
|
||||
aria-hidden
|
||||
className="flex items-start gap-2.5 px-4 py-2.5"
|
||||
>
|
||||
<span className="size-4 animate-pulse rounded bg-background-section motion-reduce:animate-none" />
|
||||
<span className="min-w-0 flex-1 space-y-1.5">
|
||||
<span className="block h-3 w-2/3 animate-pulse rounded bg-background-section motion-reduce:animate-none" />
|
||||
<span className="block h-2.5 w-full animate-pulse rounded bg-background-section motion-reduce:animate-none" />
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
function EmptyPreview() {
|
||||
const { t } = useTranslation('dataset')
|
||||
return (
|
||||
<div className="flex min-h-40 flex-col items-center justify-center rounded-xl border border-dashed border-divider-regular px-6 text-center">
|
||||
<span className="flex size-10 items-center justify-center rounded-lg bg-background-section">
|
||||
<span aria-hidden className="i-ri-global-line size-5 text-text-tertiary" />
|
||||
</span>
|
||||
<p className="mt-2 system-xs-semibold text-text-primary">
|
||||
{t(($) => $['newKnowledge.pagesAppearTitle'])}
|
||||
</p>
|
||||
<p className="mt-2 system-xs-regular text-text-tertiary">
|
||||
{t(($) => $['newKnowledge.pagesAppearDescription'])}
|
||||
</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export function WebsiteCrawlPreview({
|
||||
connection,
|
||||
knowledgeSpaceId,
|
||||
onDraftChange,
|
||||
onPendingOperation,
|
||||
onProvisionalSource,
|
||||
providerName,
|
||||
}: {
|
||||
connection: ConnectionReference
|
||||
knowledgeSpaceId: string
|
||||
onDraftChange?: (dirty: boolean) => void
|
||||
onPendingOperation?: (operation: Promise<void>) => void
|
||||
onProvisionalSource?: (source: Source) => void
|
||||
providerName: string
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
const rootUrlErrorId = useId()
|
||||
const [rootUrl, setRootUrl] = useState('')
|
||||
const [sourceName, setSourceName] = useState('')
|
||||
const [urlTouched, setUrlTouched] = useState(false)
|
||||
const [optionsExpanded, setOptionsExpanded] = useState(false)
|
||||
const [includeSubpages, setIncludeSubpages] = useState(true)
|
||||
const [pageLimit, setPageLimit] = useState(DEFAULT_PAGE_LIMIT)
|
||||
const [run, setRun] = useState<SourceWorkflowRun>()
|
||||
const [previewConfiguration, setPreviewConfiguration] = useState<CrawlConfiguration>()
|
||||
const [pages, setPages] = useState<PreviewPage[]>([])
|
||||
const [pagesLoaded, setPagesLoaded] = useState(false)
|
||||
const [starting, setStarting] = useState(false)
|
||||
const [stopping, setStopping] = useState(false)
|
||||
const [pollPaused, setPollPaused] = useState(false)
|
||||
const [requestError, setRequestError] = useState<string>()
|
||||
const draftRef = useRef<PreviewDraft | undefined>(undefined)
|
||||
const actionPendingRef = useRef(false)
|
||||
const retryFingerprintRef = useRef<string | undefined>(undefined)
|
||||
const cancelFingerprintRef = useRef<string | undefined>(undefined)
|
||||
const rootUrlInputRef = useRef<HTMLInputElement>(null)
|
||||
const sourceNameInputRef = useRef<HTMLInputElement>(null)
|
||||
const pageMapRef = useRef(new Map<string, PreviewPage>())
|
||||
const pageCursorRef = useRef<string | undefined>(undefined)
|
||||
|
||||
const resetPreviewPages = useCallback(() => {
|
||||
pageMapRef.current.clear()
|
||||
pageCursorRef.current = undefined
|
||||
setPages([])
|
||||
setPagesLoaded(false)
|
||||
}, [])
|
||||
|
||||
const normalizedURL = useMemo(() => normalizeURL(rootUrl), [rootUrl])
|
||||
const normalizedLimit = Math.min(Math.max(Math.trunc(pageLimit) || 1, 1), MAX_PAGE_LIMIT)
|
||||
const configuration = useMemo<CrawlConfiguration | undefined>(
|
||||
() =>
|
||||
normalizedURL && sourceName.trim() && sourceName.trim().length <= MAX_SOURCE_NAME_LENGTH
|
||||
? {
|
||||
includeSubpages,
|
||||
limit: normalizedLimit,
|
||||
name: sourceName.trim(),
|
||||
url: normalizedURL.toString(),
|
||||
}
|
||||
: undefined,
|
||||
[includeSubpages, normalizedLimit, normalizedURL, sourceName],
|
||||
)
|
||||
const active = Boolean(run && !isTerminal(run.state))
|
||||
const shouldPoll = Boolean(
|
||||
run && !starting && !stopping && !pollPaused && (active || !pagesLoaded),
|
||||
)
|
||||
const runId = run?.id
|
||||
const locked = starting || stopping || active
|
||||
const draftHost = normalizedURL?.host ?? ''
|
||||
const previewHost = previewConfiguration ? new URL(previewConfiguration.url).host : draftHost
|
||||
const completedCount = Math.max(run?.progressCompleted ?? 0, pages.length)
|
||||
const crawlingStatusText = t(($) => $['newKnowledge.crawlingPages'], {
|
||||
count: completedCount,
|
||||
host: previewHost,
|
||||
})
|
||||
const completedStatusText = t(($) => $['newKnowledge.pagesCrawled'], {
|
||||
count: pages.length,
|
||||
host: previewHost,
|
||||
})
|
||||
const hasDraftChanges = Boolean(
|
||||
rootUrl ||
|
||||
sourceName ||
|
||||
!includeSubpages ||
|
||||
pageLimit !== DEFAULT_PAGE_LIMIT ||
|
||||
run ||
|
||||
requestError,
|
||||
)
|
||||
|
||||
useEffect(() => {
|
||||
onDraftChange?.(hasDraftChanges)
|
||||
}, [hasDraftChanges, onDraftChange])
|
||||
|
||||
useEffect(
|
||||
() => () => {
|
||||
onDraftChange?.(false)
|
||||
},
|
||||
[onDraftChange],
|
||||
)
|
||||
|
||||
const ensureProvisionalSource = useCallback(
|
||||
async (nextConfiguration: CrawlConfiguration) => {
|
||||
const nextConfigurationKey = configurationKey(nextConfiguration)
|
||||
let draft = draftRef.current
|
||||
if (!draft || draft.configurationKey !== nextConfigurationKey) {
|
||||
draft = {
|
||||
clientRequestId: createRequestId(),
|
||||
configurationKey: nextConfigurationKey,
|
||||
previewRequestId: createRequestId(),
|
||||
}
|
||||
draftRef.current = draft
|
||||
}
|
||||
if (draft.source) return draft
|
||||
|
||||
if (draft.creationAttempted) {
|
||||
const reconciled = await findProvisionalSource(knowledgeSpaceId, draft.clientRequestId)
|
||||
if (reconciled) {
|
||||
draft.source = reconciled
|
||||
onProvisionalSource?.(reconciled)
|
||||
return draft
|
||||
}
|
||||
draft.reconciliationAttempts = (draft.reconciliationAttempts ?? 0) + 1
|
||||
if (draft.reconciliationAttempts < MAX_SOURCE_RECONCILIATION_ATTEMPTS)
|
||||
throw new Error('Provisional source creation is still reconciling')
|
||||
draft.creationAttempted = false
|
||||
draft.reconciliationAttempts = 0
|
||||
}
|
||||
|
||||
draft.creationAttempted = true
|
||||
try {
|
||||
draft.source = await consoleClient.knowledgeFs.postKnowledgeSpacesByIdSources({
|
||||
body: {
|
||||
connectionId: connection.id,
|
||||
metadata: {
|
||||
clientRequestId: draft.clientRequestId,
|
||||
crawlOptions: {
|
||||
includeSubpages: nextConfiguration.includeSubpages,
|
||||
limit: nextConfiguration.limit,
|
||||
},
|
||||
preview: true,
|
||||
providerId: connection.providerId,
|
||||
},
|
||||
name: nextConfiguration.name,
|
||||
status: 'disabled',
|
||||
type: 'web',
|
||||
uri: nextConfiguration.url,
|
||||
},
|
||||
params: { id: knowledgeSpaceId },
|
||||
})
|
||||
onProvisionalSource?.(draft.source)
|
||||
} catch (error) {
|
||||
if (isDefinitiveRequestFailure(error)) {
|
||||
draft.creationAttempted = false
|
||||
draft.reconciliationAttempts = 0
|
||||
throw error
|
||||
}
|
||||
const reconciled = await findProvisionalSource(knowledgeSpaceId, draft.clientRequestId)
|
||||
if (!reconciled) {
|
||||
draft.reconciliationAttempts = (draft.reconciliationAttempts ?? 0) + 1
|
||||
throw error
|
||||
}
|
||||
draft.source = reconciled
|
||||
onProvisionalSource?.(reconciled)
|
||||
}
|
||||
return draft
|
||||
},
|
||||
[connection.id, connection.providerId, knowledgeSpaceId, onProvisionalSource],
|
||||
)
|
||||
|
||||
const startPreview = useCallback(
|
||||
async (nextConfiguration: CrawlConfiguration, forceNewRun = false) => {
|
||||
if (actionPendingRef.current) return
|
||||
actionPendingRef.current = true
|
||||
setStarting(true)
|
||||
setRequestError(undefined)
|
||||
setPollPaused(false)
|
||||
setPreviewConfiguration(nextConfiguration)
|
||||
resetPreviewPages()
|
||||
setRun(undefined)
|
||||
try {
|
||||
const draft = await ensureProvisionalSource(nextConfiguration)
|
||||
if (!draft.source) throw new Error('Provisional source is missing')
|
||||
if (forceNewRun) draft.previewRequestId = createRequestId()
|
||||
const nextRun =
|
||||
await consoleClient.knowledgeFs.postKnowledgeSpacesByIdSourcesBySourceIdCrawlPreview({
|
||||
headers: { 'Idempotency-Key': draft.previewRequestId },
|
||||
params: { id: knowledgeSpaceId, sourceId: draft.source.id },
|
||||
})
|
||||
setRun(nextRun)
|
||||
} catch {
|
||||
setRequestError('START_FAILED')
|
||||
} finally {
|
||||
actionPendingRef.current = false
|
||||
setStarting(false)
|
||||
}
|
||||
},
|
||||
[ensureProvisionalSource, knowledgeSpaceId, resetPreviewPages],
|
||||
)
|
||||
|
||||
const retryRun = useCallback(async () => {
|
||||
if (!run || actionPendingRef.current) return
|
||||
const attemptKey = workflowAttemptKey(run)
|
||||
const retryAlreadySent = retryFingerprintRef.current === attemptKey
|
||||
actionPendingRef.current = true
|
||||
setStarting(true)
|
||||
setRequestError(undefined)
|
||||
setPollPaused(false)
|
||||
try {
|
||||
if (retryAlreadySent) {
|
||||
try {
|
||||
const reconciled =
|
||||
await consoleClient.knowledgeFs.getKnowledgeSpacesByIdSourceWorkflowsByRunId({
|
||||
params: { id: knowledgeSpaceId, runId: run.id },
|
||||
})
|
||||
if (!isRetryConfirmed(run, reconciled)) {
|
||||
setRequestError('RETRY_FAILED')
|
||||
return
|
||||
}
|
||||
resetPreviewPages()
|
||||
setRun(reconciled)
|
||||
} catch {
|
||||
setRequestError('RETRY_FAILED')
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
retryFingerprintRef.current = attemptKey
|
||||
try {
|
||||
const retried =
|
||||
await consoleClient.knowledgeFs.postKnowledgeSpacesByIdSourceWorkflowsByRunIdRetry({
|
||||
params: { id: knowledgeSpaceId, runId: run.id },
|
||||
})
|
||||
resetPreviewPages()
|
||||
setRun(retried)
|
||||
} catch (error) {
|
||||
if (isDefinitiveRequestFailure(error)) {
|
||||
retryFingerprintRef.current = undefined
|
||||
setRequestError('RETRY_FAILED')
|
||||
return
|
||||
}
|
||||
try {
|
||||
const reconciled =
|
||||
await consoleClient.knowledgeFs.getKnowledgeSpacesByIdSourceWorkflowsByRunId({
|
||||
params: { id: knowledgeSpaceId, runId: run.id },
|
||||
})
|
||||
if (!isRetryConfirmed(run, reconciled)) {
|
||||
setRequestError('RETRY_FAILED')
|
||||
} else {
|
||||
resetPreviewPages()
|
||||
setRun(reconciled)
|
||||
}
|
||||
} catch {
|
||||
setRequestError('RETRY_FAILED')
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
actionPendingRef.current = false
|
||||
setStarting(false)
|
||||
}
|
||||
}, [knowledgeSpaceId, resetPreviewPages, run])
|
||||
|
||||
useEffect(() => {
|
||||
if (!runId || !shouldPoll) return
|
||||
let disposed = false
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
|
||||
const poll = async () => {
|
||||
try {
|
||||
const nextRun =
|
||||
await consoleClient.knowledgeFs.getKnowledgeSpacesByIdSourceWorkflowsByRunId({
|
||||
params: { id: knowledgeSpaceId, runId },
|
||||
})
|
||||
if (disposed) return
|
||||
let pageUpdates: Awaited<ReturnType<typeof listWorkflowPageUpdates>>
|
||||
const finalSnapshot = isSuccessful(nextRun.state)
|
||||
try {
|
||||
pageUpdates = await listWorkflowPageUpdates(
|
||||
knowledgeSpaceId,
|
||||
runId,
|
||||
finalSnapshot ? undefined : pageCursorRef.current,
|
||||
)
|
||||
} catch (error) {
|
||||
if (isFailed(nextRun.state) || isCanceled(nextRun.state)) {
|
||||
setRun(nextRun)
|
||||
setPagesLoaded(true)
|
||||
setRequestError(undefined)
|
||||
return
|
||||
}
|
||||
throw error
|
||||
}
|
||||
if (disposed) return
|
||||
let pagesChanged = false
|
||||
if (finalSnapshot) {
|
||||
const currentPages = [...pageMapRef.current.values()]
|
||||
const finalPageMap = new Map(pageUpdates.items.map((page) => [page.pageId, page]))
|
||||
pagesChanged =
|
||||
currentPages.length !== pageUpdates.items.length ||
|
||||
pageUpdates.items.some(
|
||||
(page, index) =>
|
||||
!currentPages[index] || !previewPagesEqual(currentPages[index], page),
|
||||
)
|
||||
pageMapRef.current = finalPageMap
|
||||
} else {
|
||||
for (const page of pageUpdates.items) {
|
||||
const current = pageMapRef.current.get(page.pageId)
|
||||
if (!current || !previewPagesEqual(current, page)) {
|
||||
pageMapRef.current.set(page.pageId, page)
|
||||
pagesChanged = true
|
||||
}
|
||||
}
|
||||
}
|
||||
pageCursorRef.current = pageUpdates.resumeCursor
|
||||
setRun(nextRun)
|
||||
if (pagesChanged) setPages([...pageMapRef.current.values()])
|
||||
setPagesLoaded(true)
|
||||
setRequestError(undefined)
|
||||
if (!isTerminal(nextRun.state)) timer = setTimeout(() => void poll(), POLL_INTERVAL_MS)
|
||||
} catch {
|
||||
if (disposed) return
|
||||
setRequestError('POLL_FAILED')
|
||||
setPollPaused(true)
|
||||
}
|
||||
}
|
||||
|
||||
void poll()
|
||||
return () => {
|
||||
disposed = true
|
||||
if (timer) clearTimeout(timer)
|
||||
}
|
||||
}, [knowledgeSpaceId, runId, shouldPoll])
|
||||
|
||||
const stop = async () => {
|
||||
if (!run || actionPendingRef.current || !active) return
|
||||
const attemptKey = workflowAttemptKey(run)
|
||||
const cancelAlreadySent = cancelFingerprintRef.current === attemptKey
|
||||
actionPendingRef.current = true
|
||||
setStopping(true)
|
||||
setRequestError(undefined)
|
||||
try {
|
||||
if (cancelAlreadySent) {
|
||||
try {
|
||||
const reconciled =
|
||||
await consoleClient.knowledgeFs.getKnowledgeSpacesByIdSourceWorkflowsByRunId({
|
||||
params: { id: knowledgeSpaceId, runId: run.id },
|
||||
})
|
||||
if (!isCancelConfirmed(reconciled)) {
|
||||
setRequestError('CANCEL_FAILED')
|
||||
return
|
||||
}
|
||||
cancelFingerprintRef.current = undefined
|
||||
setRun(reconciled)
|
||||
} catch {
|
||||
setRequestError('CANCEL_FAILED')
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
cancelFingerprintRef.current = attemptKey
|
||||
try {
|
||||
const canceled =
|
||||
await consoleClient.knowledgeFs.postKnowledgeSpacesByIdSourceWorkflowsByRunIdCancel({
|
||||
body: { reason: 'user_requested' },
|
||||
params: { id: knowledgeSpaceId, runId: run.id },
|
||||
})
|
||||
cancelFingerprintRef.current = undefined
|
||||
setRun(canceled)
|
||||
} catch (error) {
|
||||
if (isDefinitiveRequestFailure(error)) {
|
||||
cancelFingerprintRef.current = undefined
|
||||
setRequestError('CANCEL_FAILED')
|
||||
return
|
||||
}
|
||||
try {
|
||||
const reconciled =
|
||||
await consoleClient.knowledgeFs.getKnowledgeSpacesByIdSourceWorkflowsByRunId({
|
||||
params: { id: knowledgeSpaceId, runId: run.id },
|
||||
})
|
||||
if (!isCancelConfirmed(reconciled)) {
|
||||
setRequestError('CANCEL_FAILED')
|
||||
} else {
|
||||
cancelFingerprintRef.current = undefined
|
||||
setRun(reconciled)
|
||||
}
|
||||
} catch {
|
||||
setRequestError('CANCEL_FAILED')
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
actionPendingRef.current = false
|
||||
setStopping(false)
|
||||
}
|
||||
}
|
||||
|
||||
const dispatchOperation = (operation: Promise<void>) => {
|
||||
onPendingOperation?.(operation)
|
||||
void operation
|
||||
}
|
||||
|
||||
const handlePrimaryAction = () => {
|
||||
if (!configuration) {
|
||||
setUrlTouched(true)
|
||||
if (!normalizedURL) rootUrlInputRef.current?.focus()
|
||||
else sourceNameInputRef.current?.focus()
|
||||
return
|
||||
}
|
||||
if (requestError === 'POLL_FAILED' && run) {
|
||||
setRequestError(undefined)
|
||||
setPollPaused(false)
|
||||
return
|
||||
}
|
||||
if (run && isSuccessful(run.state)) {
|
||||
const currentKey = configurationKey(configuration)
|
||||
dispatchOperation(
|
||||
startPreview(configuration, draftRef.current?.configurationKey === currentKey),
|
||||
)
|
||||
return
|
||||
}
|
||||
if (run && (isFailed(run.state) || isCanceled(run.state))) {
|
||||
const currentKey = configurationKey(configuration)
|
||||
if (draftRef.current?.configurationKey === currentKey) {
|
||||
dispatchOperation(retryRun())
|
||||
return
|
||||
}
|
||||
}
|
||||
dispatchOperation(startPreview(configuration))
|
||||
}
|
||||
|
||||
const handleSubmit = () => handlePrimaryAction()
|
||||
|
||||
const primaryLabel =
|
||||
starting || (active && !pollPaused)
|
||||
? t(($) => $['newKnowledge.crawling'])
|
||||
: (requestError && requestError !== 'CANCEL_FAILED') ||
|
||||
(run && (isFailed(run.state) || isCanceled(run.state)))
|
||||
? t(($) => $['newKnowledge.retryCrawl'])
|
||||
: run && isSuccessful(run.state) && pagesLoaded && pages.length === 0
|
||||
? t(($) => $['newKnowledge.adjustAndRecrawl'])
|
||||
: t(($) => $['newKnowledge.crawlAndPreview'])
|
||||
|
||||
const showFailure = Boolean(
|
||||
requestError === 'START_FAILED' ||
|
||||
requestError === 'RETRY_FAILED' ||
|
||||
requestError === 'POLL_FAILED' ||
|
||||
(run && isFailed(run.state)),
|
||||
)
|
||||
const showZero = Boolean(run && isSuccessful(run.state) && pagesLoaded && pages.length === 0)
|
||||
const showSuccess = Boolean(run && isSuccessful(run.state) && pages.length > 0)
|
||||
const showCanceled = Boolean(run && isCanceled(run.state))
|
||||
const errorCode = run?.lastErrorCode ?? requestError
|
||||
const is403 = errorCode?.includes('403')
|
||||
const isTimeout =
|
||||
errorCode?.toUpperCase().includes('TIMEOUT') ||
|
||||
(run ? ['timed_out', 'timeout'].includes(normalizedState(run.state)) : false)
|
||||
const isProviderError = errorCode?.toUpperCase().includes('PROVIDER')
|
||||
|
||||
return (
|
||||
<section aria-label={t(($) => $['newKnowledge.crawlAndPreview'])}>
|
||||
<p role="status" className="sr-only">
|
||||
{t(($) => $['newKnowledge.providerConnected'], { provider: providerName })}
|
||||
</p>
|
||||
<Form onFormSubmit={handleSubmit}>
|
||||
<div className={cn('mt-4 space-y-4', locked && 'opacity-70')}>
|
||||
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
|
||||
<Field
|
||||
name="root-url"
|
||||
className="gap-1.5"
|
||||
disabled={locked}
|
||||
invalid={urlTouched && !normalizedURL}
|
||||
touched={urlTouched}
|
||||
>
|
||||
<FieldLabel>
|
||||
{t(($) => $['newKnowledge.rootUrl'])}
|
||||
<span aria-hidden className="ml-0.5 text-text-destructive">
|
||||
*
|
||||
</span>
|
||||
</FieldLabel>
|
||||
<Input
|
||||
ref={rootUrlInputRef}
|
||||
name="root-url"
|
||||
type="url"
|
||||
required
|
||||
value={rootUrl}
|
||||
autoComplete="off"
|
||||
placeholder={t(($) => $['newKnowledge.rootUrlPlaceholder'])}
|
||||
onBlur={() => setUrlTouched(true)}
|
||||
onValueChange={setRootUrl}
|
||||
/>
|
||||
<FieldError id={rootUrlErrorId} match={urlTouched && !normalizedURL}>
|
||||
{t(($) => $['newKnowledge.invalidRootUrl'])}
|
||||
</FieldError>
|
||||
</Field>
|
||||
<Field name="source-name" className="gap-1.5" disabled={locked}>
|
||||
<FieldLabel>
|
||||
{t(($) => $['newKnowledge.sourceName'])}
|
||||
<span aria-hidden className="ml-0.5 text-text-destructive">
|
||||
*
|
||||
</span>
|
||||
</FieldLabel>
|
||||
<Input
|
||||
ref={sourceNameInputRef}
|
||||
name="source-name"
|
||||
type="text"
|
||||
required
|
||||
maxLength={MAX_SOURCE_NAME_LENGTH}
|
||||
value={sourceName}
|
||||
autoComplete="off"
|
||||
placeholder={t(($) => $['newKnowledge.sourceNamePlaceholder'])}
|
||||
onValueChange={setSourceName}
|
||||
/>
|
||||
<FieldError match="valueMissing">
|
||||
{t(($) => $['newKnowledge.sourceNameRequired'])}
|
||||
</FieldError>
|
||||
</Field>
|
||||
</div>
|
||||
<div className="overflow-hidden rounded-lg border border-components-option-card-option-border bg-background-default">
|
||||
<button
|
||||
type="button"
|
||||
aria-expanded={optionsExpanded}
|
||||
disabled={locked}
|
||||
className="flex h-9 w-full items-center gap-2 px-3 text-left outline-hidden focus-visible:ring-2 focus-visible:ring-state-accent-solid focus-visible:ring-inset"
|
||||
onClick={() => setOptionsExpanded((expanded) => !expanded)}
|
||||
>
|
||||
<span
|
||||
aria-hidden
|
||||
className={cn(
|
||||
'i-ri-arrow-right-s-line size-4 text-text-tertiary transition-transform',
|
||||
optionsExpanded && 'rotate-90',
|
||||
)}
|
||||
/>
|
||||
<span className="system-xs-medium text-text-primary">
|
||||
{t(($) => $['newKnowledge.crawlOptions'])}
|
||||
</span>
|
||||
{!optionsExpanded && includeSubpages && pageLimit === DEFAULT_PAGE_LIMIT && (
|
||||
<span className="ml-auto system-xs-regular text-text-tertiary">
|
||||
{t(($) => $['newKnowledge.usingDefaults'])}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
{optionsExpanded && (
|
||||
<div className="grid grid-cols-1 gap-3 border-t border-divider-subtle p-3 sm:grid-cols-2">
|
||||
<Field name="include-subpages" className="block" disabled={locked}>
|
||||
<FieldLabel className="flex h-9 items-center gap-2 system-xs-regular text-text-secondary">
|
||||
<Checkbox
|
||||
name="include-subpages"
|
||||
checked={includeSubpages}
|
||||
onCheckedChange={setIncludeSubpages}
|
||||
/>
|
||||
{t(($) => $['newKnowledge.includeSubpages'])}
|
||||
</FieldLabel>
|
||||
</Field>
|
||||
<Field
|
||||
name="page-limit"
|
||||
className="grid grid-cols-[1fr_6rem] items-center gap-2"
|
||||
disabled={locked}
|
||||
>
|
||||
<FieldLabel className="system-xs-regular text-text-secondary">
|
||||
{t(($) => $['newKnowledge.maxPages'])}
|
||||
</FieldLabel>
|
||||
<NumberField
|
||||
disabled={locked}
|
||||
min={1}
|
||||
max={MAX_PAGE_LIMIT}
|
||||
value={pageLimit}
|
||||
onValueChange={(value) => setPageLimit(value ?? 1)}
|
||||
>
|
||||
<NumberFieldGroup>
|
||||
<NumberFieldInput name="page-limit" autoComplete="off" />
|
||||
</NumberFieldGroup>
|
||||
</NumberField>
|
||||
</Field>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!showSuccess && (
|
||||
<Button
|
||||
type="submit"
|
||||
variant="primary"
|
||||
className="mt-4 w-full"
|
||||
disabled={locked && requestError !== 'POLL_FAILED'}
|
||||
loading={starting}
|
||||
>
|
||||
{primaryLabel}
|
||||
</Button>
|
||||
)}
|
||||
</Form>
|
||||
|
||||
<div className="mt-4">
|
||||
{!run && !requestError && <EmptyPreview />}
|
||||
{run && active && !pollPaused && (
|
||||
<div className="overflow-hidden rounded-xl border border-divider-regular">
|
||||
<div className="flex flex-wrap items-center gap-2 px-4 py-3">
|
||||
<span
|
||||
aria-hidden
|
||||
className="i-ri-loader-4-line size-4 animate-spin text-text-accent motion-reduce:animate-none"
|
||||
/>
|
||||
<span
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
className="min-w-0 flex-1 truncate system-xs-medium text-text-primary"
|
||||
title={crawlingStatusText}
|
||||
>
|
||||
{crawlingStatusText}
|
||||
</span>
|
||||
<Button
|
||||
type="button"
|
||||
variant="tertiary"
|
||||
size="small"
|
||||
className="ml-auto shrink-0"
|
||||
disabled={stopping}
|
||||
onClick={() => dispatchOperation(stop())}
|
||||
>
|
||||
{stopping
|
||||
? t(($) => $['newKnowledge.stoppingCrawl'])
|
||||
: t(($) => $['newKnowledge.stopCrawl'])}
|
||||
</Button>
|
||||
</div>
|
||||
{requestError === 'CANCEL_FAILED' && (
|
||||
<p role="alert" className="px-4 pb-3 system-xs-regular text-text-destructive">
|
||||
{t(($) => $['newKnowledge.crawlFailedDescription'])}
|
||||
</p>
|
||||
)}
|
||||
{run.progressTotal !== undefined && run.progressTotal > 0 && (
|
||||
<progress
|
||||
max={run.progressTotal}
|
||||
value={Math.min(completedCount, run.progressTotal)}
|
||||
aria-label={t(($) => $['newKnowledge.crawlProgress'], { host: previewHost })}
|
||||
className="block h-1 w-full accent-state-accent-solid"
|
||||
/>
|
||||
)}
|
||||
<CrawlPageList pages={pages} loading />
|
||||
</div>
|
||||
)}
|
||||
{showSuccess && (
|
||||
<div className="overflow-hidden rounded-xl border border-divider-regular">
|
||||
<div className="flex flex-wrap items-center gap-2 px-4 py-3">
|
||||
<p
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
className="min-w-0 flex-1 truncate system-xs-semibold text-text-primary"
|
||||
title={completedStatusText}
|
||||
>
|
||||
{completedStatusText}
|
||||
</p>
|
||||
<Button
|
||||
type="button"
|
||||
variant="tertiary"
|
||||
size="small"
|
||||
className="ml-auto shrink-0"
|
||||
disabled={starting || !configuration}
|
||||
loading={starting}
|
||||
onClick={handlePrimaryAction}
|
||||
>
|
||||
{t(($) => $['newKnowledge.reCrawl'])}
|
||||
</Button>
|
||||
</div>
|
||||
<CrawlPageList pages={pages} />
|
||||
</div>
|
||||
)}
|
||||
{showCanceled && (
|
||||
<div className="overflow-hidden rounded-xl border border-divider-regular">
|
||||
<p
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
className="px-4 py-3 system-xs-semibold text-text-primary"
|
||||
>
|
||||
{t(($) => $['newKnowledge.crawlStopped'])}
|
||||
</p>
|
||||
<CrawlPageList pages={pages} />
|
||||
</div>
|
||||
)}
|
||||
{showFailure && (
|
||||
<div
|
||||
role="alert"
|
||||
className="flex min-h-36 flex-col items-center justify-center rounded-xl border border-divider-regular px-6 text-center"
|
||||
>
|
||||
<span aria-hidden className="i-ri-error-warning-fill size-6 text-text-destructive" />
|
||||
<p className="mt-2 system-sm-semibold text-text-primary">
|
||||
{t(($) => $['newKnowledge.crawlFailed'], { host: previewHost })}
|
||||
</p>
|
||||
<p className="mt-1 max-w-lg system-xs-regular text-text-tertiary">
|
||||
{is403
|
||||
? t(($) => $['newKnowledge.crawlFailed403'])
|
||||
: isTimeout
|
||||
? t(($) => $['newKnowledge.crawlFailedTimeout'])
|
||||
: isProviderError
|
||||
? t(($) => $['newKnowledge.crawlFailedProvider'], {
|
||||
provider: providerName,
|
||||
})
|
||||
: requestError === 'START_FAILED'
|
||||
? t(($) => $['newKnowledge.crawlStartFailed'])
|
||||
: t(($) => $['newKnowledge.crawlFailedDescription'])}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
{showZero && !showFailure && (
|
||||
<div
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
className="flex min-h-40 flex-col items-center justify-center rounded-xl border border-divider-regular px-6 text-center"
|
||||
>
|
||||
<span className="flex size-10 items-center justify-center rounded-lg bg-background-section">
|
||||
<span aria-hidden className="i-ri-global-line size-5 text-text-tertiary" />
|
||||
</span>
|
||||
<p className="mt-2 system-xs-semibold text-text-primary">
|
||||
{t(($) => $['newKnowledge.noPagesFound'], { host: previewHost })}
|
||||
</p>
|
||||
<p className="mt-2 max-w-lg system-xs-regular text-text-tertiary">
|
||||
{t(($) => $['newKnowledge.noPagesFoundDescription'])}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user