mirror of
https://github.com/langgenius/dify.git
synced 2026-09-24 23:22:26 +08:00
feat(dataset): add crawl source selection (#39325)
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
Stephen Zhou
co-authored by
autofix-ci[bot]
autofix-ci[bot]
parent
e4dc29f98e
commit
eac9f69ad1
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,723 @@
|
||||
import type {
|
||||
GetKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyResponse,
|
||||
GetKnowledgeSpacesByIdSourceWorkflowsByRunIdPagesResponse,
|
||||
Source,
|
||||
SourceWorkflowRun,
|
||||
} from '@dify/contracts/knowledge-fs/types.gen'
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
|
||||
import { act, screen, waitFor } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { createInstance } from 'i18next'
|
||||
import { getInitOptions } from '@/i18n-config/settings'
|
||||
import datasetTranslations from '@/i18n/en-US/dataset.json'
|
||||
import { render } from '@/test/console/render'
|
||||
import { CrawlSelectionForm } from '../crawl-selection-form'
|
||||
|
||||
const clientMock = vi.hoisted(() => ({
|
||||
getPolicy: vi.fn(),
|
||||
getWorkflow: vi.fn(),
|
||||
selectPages: vi.fn(),
|
||||
updatePolicy: vi.fn(),
|
||||
}))
|
||||
|
||||
const routerMock = vi.hoisted(() => ({ push: vi.fn() }))
|
||||
const queryClientMock = vi.hoisted(() => ({ invalidateQueries: vi.fn() }))
|
||||
const policyQueryOptionsMock = vi.hoisted(() =>
|
||||
vi.fn(({ input }) => ({
|
||||
queryFn: () => clientMock.getPolicy(input),
|
||||
queryKey: ['sync-policy', input.params.sourceId],
|
||||
retry: false,
|
||||
})),
|
||||
)
|
||||
|
||||
vi.mock('@/next/navigation', () => ({ useRouter: () => routerMock }))
|
||||
|
||||
vi.mock('@tanstack/react-query', async (importOriginal) => {
|
||||
const original = await importOriginal<typeof import('@tanstack/react-query')>()
|
||||
return { ...original, useQueryClient: () => queryClientMock }
|
||||
})
|
||||
|
||||
vi.mock('@/service/client', () => ({
|
||||
consoleClient: {
|
||||
knowledgeFs: {
|
||||
getKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy: clientMock.getPolicy,
|
||||
getKnowledgeSpacesByIdSourceWorkflowsByRunId: clientMock.getWorkflow,
|
||||
},
|
||||
},
|
||||
consoleQuery: {
|
||||
knowledgeFs: {
|
||||
getKnowledgeSpacesByIdSources: {
|
||||
key: vi.fn(() => ['knowledge-sources']),
|
||||
},
|
||||
getKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy: {
|
||||
queryOptions: policyQueryOptionsMock,
|
||||
},
|
||||
postKnowledgeSpacesByIdSourceWorkflowsByRunIdSelection: {
|
||||
mutationOptions: vi.fn(() => ({
|
||||
mutationFn: (input: unknown) => clientMock.selectPages(input),
|
||||
})),
|
||||
},
|
||||
putKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy: {
|
||||
mutationOptions: vi.fn(() => ({
|
||||
mutationFn: (input: unknown) => clientMock.updatePolicy(input),
|
||||
})),
|
||||
},
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
type PreviewPage = GetKnowledgeSpacesByIdSourceWorkflowsByRunIdPagesResponse['items'][number]
|
||||
|
||||
const source: Source = {
|
||||
connectionId: 'connection-1',
|
||||
createdAt: '2026-07-20T10:00:00Z',
|
||||
id: 'source-1',
|
||||
knowledgeSpaceId: 'space-1',
|
||||
metadata: {},
|
||||
name: 'Dify docs',
|
||||
status: 'disabled',
|
||||
type: 'web',
|
||||
updatedAt: '2026-07-20T10:00:00Z',
|
||||
uri: 'https://docs.dify.ai/',
|
||||
version: 3,
|
||||
}
|
||||
|
||||
const run: SourceWorkflowRun = {
|
||||
checkpoint: 'crawl',
|
||||
createdAt: '2026-07-20T10:00:00Z',
|
||||
executionAttempts: 1,
|
||||
id: 'run-1',
|
||||
knowledgeSpaceId: 'space-1',
|
||||
kind: 'crawl-preview',
|
||||
maxExecutionAttempts: 3,
|
||||
progressCompleted: 3,
|
||||
progressFailed: 1,
|
||||
progressSkipped: 0,
|
||||
progressTotal: 4,
|
||||
sourceId: source.id,
|
||||
state: 'succeeded',
|
||||
updatedAt: '2026-07-20T10:01:00Z',
|
||||
}
|
||||
|
||||
const pages: PreviewPage[] = [
|
||||
{
|
||||
pageId: 'page-1',
|
||||
sourceUrl: 'https://docs.dify.ai/getting-started',
|
||||
title: 'Getting started',
|
||||
},
|
||||
{
|
||||
pageId: 'page-2',
|
||||
sourceUrl: 'https://docs.dify.ai/guides',
|
||||
title: 'Guides',
|
||||
},
|
||||
{
|
||||
pageId: 'off-domain',
|
||||
sourceUrl: 'https://github.com/langgenius/dify/edit/main/docs.md',
|
||||
title: 'Edit this page',
|
||||
},
|
||||
]
|
||||
|
||||
const policy = (
|
||||
overrides: Partial<GetKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyResponse> = {},
|
||||
): GetKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyResponse => ({
|
||||
createdAt: '2026-07-20T10:00:00Z',
|
||||
enabled: true,
|
||||
expectedSourceVersion: 3,
|
||||
id: 'policy-1',
|
||||
knowledgeSpaceId: 'space-1',
|
||||
mode: 'interval',
|
||||
revision: 2,
|
||||
sourceId: source.id,
|
||||
updatedAt: '2026-07-20T10:00:00Z',
|
||||
...overrides,
|
||||
})
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void
|
||||
const promise = new Promise<T>((nextResolve) => {
|
||||
resolve = nextResolve
|
||||
})
|
||||
return { promise, resolve }
|
||||
}
|
||||
|
||||
function renderSelectionForm(
|
||||
onCancel = vi.fn(),
|
||||
workflowUncertain = false,
|
||||
discardRequested = () => false,
|
||||
previewPages = pages,
|
||||
initialSyncMode?: 'custom' | 'interval' | 'manual' | 'provider',
|
||||
) {
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: { mutations: { retry: false }, queries: { retry: false } },
|
||||
})
|
||||
const onSubmissionUncertainChange = vi.fn()
|
||||
const onRecrawl = vi.fn()
|
||||
const onWorkflowPending = vi.fn()
|
||||
const onWorkflowRun = vi.fn()
|
||||
const view = render(
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<CrawlSelectionForm
|
||||
discardRequested={discardRequested}
|
||||
initialSyncMode={initialSyncMode}
|
||||
knowledgeSpaceId="space-1"
|
||||
onCancel={onCancel}
|
||||
onRecrawl={onRecrawl}
|
||||
onSubmissionUncertainChange={onSubmissionUncertainChange}
|
||||
onSubmitted={vi.fn()}
|
||||
onWorkflowPending={onWorkflowPending}
|
||||
onWorkflowRun={onWorkflowRun}
|
||||
pages={previewPages}
|
||||
rootUrl="https://docs.dify.ai/"
|
||||
run={run}
|
||||
source={source}
|
||||
workflowUncertain={workflowUncertain}
|
||||
/>
|
||||
</QueryClientProvider>,
|
||||
)
|
||||
return {
|
||||
...view,
|
||||
onRecrawl,
|
||||
onSubmissionUncertainChange,
|
||||
onWorkflowPending,
|
||||
onWorkflowRun,
|
||||
}
|
||||
}
|
||||
|
||||
describe('CrawlSelectionForm', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
clientMock.getPolicy.mockResolvedValue(policy())
|
||||
clientMock.getWorkflow.mockResolvedValue({ ...run, checkpoint: 'complete', state: 'completed' })
|
||||
clientMock.updatePolicy.mockResolvedValue(policy({ mode: 'manual', revision: 3 }))
|
||||
clientMock.selectPages.mockResolvedValue({ ...run, checkpoint: 'import', state: 'queued' })
|
||||
})
|
||||
|
||||
it('uses singular and plural copy for the crawl summary', async () => {
|
||||
const instance = createInstance()
|
||||
await instance.init({
|
||||
...getInitOptions(),
|
||||
defaultNS: 'dataset',
|
||||
lng: 'en-US',
|
||||
ns: ['dataset'],
|
||||
resources: { 'en-US': { dataset: datasetTranslations } },
|
||||
})
|
||||
const tDataset = instance.getFixedT('en-US', 'dataset')
|
||||
expect(tDataset(($) => $['newKnowledge.pagesCrawled'], { count: 1, host: 'example.com' })).toBe(
|
||||
'1 page crawled at example.com',
|
||||
)
|
||||
expect(tDataset(($) => $['newKnowledge.pagesCrawled'], { count: 2, host: 'example.com' })).toBe(
|
||||
'2 pages crawled at example.com',
|
||||
)
|
||||
})
|
||||
|
||||
it('loads a missing initial sync policy without a global error notification', async () => {
|
||||
renderSelectionForm()
|
||||
await screen.findByRole('checkbox', { name: 'Getting started' })
|
||||
|
||||
expect(policyQueryOptionsMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ context: { silent: true } }),
|
||||
)
|
||||
})
|
||||
|
||||
it('preserves the sync policy selected in the creation flow', async () => {
|
||||
renderSelectionForm(vi.fn(), false, () => false, pages, 'manual')
|
||||
|
||||
expect(
|
||||
await screen.findByRole('combobox', { name: 'dataset.newKnowledge.syncPolicy' }),
|
||||
).toHaveValue('manual')
|
||||
})
|
||||
|
||||
it('selects only valid same-domain pages and exposes an indeterminate select-all state', async () => {
|
||||
const user = userEvent.setup()
|
||||
renderSelectionForm()
|
||||
|
||||
await screen.findByRole('checkbox', { name: 'Getting started' })
|
||||
const addSource = screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' })
|
||||
expect(addSource).toBeDisabled()
|
||||
const offDomainPage = screen.getByRole('checkbox', { name: 'Edit this page' })
|
||||
expect(offDomainPage).toHaveAttribute('aria-disabled', 'true')
|
||||
expect(offDomainPage).toHaveAccessibleDescription(
|
||||
'https://github.com/langgenius/dify/edit/main/docs.md dataset.newKnowledge.skippedOffDomain',
|
||||
)
|
||||
expect(screen.getByText('dataset.newKnowledge.skippedOffDomain')).toBeInTheDocument()
|
||||
expect(screen.getByText(/dataset\.newKnowledge\.skippedFailed/)).toBeInTheDocument()
|
||||
|
||||
await user.click(screen.getByRole('checkbox', { name: 'Getting started' }))
|
||||
expect(addSource).toBeEnabled()
|
||||
expect(
|
||||
screen.getByRole('checkbox', { name: 'dataset.newKnowledge.selectAll' }),
|
||||
).toHaveAttribute('aria-checked', 'mixed')
|
||||
expect(
|
||||
screen.getByRole('checkbox', { name: 'dataset.newKnowledge.selectAll' }),
|
||||
).toHaveAttribute('data-indeterminate')
|
||||
expect(screen.getByText('dataset.newKnowledge.pagesSelected:{"count":1}')).toBeInTheDocument()
|
||||
|
||||
await user.click(screen.getByRole('checkbox', { name: 'dataset.newKnowledge.selectAll' }))
|
||||
expect(screen.getByRole('checkbox', { name: 'Getting started' })).toBeChecked()
|
||||
expect(screen.getByRole('checkbox', { name: 'Guides' })).toBeChecked()
|
||||
expect(screen.getByRole('checkbox', { name: 'Edit this page' })).not.toBeChecked()
|
||||
})
|
||||
|
||||
it('toggles selection from the visible checkbox labels', async () => {
|
||||
const user = userEvent.setup()
|
||||
renderSelectionForm()
|
||||
await screen.findByRole('checkbox', { name: 'Getting started' })
|
||||
|
||||
await user.click(screen.getByText('Getting started'))
|
||||
expect(screen.getByRole('checkbox', { name: 'Getting started' })).toBeChecked()
|
||||
await user.click(screen.getByText('dataset.newKnowledge.selectAll'))
|
||||
|
||||
expect(screen.getByRole('checkbox', { name: 'Getting started' })).toBeChecked()
|
||||
expect(screen.getByRole('checkbox', { name: 'Guides' })).toBeChecked()
|
||||
})
|
||||
|
||||
it('locks stale selections while keeping workflow reconciliation available', async () => {
|
||||
const onCancel = vi.fn()
|
||||
const user = userEvent.setup()
|
||||
const { onRecrawl } = renderSelectionForm(onCancel, true)
|
||||
await screen.findByRole('checkbox', { name: 'Getting started' })
|
||||
|
||||
expect(screen.getByRole('checkbox', { name: 'Getting started' })).toHaveAttribute(
|
||||
'aria-disabled',
|
||||
'true',
|
||||
)
|
||||
expect(screen.getByRole('combobox', { name: 'dataset.newKnowledge.syncPolicy' })).toBeDisabled()
|
||||
expect(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' })).toBeDisabled()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.reCrawl' }))
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.cancelAddSource' }))
|
||||
|
||||
expect(onRecrawl).toHaveBeenCalledOnce()
|
||||
expect(onCancel).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('never submits more page IDs than the selection contract accepts', async () => {
|
||||
const longPageList: PreviewPage[] = Array.from({ length: 201 }, (_, index) => ({
|
||||
pageId: `page-${index + 1}`,
|
||||
sourceUrl: `https://docs.dify.ai/page-${index + 1}`,
|
||||
title: `Page ${index + 1}`,
|
||||
}))
|
||||
const user = userEvent.setup()
|
||||
renderSelectionForm(vi.fn(), false, () => false, longPageList)
|
||||
await user.click(
|
||||
await screen.findByRole('checkbox', { name: 'dataset.newKnowledge.selectAll' }),
|
||||
)
|
||||
|
||||
const overflowPage = screen.getByRole('checkbox', { name: 'Page 201' })
|
||||
expect(overflowPage).toHaveAttribute('aria-disabled', 'true')
|
||||
expect(overflowPage).toHaveAccessibleDescription(
|
||||
'https://docs.dify.ai/page-201 dataset.newKnowledge.maxPages: 200',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.selectPages).toHaveBeenCalledOnce())
|
||||
expect(clientMock.selectPages.mock.calls[0]?.[0].body.pageIds).toHaveLength(200)
|
||||
})
|
||||
|
||||
it('validates a custom interval and submits policy plus selection exactly once', async () => {
|
||||
const selectionRequest = deferred<SourceWorkflowRun>()
|
||||
clientMock.selectPages.mockReturnValue(selectionRequest.promise)
|
||||
const user = userEvent.setup()
|
||||
renderSelectionForm()
|
||||
|
||||
await user.click(await screen.findByRole('checkbox', { name: 'Getting started' }))
|
||||
await user.selectOptions(
|
||||
screen.getByRole('combobox', { name: 'dataset.newKnowledge.syncPolicy' }),
|
||||
'custom',
|
||||
)
|
||||
const interval = screen.getByRole('spinbutton', {
|
||||
name: 'dataset.newKnowledge.customIntervalHours',
|
||||
})
|
||||
await user.clear(interval)
|
||||
await user.type(interval, '0')
|
||||
expect(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' })).toBeDisabled()
|
||||
expect(interval).toHaveAccessibleDescription('dataset.newKnowledge.customIntervalInvalid')
|
||||
await user.clear(interval)
|
||||
await user.type(interval, '6')
|
||||
|
||||
const addSource = screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' })
|
||||
await user.dblClick(addSource)
|
||||
|
||||
await waitFor(() => expect(clientMock.updatePolicy).toHaveBeenCalledOnce())
|
||||
expect(clientMock.updatePolicy).toHaveBeenCalledWith({
|
||||
body: {
|
||||
customIntervalSeconds: 21600,
|
||||
enabled: true,
|
||||
expectedRevision: 2,
|
||||
expectedSourceVersion: 3,
|
||||
mode: 'custom',
|
||||
},
|
||||
params: { id: 'space-1', sourceId: 'source-1' },
|
||||
})
|
||||
expect(clientMock.selectPages).toHaveBeenCalledOnce()
|
||||
expect(clientMock.selectPages).toHaveBeenCalledWith({
|
||||
body: { pageIds: ['page-1'] },
|
||||
headers: { 'Idempotency-Key': expect.any(String) },
|
||||
params: { id: 'space-1', runId: 'run-1' },
|
||||
})
|
||||
|
||||
selectionRequest.resolve({ ...run, checkpoint: 'import', state: 'queued' })
|
||||
await waitFor(() => expect(clientMock.getWorkflow).toHaveBeenCalledOnce())
|
||||
await waitFor(() =>
|
||||
expect(queryClientMock.invalidateQueries).toHaveBeenCalledWith({
|
||||
queryKey: ['knowledge-sources'],
|
||||
}),
|
||||
)
|
||||
expect(routerMock.push).toHaveBeenCalledWith('/datasets/new/space-1/sources')
|
||||
})
|
||||
|
||||
it.each(['complete', 'success'])(
|
||||
'accepts the %s workflow state as a successful import',
|
||||
async (state) => {
|
||||
clientMock.selectPages.mockResolvedValue({ ...run, checkpoint: 'complete', state })
|
||||
const user = userEvent.setup()
|
||||
renderSelectionForm()
|
||||
|
||||
await user.click(await screen.findByRole('checkbox', { name: 'Getting started' }))
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
|
||||
await waitFor(() =>
|
||||
expect(routerMock.push).toHaveBeenCalledWith('/datasets/new/space-1/sources'),
|
||||
)
|
||||
expect(clientMock.getWorkflow).not.toHaveBeenCalled()
|
||||
expect(screen.queryByRole('alert')).not.toBeInTheDocument()
|
||||
},
|
||||
)
|
||||
|
||||
it.each(['error', 'exhausted', 'superseded'])(
|
||||
'stops polling when the import reaches the %s workflow state',
|
||||
async (state) => {
|
||||
clientMock.selectPages.mockResolvedValue({ ...run, checkpoint: 'import', state })
|
||||
const user = userEvent.setup()
|
||||
renderSelectionForm()
|
||||
|
||||
await user.click(await screen.findByRole('checkbox', { name: 'Getting started' }))
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent(
|
||||
'dataset.newKnowledge.addSourceFailed',
|
||||
)
|
||||
expect(clientMock.getWorkflow).not.toHaveBeenCalled()
|
||||
expect(routerMock.push).not.toHaveBeenCalled()
|
||||
},
|
||||
)
|
||||
|
||||
it('tracks policy updates and stops before selection when discard is requested', async () => {
|
||||
const policyRequest = deferred<GetKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyResponse>()
|
||||
clientMock.updatePolicy.mockReturnValue(policyRequest.promise)
|
||||
let discardRequested = false
|
||||
const user = userEvent.setup()
|
||||
const { onWorkflowPending } = renderSelectionForm(vi.fn(), false, () => discardRequested)
|
||||
await user.click(await screen.findByRole('checkbox', { name: 'Getting started' }))
|
||||
await user.selectOptions(
|
||||
screen.getByRole('combobox', { name: 'dataset.newKnowledge.syncPolicy' }),
|
||||
'manual',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
|
||||
expect(onWorkflowPending).toHaveBeenCalledOnce()
|
||||
expect(clientMock.selectPages).not.toHaveBeenCalled()
|
||||
discardRequested = true
|
||||
await act(async () =>
|
||||
policyRequest.resolve(policy({ enabled: false, mode: 'manual', revision: 3 })),
|
||||
)
|
||||
await expect(onWorkflowPending.mock.calls[0]?.[0]).resolves.toBeUndefined()
|
||||
expect(clientMock.selectPages).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('releases a pending import for cancellation as soon as the current poll returns', async () => {
|
||||
const workflowRequest = deferred<SourceWorkflowRun>()
|
||||
clientMock.getWorkflow.mockReturnValue(workflowRequest.promise)
|
||||
let discardRequested = false
|
||||
const user = userEvent.setup()
|
||||
const { onWorkflowPending } = renderSelectionForm(vi.fn(), false, () => discardRequested)
|
||||
await user.click(await screen.findByRole('checkbox', { name: 'Getting started' }))
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.getWorkflow).toHaveBeenCalledOnce())
|
||||
expect(onWorkflowPending).toHaveBeenCalledOnce()
|
||||
discardRequested = true
|
||||
const importRun = { ...run, checkpoint: 'import', state: 'running' }
|
||||
await act(async () => workflowRequest.resolve(importRun))
|
||||
|
||||
await expect(onWorkflowPending.mock.calls[0]?.[0]).resolves.toEqual(importRun)
|
||||
expect(clientMock.getWorkflow).toHaveBeenCalledOnce()
|
||||
expect(queryClientMock.invalidateQueries).not.toHaveBeenCalled()
|
||||
expect(routerMock.push).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps Cancel available while the import is pending', async () => {
|
||||
const workflowRequest = deferred<SourceWorkflowRun>()
|
||||
clientMock.getWorkflow.mockReturnValue(workflowRequest.promise)
|
||||
const onCancel = vi.fn()
|
||||
const user = userEvent.setup()
|
||||
const { onWorkflowPending } = renderSelectionForm(onCancel)
|
||||
await user.click(await screen.findByRole('checkbox', { name: 'Getting started' }))
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.getWorkflow).toHaveBeenCalledOnce())
|
||||
const cancel = screen.getByRole('button', {
|
||||
name: 'dataset.newKnowledge.cancelAddSource',
|
||||
})
|
||||
expect(cancel).toBeEnabled()
|
||||
await user.click(cancel)
|
||||
expect(onCancel).toHaveBeenCalledOnce()
|
||||
|
||||
await act(async () =>
|
||||
workflowRequest.resolve({ ...run, checkpoint: 'complete', state: 'completed' }),
|
||||
)
|
||||
await expect(onWorkflowPending.mock.calls[0]?.[0]).resolves.toEqual(
|
||||
expect.objectContaining({ state: 'completed' }),
|
||||
)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ enabled: true, initialMode: 'interval', mode: 'provider' },
|
||||
{ enabled: false, initialMode: 'interval', mode: 'manual' },
|
||||
{ enabled: true, initialMode: 'manual', mode: 'interval' },
|
||||
] as const)(
|
||||
'submits the $mode sync policy using the KnowledgeFS contract',
|
||||
async ({ enabled, initialMode, mode }) => {
|
||||
clientMock.getPolicy.mockResolvedValue(
|
||||
policy({ enabled: initialMode !== 'manual', mode: initialMode }),
|
||||
)
|
||||
clientMock.updatePolicy.mockResolvedValue(policy({ enabled, mode, revision: 3 }))
|
||||
const user = userEvent.setup()
|
||||
renderSelectionForm()
|
||||
|
||||
await user.click(await screen.findByRole('checkbox', { name: 'Getting started' }))
|
||||
await user.selectOptions(
|
||||
screen.getByRole('combobox', { name: 'dataset.newKnowledge.syncPolicy' }),
|
||||
mode,
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.updatePolicy).toHaveBeenCalledOnce())
|
||||
expect(clientMock.updatePolicy).toHaveBeenCalledWith({
|
||||
body: {
|
||||
enabled,
|
||||
expectedRevision: 2,
|
||||
expectedSourceVersion: 3,
|
||||
mode,
|
||||
},
|
||||
params: { id: 'space-1', sourceId: 'source-1' },
|
||||
})
|
||||
expect(clientMock.selectPages).toHaveBeenCalledOnce()
|
||||
},
|
||||
)
|
||||
|
||||
it('reconciles a lost sync-policy response before importing the selected pages', async () => {
|
||||
clientMock.getPolicy
|
||||
.mockReset()
|
||||
.mockResolvedValueOnce(policy())
|
||||
.mockResolvedValueOnce(policy({ enabled: false, mode: 'manual', revision: 3 }))
|
||||
clientMock.updatePolicy.mockRejectedValue(new Error('response lost'))
|
||||
const user = userEvent.setup()
|
||||
renderSelectionForm()
|
||||
|
||||
await user.click(await screen.findByRole('checkbox', { name: 'Getting started' }))
|
||||
await user.selectOptions(
|
||||
screen.getByRole('combobox', { name: 'dataset.newKnowledge.syncPolicy' }),
|
||||
'manual',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.selectPages).toHaveBeenCalledOnce())
|
||||
expect(clientMock.getPolicy).toHaveBeenCalledTimes(2)
|
||||
expect(screen.queryByRole('alert')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('allows discard when only the sync-policy outcome is uncertain', async () => {
|
||||
clientMock.getPolicy
|
||||
.mockReset()
|
||||
.mockResolvedValueOnce(policy())
|
||||
.mockRejectedValueOnce(new Error('reconciliation unavailable'))
|
||||
clientMock.updatePolicy.mockRejectedValue(new Error('response lost'))
|
||||
const onCancel = vi.fn()
|
||||
const user = userEvent.setup()
|
||||
const { onSubmissionUncertainChange } = renderSelectionForm(onCancel)
|
||||
await user.click(await screen.findByRole('checkbox', { name: 'Getting started' }))
|
||||
await user.selectOptions(
|
||||
screen.getByRole('combobox', { name: 'dataset.newKnowledge.syncPolicy' }),
|
||||
'manual',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent(
|
||||
'dataset.newKnowledge.addSourceFailed',
|
||||
)
|
||||
expect(onSubmissionUncertainChange).not.toHaveBeenCalledWith(true)
|
||||
expect(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' })).toBeEnabled()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.cancelAddSource' }))
|
||||
expect(onCancel).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('uses the latest concurrency tokens after a conflicting sync-policy update', async () => {
|
||||
clientMock.getPolicy
|
||||
.mockReset()
|
||||
.mockResolvedValueOnce(policy())
|
||||
.mockResolvedValueOnce(
|
||||
policy({
|
||||
expectedSourceVersion: 4,
|
||||
mode: 'provider',
|
||||
revision: 3,
|
||||
}),
|
||||
)
|
||||
clientMock.updatePolicy
|
||||
.mockRejectedValueOnce(Object.assign(new Error('conflict'), { status: 409 }))
|
||||
.mockResolvedValueOnce(policy({ enabled: false, mode: 'manual', revision: 4 }))
|
||||
const user = userEvent.setup()
|
||||
renderSelectionForm()
|
||||
|
||||
await user.click(await screen.findByRole('checkbox', { name: 'Getting started' }))
|
||||
await user.selectOptions(
|
||||
screen.getByRole('combobox', { name: 'dataset.newKnowledge.syncPolicy' }),
|
||||
'manual',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent(
|
||||
'dataset.newKnowledge.addSourceFailed',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.updatePolicy).toHaveBeenCalledTimes(2))
|
||||
expect(clientMock.updatePolicy.mock.calls[1]?.[0]).toEqual({
|
||||
body: {
|
||||
enabled: false,
|
||||
expectedRevision: 3,
|
||||
expectedSourceVersion: 4,
|
||||
mode: 'manual',
|
||||
},
|
||||
params: { id: 'space-1', sourceId: 'source-1' },
|
||||
})
|
||||
expect(clientMock.selectPages).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('recovers when the initial sync policy cannot load', async () => {
|
||||
clientMock.getPolicy
|
||||
.mockReset()
|
||||
.mockRejectedValueOnce(new Error('temporary failure'))
|
||||
.mockResolvedValueOnce(policy())
|
||||
const user = userEvent.setup()
|
||||
renderSelectionForm()
|
||||
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent(
|
||||
'dataset.newKnowledge.syncPolicyLoadFailed',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.retrySyncPolicy' }))
|
||||
|
||||
await screen.findByRole('checkbox', { name: 'Getting started' })
|
||||
expect(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' })).toBeDisabled()
|
||||
})
|
||||
|
||||
it('creates the initial provider policy when a provisional source has no policy yet', async () => {
|
||||
clientMock.getPolicy.mockRejectedValue(
|
||||
Object.assign(new Error('policy not found'), { status: 404 }),
|
||||
)
|
||||
clientMock.updatePolicy.mockResolvedValue(policy({ mode: 'provider', revision: 1 }))
|
||||
const user = userEvent.setup()
|
||||
renderSelectionForm()
|
||||
|
||||
await user.click(await screen.findByRole('checkbox', { name: 'Getting started' }))
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.updatePolicy).toHaveBeenCalledOnce())
|
||||
expect(clientMock.updatePolicy).toHaveBeenCalledWith({
|
||||
body: {
|
||||
enabled: true,
|
||||
expectedRevision: 0,
|
||||
expectedSourceVersion: 3,
|
||||
mode: 'provider',
|
||||
},
|
||||
params: { id: 'space-1', sourceId: 'source-1' },
|
||||
})
|
||||
expect(clientMock.selectPages).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('keeps Cancel available when the sync policy cannot load', async () => {
|
||||
clientMock.getPolicy.mockRejectedValue(new Error('temporary failure'))
|
||||
const onCancel = vi.fn()
|
||||
const user = userEvent.setup()
|
||||
renderSelectionForm(onCancel)
|
||||
|
||||
await screen.findByRole('alert')
|
||||
expect(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' })).toBeDisabled()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.cancelAddSource' }))
|
||||
|
||||
expect(onCancel).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('keeps Cancel available while the sync policy is loading', async () => {
|
||||
const policyRequest = deferred<GetKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyResponse>()
|
||||
clientMock.getPolicy.mockReturnValue(policyRequest.promise)
|
||||
const onCancel = vi.fn()
|
||||
const user = userEvent.setup()
|
||||
renderSelectionForm(onCancel)
|
||||
|
||||
expect(
|
||||
screen.getByRole('status', { name: 'dataset.newKnowledge.loadingSyncPolicy' }),
|
||||
).toBeVisible()
|
||||
expect(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' })).toBeDisabled()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.cancelAddSource' }))
|
||||
|
||||
expect(onCancel).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('uses a new selection idempotency key after a confirmed import failure', async () => {
|
||||
clientMock.getWorkflow.mockResolvedValueOnce({ ...run, checkpoint: 'import', state: 'failed' })
|
||||
const user = userEvent.setup()
|
||||
renderSelectionForm()
|
||||
|
||||
await user.click(await screen.findByRole('checkbox', { name: 'Getting started' }))
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent(
|
||||
'dataset.newKnowledge.addSourceFailed',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.selectPages).toHaveBeenCalledTimes(2))
|
||||
expect(clientMock.selectPages.mock.calls[0]?.[0].headers['Idempotency-Key']).not.toBe(
|
||||
clientMock.selectPages.mock.calls[1]?.[0].headers['Idempotency-Key'],
|
||||
)
|
||||
})
|
||||
|
||||
it('reuses the selection idempotency key after a lost response without rewriting policy', async () => {
|
||||
clientMock.getPolicy.mockResolvedValue(policy({ enabled: false, mode: 'manual' }))
|
||||
clientMock.selectPages
|
||||
.mockRejectedValueOnce(new Error('response lost'))
|
||||
.mockResolvedValueOnce({ ...run, checkpoint: 'import', state: 'queued' })
|
||||
const user = userEvent.setup()
|
||||
const { onSubmissionUncertainChange, onWorkflowPending, onWorkflowRun } = renderSelectionForm()
|
||||
|
||||
await user.click(await screen.findByRole('checkbox', { name: 'Getting started' }))
|
||||
await user.selectOptions(
|
||||
screen.getByRole('combobox', { name: 'dataset.newKnowledge.syncPolicy' }),
|
||||
'manual',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent(
|
||||
'dataset.newKnowledge.addSourceFailed',
|
||||
)
|
||||
expect(onWorkflowPending).toHaveBeenCalledOnce()
|
||||
expect(onSubmissionUncertainChange).toHaveBeenLastCalledWith(true)
|
||||
expect(screen.getByRole('checkbox', { name: 'Guides' })).toHaveAttribute(
|
||||
'aria-disabled',
|
||||
'true',
|
||||
)
|
||||
expect(screen.getByRole('combobox', { name: 'dataset.newKnowledge.syncPolicy' })).toBeDisabled()
|
||||
expect(
|
||||
screen.getByRole('button', { name: 'dataset.newKnowledge.cancelAddSource' }),
|
||||
).toBeEnabled()
|
||||
await user.click(screen.getByText('Guides'))
|
||||
expect(screen.getByRole('checkbox', { name: 'Guides' })).not.toBeChecked()
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.addSource' }))
|
||||
|
||||
await waitFor(() => expect(clientMock.selectPages).toHaveBeenCalledTimes(2))
|
||||
expect(onWorkflowRun).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ checkpoint: 'import', state: 'queued' }),
|
||||
)
|
||||
expect(clientMock.updatePolicy).not.toHaveBeenCalled()
|
||||
expect(clientMock.selectPages.mock.calls[0]?.[0].headers).toEqual(
|
||||
clientMock.selectPages.mock.calls[1]?.[0].headers,
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -3,6 +3,7 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
|
||||
import { act, render, screen, waitFor, within } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { CreateKnowledgePage } from '../create-knowledge-page'
|
||||
import { newKnowledgeSourceDraftStorageKey } from '../routes'
|
||||
|
||||
const serviceMock = vi.hoisted(() => ({
|
||||
create: vi.fn(),
|
||||
@@ -103,6 +104,7 @@ async function choosePermission(user: ReturnType<typeof userEvent.setup>, option
|
||||
describe('CreateKnowledgePage', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
globalThis.sessionStorage.clear()
|
||||
serviceMock.create.mockResolvedValue(createdKnowledge)
|
||||
serviceMock.getPolicy.mockResolvedValue({
|
||||
id: 'policy-1',
|
||||
@@ -126,6 +128,7 @@ describe('CreateKnowledgePage', () => {
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.sessionStorage.clear()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
@@ -370,27 +373,82 @@ describe('CreateKnowledgePage', () => {
|
||||
expect(serviceMock.patchPolicy).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('keeps source and upload disabled until their stacked flows land', () => {
|
||||
it('integrates source setup while keeping the later upload flow disabled', async () => {
|
||||
const user = userEvent.setup()
|
||||
navigationMock.startMode = 'source'
|
||||
renderPage()
|
||||
|
||||
const startEmpty = screen.getByRole('radio', { name: 'dataset.newKnowledge.startEmpty' })
|
||||
expect(startEmpty).toBeChecked()
|
||||
expect(startEmpty).not.toBeChecked()
|
||||
expect(startEmpty).toHaveAccessibleDescription('dataset.newKnowledge.startEmptyDescription')
|
||||
const connectSource = screen.getByRole('radio', {
|
||||
name: 'dataset.newKnowledge.connectSource',
|
||||
})
|
||||
const uploadFiles = screen.getByRole('radio', { name: 'dataset.newKnowledge.uploadFiles' })
|
||||
expect(connectSource).toBeDisabled()
|
||||
expect(connectSource).toHaveAccessibleDescription(
|
||||
'dataset.newKnowledge.connectSourceDescription dataset.cornerLabel.unavailable',
|
||||
expect(connectSource).toBeEnabled()
|
||||
expect(connectSource).toBeChecked()
|
||||
expect(screen.getByRole('radio', { name: 'dataset.newKnowledge.websiteCrawl' })).toBeChecked()
|
||||
expect(screen.getByRole('radio', { name: 'Firecrawl' })).toBeChecked()
|
||||
expect(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' })).toBeDisabled()
|
||||
|
||||
await user.type(
|
||||
screen.getByPlaceholderText('dataset.newKnowledge.rootUrlPlaceholder'),
|
||||
'https://docs.dify.ai',
|
||||
)
|
||||
await user.type(
|
||||
screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'),
|
||||
'Dify docs',
|
||||
)
|
||||
expect(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' })).toBeEnabled()
|
||||
expect(uploadFiles).toBeDisabled()
|
||||
expect(uploadFiles).toHaveAccessibleDescription(
|
||||
'dataset.newKnowledge.uploadFilesDescription dataset.cornerLabel.unavailable',
|
||||
)
|
||||
})
|
||||
|
||||
it('hands the configured source draft to the add-source workflow after creation', async () => {
|
||||
const user = userEvent.setup()
|
||||
navigationMock.startMode = 'source'
|
||||
renderPage()
|
||||
await fillRequiredFields(user)
|
||||
await user.type(
|
||||
screen.getByPlaceholderText('dataset.newKnowledge.rootUrlPlaceholder'),
|
||||
'https://docs.dify.ai',
|
||||
)
|
||||
await user.type(
|
||||
screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'),
|
||||
'Dify docs',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlOptions' }))
|
||||
await user.click(screen.getByRole('checkbox', { name: 'dataset.newKnowledge.includeSubpages' }))
|
||||
const maxPages = screen.getByRole('spinbutton', { name: 'dataset.newKnowledge.maxPages' })
|
||||
await user.clear(maxPages)
|
||||
await user.type(maxPages, '25')
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
|
||||
|
||||
await waitFor(() =>
|
||||
expect(routerMock.replace).toHaveBeenCalledWith(
|
||||
'/datasets/new/e735c1dc-d2b8-4dc4-86dc-abaf2fb7d084/sources/new?type=websiteCrawl&draft=a9c36c57-2d84-44d6-a36d-841f0d92a179',
|
||||
),
|
||||
)
|
||||
expect(
|
||||
JSON.parse(
|
||||
globalThis.sessionStorage.getItem(
|
||||
newKnowledgeSourceDraftStorageKey('a9c36c57-2d84-44d6-a36d-841f0d92a179'),
|
||||
) ?? '',
|
||||
),
|
||||
).toEqual({
|
||||
includeSubpages: false,
|
||||
maxPages: 25,
|
||||
provider: 'Firecrawl',
|
||||
rootUrl: 'https://docs.dify.ai',
|
||||
sourceName: 'Dify docs',
|
||||
sourceType: 'websiteCrawl',
|
||||
syncPolicy: 'provider',
|
||||
})
|
||||
})
|
||||
|
||||
it('renders the approved creation modal and exposes both dismiss actions', async () => {
|
||||
const user = userEvent.setup()
|
||||
renderPage()
|
||||
@@ -453,6 +511,27 @@ describe('CreateKnowledgePage', () => {
|
||||
expect(routerMock.replace).toHaveBeenCalledWith('/datasets?view=new')
|
||||
})
|
||||
|
||||
it('asks before discarding a preserved source draft after switching source types', async () => {
|
||||
const user = userEvent.setup()
|
||||
navigationMock.startMode = 'source'
|
||||
renderPage()
|
||||
|
||||
await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDocuments' }))
|
||||
await user.type(
|
||||
screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'),
|
||||
'Release notes',
|
||||
)
|
||||
await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.websiteCrawl' }))
|
||||
await user.click(screen.getByRole('button', { name: 'common.operation.close' }))
|
||||
|
||||
expect(
|
||||
await screen.findByRole('alertdialog', {
|
||||
name: 'dataset.newKnowledge.discardDraftTitle',
|
||||
}),
|
||||
).toBeInTheDocument()
|
||||
expect(routerMock.replace).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('protects an unsaved draft from browser unload', async () => {
|
||||
const user = userEvent.setup()
|
||||
renderPage()
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import { createRequestId } from '../request-id'
|
||||
|
||||
const uuidV4Mock = vi.hoisted(() => vi.fn())
|
||||
|
||||
vi.mock('uuid', () => ({
|
||||
v4: uuidV4Mock,
|
||||
}))
|
||||
|
||||
describe('createRequestId', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
uuidV4Mock.mockReturnValue('fallback-request-id')
|
||||
})
|
||||
|
||||
it('uses the native random UUID when it is available', () => {
|
||||
const randomUUID = vi
|
||||
.spyOn(globalThis.crypto, 'randomUUID')
|
||||
.mockReturnValue('11111111-1111-4111-8111-111111111111')
|
||||
|
||||
expect(createRequestId()).toBe('11111111-1111-4111-8111-111111111111')
|
||||
expect(uuidV4Mock).not.toHaveBeenCalled()
|
||||
|
||||
randomUUID.mockRestore()
|
||||
})
|
||||
|
||||
it('falls back when randomUUID is unavailable', () => {
|
||||
const descriptor = Object.getOwnPropertyDescriptor(globalThis.crypto, 'randomUUID')
|
||||
Object.defineProperty(globalThis.crypto, 'randomUUID', {
|
||||
configurable: true,
|
||||
value: undefined,
|
||||
})
|
||||
|
||||
try {
|
||||
expect(createRequestId()).toBe('fallback-request-id')
|
||||
expect(uuidV4Mock).toHaveBeenCalledOnce()
|
||||
} finally {
|
||||
if (descriptor) Object.defineProperty(globalThis.crypto, 'randomUUID', descriptor)
|
||||
else Reflect.deleteProperty(globalThis.crypto, 'randomUUID')
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -154,14 +154,19 @@ describe('SourcesPage', () => {
|
||||
'href',
|
||||
'/datasets/new/space-1/sources/new',
|
||||
)
|
||||
for (const brand of ['firecrawl', 'jina', 'notion', 'google-drive', 'confluence', 'more'])
|
||||
expect(container.querySelector(`[data-brand="${brand}"]`)).toBeInTheDocument()
|
||||
expect(container.querySelector('[data-brand="firecrawl"]')?.tagName).toBe('SPAN')
|
||||
expect(container.querySelector('[data-brand="firecrawl"]')).toHaveClass('size-8')
|
||||
expect(container.querySelector('[data-brand="jina"]')).toHaveClass('i-custom-public-llm-jina')
|
||||
expect(container.querySelector('[data-brand="confluence"]')).toHaveClass(
|
||||
'i-custom-public-common-confluence',
|
||||
)
|
||||
for (const [brand, iconClass] of [
|
||||
['firecrawl', 'i-custom-public-common-firecrawl'],
|
||||
['jina', 'i-custom-public-llm-jina'],
|
||||
['notion', 'i-custom-public-common-notion'],
|
||||
['google-drive', 'i-custom-public-common-google-drive'],
|
||||
['confluence', 'i-custom-public-common-confluence'],
|
||||
['dropbox', 'i-custom-public-common-dropbox'],
|
||||
] as const) {
|
||||
const icon = container.querySelector(`[data-brand="${brand}"]`)
|
||||
expect(icon).toBeInTheDocument()
|
||||
expect(icon?.tagName).toBe('SPAN')
|
||||
expect(icon).toHaveClass(iconClass)
|
||||
}
|
||||
})
|
||||
|
||||
it('renders real source statuses and filters by status and search text', async () => {
|
||||
@@ -231,6 +236,102 @@ describe('SourcesPage', () => {
|
||||
expect(within(rowsDescending[1]!).getByText('Alpha docs')).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('places the newest source first until the user selects a name sort', () => {
|
||||
sourcesQuery.data = {
|
||||
pages: [
|
||||
{
|
||||
items: [
|
||||
source({ id: 'older', name: 'Older source', createdAt: '2026-07-20T10:00:00Z' }),
|
||||
source({ id: 'newer', name: 'Newest source', createdAt: '2026-07-20T10:01:00Z' }),
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
render(<SourcesPage knowledgeSpaceId="space-1" />)
|
||||
|
||||
const rows = screen.getAllByRole('row').slice(1)
|
||||
expect(within(rows[0]!).getByText('Newest source')).toBeInTheDocument()
|
||||
expect(within(rows[1]!).getByText('Older source')).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('keeps provisional crawl sources out of the source list', () => {
|
||||
sourcesQuery.data = {
|
||||
pages: [
|
||||
{
|
||||
items: [
|
||||
source({
|
||||
id: 'preview',
|
||||
metadata: { preview: true },
|
||||
name: 'Discarded preview',
|
||||
status: 'disabled',
|
||||
}),
|
||||
source({ id: 'connected', name: 'Connected source' }),
|
||||
source({
|
||||
id: 'submitted-preview',
|
||||
metadata: { preview: true },
|
||||
name: 'Submitted preview',
|
||||
status: 'syncing',
|
||||
}),
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
render(<SourcesPage knowledgeSpaceId="space-1" />)
|
||||
|
||||
expect(screen.queryByText('Discarded preview')).not.toBeInTheDocument()
|
||||
expect(screen.getByText('Connected source')).toBeInTheDocument()
|
||||
expect(screen.getByText('Submitted preview')).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('continues past cursor pages containing only hidden preview drafts', () => {
|
||||
sourcesQuery.data = {
|
||||
pages: [
|
||||
{
|
||||
items: [
|
||||
source({
|
||||
id: 'preview',
|
||||
metadata: { preview: true },
|
||||
status: 'disabled',
|
||||
}),
|
||||
],
|
||||
nextCursor: 'next',
|
||||
},
|
||||
],
|
||||
}
|
||||
sourcesQuery.hasNextPage = true
|
||||
|
||||
render(<SourcesPage knowledgeSpaceId="space-1" />)
|
||||
|
||||
expect(sourcesQuery.fetchNextPage).toHaveBeenCalledOnce()
|
||||
expect(screen.queryByText('dataset.newKnowledge.sourcesEmptyTitle')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('continues when the newest loaded page contributes only hidden preview drafts', () => {
|
||||
sourcesQuery.data = {
|
||||
pages: [
|
||||
{ items: [source({ id: 'connected', name: 'Connected source' })] },
|
||||
{
|
||||
items: [
|
||||
source({
|
||||
id: 'preview',
|
||||
metadata: { preview: true },
|
||||
status: 'disabled',
|
||||
}),
|
||||
],
|
||||
nextCursor: 'next',
|
||||
},
|
||||
],
|
||||
}
|
||||
sourcesQuery.hasNextPage = true
|
||||
|
||||
render(<SourcesPage knowledgeSpaceId="space-1" />)
|
||||
|
||||
expect(screen.getByText('Connected source')).toBeInTheDocument()
|
||||
expect(sourcesQuery.fetchNextPage).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('opens a source URI from the row action menu', async () => {
|
||||
const user = userEvent.setup()
|
||||
sourcesQuery.data = { pages: [{ items: [source({})] }] }
|
||||
@@ -361,19 +462,21 @@ describe('SourcesPage', () => {
|
||||
sourcesQuery.data = {
|
||||
pages: [
|
||||
{
|
||||
items: [source({ id: 'active-source' }), source({ id: 'disabled', status: 'disabled' })],
|
||||
items: [
|
||||
source({ id: 'active-source', name: 'Active source' }),
|
||||
source({ id: 'disabled', name: 'Disabled source', status: 'disabled' }),
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
render(<SourcesPage knowledgeSpaceId="space-1" />)
|
||||
const actionButtons = screen.getAllByRole('button', {
|
||||
name: /dataset.newKnowledge.sourceActions/,
|
||||
})
|
||||
const activeSourceActions = actionButtons[0]
|
||||
const disabledSourceActions = actionButtons[1]
|
||||
if (!activeSourceActions || !disabledSourceActions)
|
||||
throw new Error('Expected actions for both sources')
|
||||
const activeSourceActions = within(
|
||||
screen.getByRole('row', { name: /Active source/ }),
|
||||
).getByRole('button', { name: /dataset.newKnowledge.sourceActions/ })
|
||||
const disabledSourceActions = within(
|
||||
screen.getByRole('row', { name: /Disabled source/ }),
|
||||
).getByRole('button', { name: /dataset.newKnowledge.sourceActions/ })
|
||||
await user.click(activeSourceActions)
|
||||
await user.click(screen.getByRole('menuitem', { name: 'dataset.newKnowledge.disableSource' }))
|
||||
await waitFor(() =>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,269 @@
|
||||
'use client'
|
||||
|
||||
import type {
|
||||
NewKnowledgeOnlineDocumentsProvider,
|
||||
NewKnowledgeOnlineDocumentsSourceDraft,
|
||||
NewKnowledgeOnlineDriveProvider,
|
||||
NewKnowledgeOnlineDriveSourceDraft,
|
||||
NewKnowledgeSourceDraft,
|
||||
NewKnowledgeWebsiteSourceDraft,
|
||||
} from './routes'
|
||||
import { Button } from '@langgenius/dify-ui/button'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import {
|
||||
isValidWebsiteSourceDraft,
|
||||
NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH,
|
||||
NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH,
|
||||
} from './routes'
|
||||
|
||||
const connectedProviders = {
|
||||
onlineDocuments: [
|
||||
{ icon: 'i-custom-public-common-notion', label: 'Notion' },
|
||||
{ icon: 'i-ri-file-text-line', label: 'Google Docs' },
|
||||
{ icon: 'i-ri-links-line', label: 'Confluence' },
|
||||
],
|
||||
onlineDrive: [
|
||||
{ icon: 'i-custom-public-common-google-drive', label: 'Google Drive' },
|
||||
{ icon: 'i-ri-cloud-line', label: 'OneDrive' },
|
||||
{ icon: 'i-ri-box-3-line', label: 'Amazon S3' },
|
||||
],
|
||||
} as const
|
||||
|
||||
export function PendingWebsiteSetup({
|
||||
draft,
|
||||
onDraftChange,
|
||||
}: {
|
||||
draft: NewKnowledgeWebsiteSourceDraft
|
||||
onDraftChange: (draft: NewKnowledgeWebsiteSourceDraft) => void
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
const [optionsExpanded, setOptionsExpanded] = useState(false)
|
||||
const [backendBoundaryVisible, setBackendBoundaryVisible] = useState(false)
|
||||
const updateDraft = (nextDraft: NewKnowledgeWebsiteSourceDraft) => {
|
||||
onDraftChange(nextDraft)
|
||||
setBackendBoundaryVisible(false)
|
||||
}
|
||||
|
||||
return (
|
||||
<section className="space-y-4" aria-label={t(($) => $['newKnowledge.crawlAndPreview'])}>
|
||||
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
|
||||
<label className="block system-xs-medium text-text-secondary">
|
||||
{t(($) => $['newKnowledge.rootUrl'])}
|
||||
<input
|
||||
type="url"
|
||||
inputMode="url"
|
||||
autoComplete="off"
|
||||
maxLength={NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH}
|
||||
value={draft.rootUrl}
|
||||
placeholder={t(($) => $['newKnowledge.rootUrlPlaceholder'])}
|
||||
className="mt-1.5 h-9 w-full rounded-lg border-0 bg-components-input-bg-normal px-3 system-sm-regular text-text-primary outline-hidden placeholder:text-text-quaternary focus:ring-2 focus:ring-state-accent-solid"
|
||||
onChange={(event) => {
|
||||
updateDraft({ ...draft, rootUrl: event.target.value })
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
<label className="block system-xs-medium text-text-secondary">
|
||||
{t(($) => $['newKnowledge.sourceName'])}
|
||||
<input
|
||||
type="text"
|
||||
autoComplete="off"
|
||||
maxLength={NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH}
|
||||
value={draft.sourceName}
|
||||
placeholder={t(($) => $['newKnowledge.sourceNamePlaceholder'])}
|
||||
className="mt-1.5 h-9 w-full rounded-lg border-0 bg-components-input-bg-normal px-3 system-sm-regular text-text-primary outline-hidden placeholder:text-text-quaternary focus:ring-2 focus:ring-state-accent-solid"
|
||||
onChange={(event) => {
|
||||
updateDraft({ ...draft, sourceName: event.target.value })
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
<div className="overflow-hidden rounded-lg bg-background-section">
|
||||
<button
|
||||
type="button"
|
||||
aria-expanded={optionsExpanded}
|
||||
className="flex min-h-9 w-full items-center gap-2 px-3 text-left system-xs-medium text-text-secondary outline-hidden hover:bg-state-base-hover focus-visible:ring-2 focus-visible:ring-state-accent-solid"
|
||||
onClick={() => setOptionsExpanded((value) => !value)}
|
||||
>
|
||||
<span
|
||||
aria-hidden
|
||||
className={cn(
|
||||
'size-4',
|
||||
optionsExpanded ? 'i-ri-arrow-down-s-line' : 'i-ri-arrow-right-s-line',
|
||||
)}
|
||||
/>
|
||||
{t(($) => $['newKnowledge.crawlOptions'])}
|
||||
{!optionsExpanded && (
|
||||
<span className="ml-auto system-xs-regular text-text-tertiary">
|
||||
{t(($) => $['newKnowledge.usingDefaults'])}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
{optionsExpanded && (
|
||||
<fieldset className="grid grid-cols-1 gap-3 px-3 pb-3 sm:grid-cols-2">
|
||||
<label className="flex items-center gap-2 system-xs-regular text-text-secondary">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={draft.includeSubpages}
|
||||
onChange={(event) =>
|
||||
updateDraft({ ...draft, includeSubpages: event.target.checked })
|
||||
}
|
||||
/>
|
||||
{t(($) => $['newKnowledge.includeSubpages'])}
|
||||
</label>
|
||||
<label className="system-xs-medium text-text-secondary">
|
||||
{t(($) => $['newKnowledge.maxPages'])}
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={200}
|
||||
value={draft.maxPages}
|
||||
className="mt-1.5 h-9 w-full rounded-lg border-0 bg-components-input-bg-normal px-3 system-sm-regular text-text-primary outline-hidden focus:ring-2 focus:ring-state-accent-solid"
|
||||
onChange={(event) =>
|
||||
updateDraft({ ...draft, maxPages: event.target.valueAsNumber || 0 })
|
||||
}
|
||||
/>
|
||||
</label>
|
||||
</fieldset>
|
||||
)}
|
||||
</div>
|
||||
<Button
|
||||
type="button"
|
||||
variant="primary"
|
||||
className="w-full"
|
||||
disabled={!isValidWebsiteSourceDraft(draft)}
|
||||
onClick={() => setBackendBoundaryVisible(true)}
|
||||
>
|
||||
{t(($) => $['newKnowledge.crawlAndPreview'])}
|
||||
</Button>
|
||||
<div className="flex min-h-40 flex-col items-center justify-center rounded-lg border border-dashed border-divider-regular px-6 text-center">
|
||||
<span className="flex size-10 items-center justify-center rounded-lg bg-background-section">
|
||||
<span aria-hidden className="i-ri-global-line size-5 text-text-tertiary" />
|
||||
</span>
|
||||
<p className="mt-2 system-xs-semibold text-text-primary">
|
||||
{t(($) => $['newKnowledge.pagesAppearTitle'])}
|
||||
</p>
|
||||
<p className="mt-2 system-xs-regular text-text-tertiary">
|
||||
{t(($) => $['newKnowledge.pagesAppearDescription'])}
|
||||
</p>
|
||||
</div>
|
||||
{backendBoundaryVisible && (
|
||||
<p
|
||||
role="alert"
|
||||
className="rounded-md bg-components-badge-status-light-warning-bg px-3 py-2 system-xs-regular text-text-warning"
|
||||
>
|
||||
{t(($) => $['newKnowledge.sourceSetupBackendDependency'])}
|
||||
</p>
|
||||
)}
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
export function UnavailableConnectedSourceSetup({
|
||||
draft,
|
||||
onDraftChange,
|
||||
}: {
|
||||
draft: NewKnowledgeOnlineDocumentsSourceDraft | NewKnowledgeOnlineDriveSourceDraft
|
||||
onDraftChange: (draft: NewKnowledgeSourceDraft) => void
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
const { t: tCreation } = useTranslation('datasetCreation')
|
||||
const sourceType = draft.sourceType
|
||||
const providers = connectedProviders[sourceType]
|
||||
const activeProvider = draft.provider
|
||||
const selectProvider = (provider: string) => {
|
||||
if (draft.sourceType === 'onlineDocuments')
|
||||
onDraftChange({ ...draft, provider: provider as NewKnowledgeOnlineDocumentsProvider })
|
||||
else onDraftChange({ ...draft, provider: provider as NewKnowledgeOnlineDriveProvider })
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<fieldset>
|
||||
<legend className="mb-1.5 system-xs-medium text-text-secondary">
|
||||
{tCreation(($) => $['stepOne.website.chooseProvider'])}
|
||||
</legend>
|
||||
<div className="grid grid-cols-1 gap-2 sm:grid-cols-3">
|
||||
{providers.map((option) => (
|
||||
<label
|
||||
key={option.label}
|
||||
className={cn(
|
||||
'flex min-h-9 cursor-pointer items-center justify-center gap-2 rounded-lg border px-3 system-xs-medium outline-hidden has-focus-visible:ring-2 has-focus-visible:ring-state-accent-solid',
|
||||
activeProvider === option.label
|
||||
? 'border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg text-text-primary'
|
||||
: 'border-divider-subtle text-text-secondary hover:bg-state-base-hover',
|
||||
)}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={`${sourceType}-provider`}
|
||||
value={option.label}
|
||||
checked={activeProvider === option.label}
|
||||
onChange={() => selectProvider(option.label)}
|
||||
className="sr-only"
|
||||
/>
|
||||
<span aria-hidden className={`${option.icon} size-4 shrink-0`} />
|
||||
<span className="truncate">{option.label}</span>
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
</fieldset>
|
||||
|
||||
{sourceType === 'onlineDrive' && (
|
||||
<section
|
||||
aria-label={t(($) => $['newKnowledge.selectFilesAndFolders'])}
|
||||
className="flex min-h-40 flex-col items-center justify-center rounded-lg border border-dashed border-divider-regular px-6 text-center"
|
||||
>
|
||||
<span className="flex size-10 items-center justify-center rounded-lg bg-background-section">
|
||||
<span aria-hidden className="i-ri-folder-open-line size-5 text-text-tertiary" />
|
||||
</span>
|
||||
<p className="mt-2 system-xs-semibold text-text-primary">
|
||||
{t(($) => $['newKnowledge.selectFilesAndFolders'])}
|
||||
</p>
|
||||
</section>
|
||||
)}
|
||||
|
||||
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
|
||||
<label className="block system-xs-medium text-text-secondary">
|
||||
{t(($) => $['newKnowledge.sourceName'])}
|
||||
<span aria-hidden className="ml-0.5 text-text-destructive">
|
||||
*
|
||||
</span>
|
||||
<input
|
||||
type="text"
|
||||
autoComplete="off"
|
||||
maxLength={NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH}
|
||||
value={draft.sourceName}
|
||||
placeholder={t(($) => $['newKnowledge.sourceNamePlaceholder'])}
|
||||
className="mt-1.5 h-9 w-full rounded-lg border-0 bg-components-input-bg-normal px-3 system-sm-regular text-text-primary outline-hidden placeholder:text-text-quaternary focus:ring-2 focus:ring-state-accent-solid"
|
||||
onChange={(event) => onDraftChange({ ...draft, sourceName: event.target.value })}
|
||||
/>
|
||||
</label>
|
||||
<label className="block system-xs-medium text-text-secondary">
|
||||
{t(($) => $['newKnowledge.syncPolicy'])}
|
||||
<select
|
||||
value={draft.syncPolicy}
|
||||
className="mt-1.5 h-9 w-full rounded-lg border-0 bg-components-input-bg-normal px-3 system-sm-regular text-text-primary outline-hidden focus:ring-2 focus:ring-state-accent-solid"
|
||||
onChange={(event) =>
|
||||
onDraftChange({
|
||||
...draft,
|
||||
syncPolicy: event.target.value as NewKnowledgeSourceDraft['syncPolicy'],
|
||||
})
|
||||
}
|
||||
>
|
||||
<option value="provider">{t(($) => $['newKnowledge.syncPolicyProvider'])}</option>
|
||||
<option value="daily">{t(($) => $['newKnowledge.syncPolicyDaily'])}</option>
|
||||
<option value="manual">{t(($) => $['newKnowledge.syncPolicyManual'])}</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div role="status" className="rounded-lg bg-background-section px-3 py-2">
|
||||
<p className="system-xs-regular text-text-tertiary">
|
||||
{t(($) => $['newKnowledge.providerUnavailable'])}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
'use client'
|
||||
|
||||
import type { ReactNode } from 'react'
|
||||
import type { NewKnowledgeStartMode } from '../routes'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { RadioControl, RadioItem } from '@langgenius/dify-ui/radio'
|
||||
@@ -7,15 +8,19 @@ import { useId } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
export function StartMode({
|
||||
children,
|
||||
description,
|
||||
disabled = false,
|
||||
icon,
|
||||
selected = false,
|
||||
title,
|
||||
value,
|
||||
}: {
|
||||
children?: ReactNode
|
||||
description: string
|
||||
disabled?: boolean
|
||||
icon: string
|
||||
selected?: boolean
|
||||
title: string
|
||||
value: NewKnowledgeStartMode
|
||||
}) {
|
||||
@@ -25,44 +30,52 @@ export function StartMode({
|
||||
const unavailableId = useId()
|
||||
|
||||
return (
|
||||
<RadioItem
|
||||
value={value}
|
||||
nativeButton
|
||||
render={<button type="button" />}
|
||||
aria-labelledby={titleId}
|
||||
aria-describedby={disabled ? `${descriptionId} ${unavailableId}` : descriptionId}
|
||||
disabled={disabled}
|
||||
<div
|
||||
className={cn(
|
||||
'relative flex min-h-16 w-full items-center gap-3 overflow-hidden rounded-xl border border-components-option-card-option-border bg-components-option-card-option-bg px-4 py-3.5 text-left outline-hidden transition-colors motion-reduce:transition-none',
|
||||
'hover:bg-state-base-hover focus-visible:ring-2 focus-visible:ring-state-accent-solid',
|
||||
'data-checked:border-[1.5px] data-checked:border-components-option-card-option-selected-border data-checked:bg-components-option-card-option-selected-bg',
|
||||
'data-disabled:cursor-not-allowed data-disabled:opacity-50 data-disabled:hover:bg-components-option-card-option-bg',
|
||||
'overflow-hidden rounded-xl border border-components-option-card-option-border bg-components-option-card-option-bg transition-colors motion-reduce:transition-none',
|
||||
selected &&
|
||||
'border-[1.5px] border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg',
|
||||
)}
|
||||
>
|
||||
<RadioControl aria-hidden />
|
||||
<span className="flex size-9 shrink-0 items-center justify-center rounded-lg border-[0.5px] border-components-option-card-option-border bg-background-default">
|
||||
<span aria-hidden className={`${icon} size-[18px] text-text-accent`} />
|
||||
</span>
|
||||
<span className="min-w-0 flex-1">
|
||||
<span id={titleId} className="block system-sm-medium text-text-primary">
|
||||
{title}
|
||||
<RadioItem
|
||||
value={value}
|
||||
nativeButton
|
||||
render={<button type="button" />}
|
||||
aria-labelledby={titleId}
|
||||
aria-describedby={disabled ? `${descriptionId} ${unavailableId}` : descriptionId}
|
||||
disabled={disabled}
|
||||
className={cn(
|
||||
'relative flex min-h-16 w-full items-center gap-3 px-4 py-3.5 text-left outline-hidden',
|
||||
'hover:bg-state-base-hover focus-visible:ring-2 focus-visible:ring-state-accent-solid focus-visible:ring-inset',
|
||||
'data-disabled:cursor-not-allowed data-disabled:opacity-50 data-disabled:hover:bg-transparent',
|
||||
)}
|
||||
>
|
||||
<RadioControl aria-hidden />
|
||||
<span className="flex size-9 shrink-0 items-center justify-center rounded-lg border-[0.5px] border-components-option-card-option-border bg-background-default">
|
||||
<span aria-hidden className={`${icon} size-[18px] text-text-accent`} />
|
||||
</span>
|
||||
<span id={descriptionId} className="mt-0.5 block system-xs-regular text-text-tertiary">
|
||||
{description}
|
||||
<span className="min-w-0 flex-1">
|
||||
<span id={titleId} className="block system-sm-medium text-text-primary">
|
||||
{title}
|
||||
</span>
|
||||
<span id={descriptionId} className="mt-0.5 block system-xs-regular text-text-tertiary">
|
||||
{description}
|
||||
</span>
|
||||
</span>
|
||||
</span>
|
||||
{value === 'source' && (
|
||||
<span
|
||||
aria-hidden
|
||||
className="h-4 w-[82px] shrink-0 bg-[url('/images/new-rag/create-knowledge-connectors.svg')] bg-contain bg-center bg-no-repeat"
|
||||
/>
|
||||
)}
|
||||
{disabled && (
|
||||
<span id={unavailableId} className="ml-3 shrink-0 system-xs-medium text-text-disabled">
|
||||
{t(($) => $['cornerLabel.unavailable'])}
|
||||
</span>
|
||||
)}
|
||||
</RadioItem>
|
||||
{value === 'source' && (
|
||||
<span
|
||||
aria-hidden
|
||||
className="h-4 w-[82px] shrink-0 bg-[url('/images/new-rag/create-knowledge-connectors.svg')] bg-contain bg-center bg-no-repeat"
|
||||
/>
|
||||
)}
|
||||
{disabled && (
|
||||
<span id={unavailableId} className="ml-3 shrink-0 system-xs-medium text-text-disabled">
|
||||
{t(($) => $['cornerLabel.unavailable'])}
|
||||
</span>
|
||||
)}
|
||||
</RadioItem>
|
||||
{selected && children}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,675 @@
|
||||
'use client'
|
||||
|
||||
import type {
|
||||
GetKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyResponse,
|
||||
GetKnowledgeSpacesByIdSourceWorkflowsByRunIdPagesResponse,
|
||||
PutKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyData,
|
||||
Source,
|
||||
SourceWorkflowRun,
|
||||
} from '@dify/contracts/knowledge-fs/types.gen'
|
||||
import type { FormEvent } from 'react'
|
||||
import { Button } from '@langgenius/dify-ui/button'
|
||||
import { Checkbox } from '@langgenius/dify-ui/checkbox'
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { useId, useMemo, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useRouter } from '@/next/navigation'
|
||||
import { consoleClient, consoleQuery } from '@/service/client'
|
||||
import { createRequestId } from './request-id'
|
||||
import { newKnowledgeDetailPath } from './routes'
|
||||
|
||||
type PreviewPage = GetKnowledgeSpacesByIdSourceWorkflowsByRunIdPagesResponse['items'][number]
|
||||
type SyncPolicy = GetKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyResponse
|
||||
type SyncMode = SyncPolicy['mode']
|
||||
type SyncPolicyBody = PutKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyData['body']
|
||||
|
||||
const MIN_CUSTOM_INTERVAL_HOURS = 1
|
||||
const MAX_CUSTOM_INTERVAL_HOURS = 720
|
||||
const MAX_SELECTED_PAGES = 200
|
||||
const IMPORT_POLL_INTERVAL_MS = 1_000
|
||||
const IMPORT_POLL_ATTEMPTS = 120
|
||||
const SUCCESSFUL_IMPORT_STATES = new Set(['complete', 'completed', 'success', 'succeeded'])
|
||||
const TERMINAL_IMPORT_STATES = new Set([
|
||||
...SUCCESSFUL_IMPORT_STATES,
|
||||
'canceled',
|
||||
'cancelled',
|
||||
'error',
|
||||
'exhausted',
|
||||
'failed',
|
||||
'superseded',
|
||||
'timed_out',
|
||||
'timeout',
|
||||
'zero_results',
|
||||
])
|
||||
|
||||
type PageSkipReason = 'failed' | 'off-domain'
|
||||
|
||||
function requestStatus(error: unknown) {
|
||||
if (error instanceof Response) return error.status
|
||||
if (!error || typeof error !== 'object') return undefined
|
||||
if ('status' in error && typeof error.status === 'number') return error.status
|
||||
if ('data' in error && error.data && typeof error.data === 'object' && 'status' in error.data)
|
||||
return typeof error.data.status === 'number' ? error.data.status : undefined
|
||||
}
|
||||
|
||||
function isDefinitiveRequestFailure(error: unknown) {
|
||||
const status = requestStatus(error)
|
||||
return status !== undefined && [400, 401, 403, 404, 409, 422, 429].includes(status)
|
||||
}
|
||||
|
||||
function normalizedWorkflowState(run: SourceWorkflowRun) {
|
||||
return run.state.trim().toLowerCase().replaceAll('-', '_').replaceAll(' ', '_')
|
||||
}
|
||||
|
||||
function isSuccessfulImport(run: SourceWorkflowRun) {
|
||||
return SUCCESSFUL_IMPORT_STATES.has(normalizedWorkflowState(run))
|
||||
}
|
||||
|
||||
function isTerminalImport(run: SourceWorkflowRun) {
|
||||
return TERMINAL_IMPORT_STATES.has(normalizedWorkflowState(run))
|
||||
}
|
||||
|
||||
async function waitForImportTerminal(
|
||||
knowledgeSpaceId: string,
|
||||
initialRun: SourceWorkflowRun,
|
||||
onWorkflowRun: (run: SourceWorkflowRun) => void,
|
||||
discardRequested: () => boolean,
|
||||
) {
|
||||
let current = initialRun
|
||||
for (let attempt = 0; attempt < IMPORT_POLL_ATTEMPTS; attempt += 1) {
|
||||
if (discardRequested() || isTerminalImport(current)) return current
|
||||
current = await consoleClient.knowledgeFs.getKnowledgeSpacesByIdSourceWorkflowsByRunId({
|
||||
params: { id: knowledgeSpaceId, runId: current.id },
|
||||
})
|
||||
onWorkflowRun(current)
|
||||
if (discardRequested() || isTerminalImport(current)) return current
|
||||
await new Promise((resolve) => setTimeout(resolve, IMPORT_POLL_INTERVAL_MS))
|
||||
}
|
||||
throw new Error('Source import did not reach a terminal state')
|
||||
}
|
||||
|
||||
function pageSkipReason(page: PreviewPage, rootUrl: string): PageSkipReason | undefined {
|
||||
try {
|
||||
const root = new URL(rootUrl)
|
||||
const candidate = new URL(page.sourceUrl)
|
||||
if (
|
||||
!['http:', 'https:'].includes(candidate.protocol) ||
|
||||
candidate.username ||
|
||||
candidate.password
|
||||
)
|
||||
return 'failed'
|
||||
if (candidate.hostname.toLocaleLowerCase() !== root.hostname.toLocaleLowerCase())
|
||||
return 'off-domain'
|
||||
return undefined
|
||||
} catch {
|
||||
return 'failed'
|
||||
}
|
||||
}
|
||||
|
||||
function policyConfiguration(mode: SyncMode, customIntervalHours: number) {
|
||||
if (mode === 'manual') return { enabled: false, mode } as const
|
||||
if (mode === 'custom') {
|
||||
return {
|
||||
customIntervalSeconds: customIntervalHours * 3600,
|
||||
enabled: true,
|
||||
mode,
|
||||
} as const
|
||||
}
|
||||
return { enabled: true, mode } as const
|
||||
}
|
||||
|
||||
function policyMatches(policy: SyncPolicy, desired: ReturnType<typeof policyConfiguration>) {
|
||||
return (
|
||||
policy.revision > 0 &&
|
||||
policy.enabled === desired.enabled &&
|
||||
policy.mode === desired.mode &&
|
||||
(desired.mode !== 'custom' || policy.customIntervalSeconds === desired.customIntervalSeconds)
|
||||
)
|
||||
}
|
||||
|
||||
function initialSyncPolicy(source: Source): SyncPolicy | undefined {
|
||||
if (!source.version) return undefined
|
||||
return {
|
||||
createdAt: source.createdAt,
|
||||
enabled: true,
|
||||
expectedSourceVersion: source.version,
|
||||
id: source.id,
|
||||
knowledgeSpaceId: source.knowledgeSpaceId,
|
||||
mode: 'provider',
|
||||
revision: 0,
|
||||
sourceId: source.id,
|
||||
updatedAt: source.updatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
function PolicyLoading() {
|
||||
const { t } = useTranslation('dataset')
|
||||
return (
|
||||
<div
|
||||
role="status"
|
||||
aria-label={t(($) => $['newKnowledge.loadingSyncPolicy'])}
|
||||
className="space-y-3"
|
||||
>
|
||||
<div className="h-6 w-28 animate-pulse rounded bg-background-section" />
|
||||
<div className="h-9 w-full animate-pulse rounded-lg bg-background-section" />
|
||||
<div className="h-8 w-full animate-pulse rounded-lg bg-background-section" />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function ReadyCrawlSelectionForm({
|
||||
busy,
|
||||
discardRequested,
|
||||
initialSyncMode,
|
||||
knowledgeSpaceId,
|
||||
onCancel,
|
||||
onRecrawl,
|
||||
onSubmissionUncertainChange,
|
||||
onSubmitted,
|
||||
onWorkflowPending,
|
||||
onWorkflowRun,
|
||||
pages,
|
||||
policy,
|
||||
rootUrl,
|
||||
run,
|
||||
source,
|
||||
workflowUncertain,
|
||||
}: {
|
||||
busy: boolean
|
||||
discardRequested: () => boolean
|
||||
initialSyncMode?: SyncMode
|
||||
knowledgeSpaceId: string
|
||||
onCancel: () => void
|
||||
onRecrawl: () => void
|
||||
onSubmissionUncertainChange: (uncertain: boolean) => void
|
||||
onSubmitted: () => Promise<void> | void
|
||||
onWorkflowPending: (request: Promise<SourceWorkflowRun | undefined>) => void
|
||||
onWorkflowRun: (run: SourceWorkflowRun) => void
|
||||
pages: PreviewPage[]
|
||||
policy: SyncPolicy
|
||||
rootUrl: string
|
||||
run: SourceWorkflowRun
|
||||
source: Source
|
||||
workflowUncertain: boolean
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
const router = useRouter()
|
||||
const queryClient = useQueryClient()
|
||||
const customIntervalErrorId = 'crawl-custom-interval-error'
|
||||
const pageDescriptionPrefixId = useId()
|
||||
const pageSkipReasons = useMemo(
|
||||
() => new Map(pages.map((page) => [page.pageId, pageSkipReason(page, rootUrl)])),
|
||||
[pages, rootUrl],
|
||||
)
|
||||
const selectablePages = useMemo(
|
||||
() => pages.filter((page) => !pageSkipReasons.get(page.pageId)),
|
||||
[pageSkipReasons, pages],
|
||||
)
|
||||
const selectablePageIds = useMemo(
|
||||
() => new Set(selectablePages.map((page) => page.pageId)),
|
||||
[selectablePages],
|
||||
)
|
||||
const bulkSelectablePages = selectablePages.slice(0, MAX_SELECTED_PAGES)
|
||||
const [selectedPageIds, setSelectedPageIds] = useState<Set<string>>(() => new Set())
|
||||
const [syncMode, setSyncMode] = useState<SyncMode>(
|
||||
initialSyncMode ?? (policy.enabled ? policy.mode : 'manual'),
|
||||
)
|
||||
const [customIntervalHours, setCustomIntervalHours] = useState<number | ''>(
|
||||
policy.customIntervalSeconds ? policy.customIntervalSeconds / 3600 : MIN_CUSTOM_INTERVAL_HOURS,
|
||||
)
|
||||
const [submitting, setSubmitting] = useState(false)
|
||||
const [submitError, setSubmitError] = useState(false)
|
||||
const [policyUncertain, setPolicyUncertain] = useState(false)
|
||||
const [selectionUncertain, setSelectionUncertain] = useState(false)
|
||||
const policySnapshotRef = useRef(policy)
|
||||
const submissionPendingRef = useRef(false)
|
||||
const selectionRequestRef = useRef<{ fingerprint: string; requestId: string } | undefined>(
|
||||
undefined,
|
||||
)
|
||||
const updatePolicy = useMutation(
|
||||
consoleQuery.knowledgeFs.putKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy.mutationOptions(),
|
||||
)
|
||||
const selectPages = useMutation(
|
||||
consoleQuery.knowledgeFs.postKnowledgeSpacesByIdSourceWorkflowsByRunIdSelection.mutationOptions(),
|
||||
)
|
||||
const allSelected =
|
||||
bulkSelectablePages.length > 0 &&
|
||||
bulkSelectablePages.every((page) => selectedPageIds.has(page.pageId))
|
||||
const someSelected = selectedPageIds.size > 0
|
||||
const selectionAtLimit = selectedPageIds.size >= MAX_SELECTED_PAGES
|
||||
const customIntervalValid =
|
||||
typeof customIntervalHours === 'number' &&
|
||||
Number.isInteger(customIntervalHours) &&
|
||||
customIntervalHours >= MIN_CUSTOM_INTERVAL_HOURS &&
|
||||
customIntervalHours <= MAX_CUSTOM_INTERVAL_HOURS
|
||||
const canSubmit = selectedPageIds.size > 0 && (syncMode !== 'custom' || customIntervalValid)
|
||||
const formBusy = busy || submitting
|
||||
const submissionLocked = formBusy || policyUncertain || selectionUncertain
|
||||
const selectionLocked = submissionLocked || workflowUncertain
|
||||
const updateSelectionUncertain = (uncertain: boolean) => {
|
||||
setSelectionUncertain(uncertain)
|
||||
onSubmissionUncertainChange(uncertain)
|
||||
}
|
||||
|
||||
const togglePage = (pageId: string) => {
|
||||
if (
|
||||
!selectablePageIds.has(pageId) ||
|
||||
submissionPendingRef.current ||
|
||||
policyUncertain ||
|
||||
selectionUncertain
|
||||
)
|
||||
return
|
||||
setSelectedPageIds((current) => {
|
||||
const next = new Set(current)
|
||||
if (next.has(pageId)) next.delete(pageId)
|
||||
else if (next.size < MAX_SELECTED_PAGES) next.add(pageId)
|
||||
return next
|
||||
})
|
||||
setSubmitError(false)
|
||||
}
|
||||
|
||||
const toggleAll = () => {
|
||||
if (submissionPendingRef.current || policyUncertain || selectionUncertain) return
|
||||
setSelectedPageIds(
|
||||
allSelected ? new Set() : new Set(bulkSelectablePages.map((page) => page.pageId)),
|
||||
)
|
||||
setSubmitError(false)
|
||||
}
|
||||
|
||||
const submit = async (event: FormEvent<HTMLFormElement>) => {
|
||||
event.preventDefault()
|
||||
if (!canSubmit || busy || submissionPendingRef.current) return
|
||||
submissionPendingRef.current = true
|
||||
setSubmitting(true)
|
||||
setSubmitError(false)
|
||||
const desiredPolicy = policyConfiguration(
|
||||
syncMode,
|
||||
typeof customIntervalHours === 'number' ? customIntervalHours : 0,
|
||||
)
|
||||
const sortedPageIds = [...selectedPageIds].sort()
|
||||
const fingerprint = JSON.stringify({ pageIds: sortedPageIds, policy: desiredPolicy })
|
||||
if (
|
||||
(policyUncertain || selectionUncertain) &&
|
||||
selectionRequestRef.current?.fingerprint !== fingerprint
|
||||
) {
|
||||
submissionPendingRef.current = false
|
||||
setSubmitting(false)
|
||||
setSubmitError(true)
|
||||
return
|
||||
}
|
||||
if (selectionRequestRef.current?.fingerprint !== fingerprint) {
|
||||
selectionRequestRef.current = {
|
||||
fingerprint,
|
||||
requestId: createRequestId(),
|
||||
}
|
||||
}
|
||||
|
||||
let resolveTransaction!: (run: SourceWorkflowRun | undefined) => void
|
||||
let transactionRun: SourceWorkflowRun | undefined
|
||||
const transaction = new Promise<SourceWorkflowRun | undefined>((resolve) => {
|
||||
resolveTransaction = resolve
|
||||
})
|
||||
onWorkflowPending(transaction)
|
||||
|
||||
try {
|
||||
let currentPolicy = policySnapshotRef.current
|
||||
if (!policyMatches(currentPolicy, desiredPolicy)) {
|
||||
const body: SyncPolicyBody = {
|
||||
...desiredPolicy,
|
||||
expectedRevision: currentPolicy.revision,
|
||||
expectedSourceVersion: currentPolicy.expectedSourceVersion,
|
||||
}
|
||||
try {
|
||||
currentPolicy = await updatePolicy.mutateAsync({
|
||||
body,
|
||||
params: { id: knowledgeSpaceId, sourceId: source.id },
|
||||
})
|
||||
} catch (error) {
|
||||
let reconciled: SyncPolicy
|
||||
try {
|
||||
reconciled =
|
||||
await consoleClient.knowledgeFs.getKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy({
|
||||
params: { id: knowledgeSpaceId, sourceId: source.id },
|
||||
})
|
||||
} catch (reconciliationError) {
|
||||
setPolicyUncertain(!isDefinitiveRequestFailure(error))
|
||||
throw reconciliationError
|
||||
}
|
||||
policySnapshotRef.current = reconciled
|
||||
if (!policyMatches(reconciled, desiredPolicy)) {
|
||||
setPolicyUncertain(!isDefinitiveRequestFailure(error))
|
||||
throw error
|
||||
}
|
||||
currentPolicy = reconciled
|
||||
}
|
||||
policySnapshotRef.current = currentPolicy
|
||||
}
|
||||
setPolicyUncertain(false)
|
||||
|
||||
if (discardRequested()) return
|
||||
|
||||
try {
|
||||
const selectionRequest = selectPages.mutateAsync({
|
||||
body: { pageIds: sortedPageIds },
|
||||
headers: { 'Idempotency-Key': selectionRequestRef.current.requestId },
|
||||
params: { id: knowledgeSpaceId, runId: run.id },
|
||||
})
|
||||
const selectionRun = await selectionRequest
|
||||
transactionRun = selectionRun
|
||||
onWorkflowRun(selectionRun)
|
||||
} catch (error) {
|
||||
updateSelectionUncertain(!isDefinitiveRequestFailure(error))
|
||||
throw error
|
||||
}
|
||||
updateSelectionUncertain(true)
|
||||
if (discardRequested()) return
|
||||
const terminalRun = await waitForImportTerminal(
|
||||
knowledgeSpaceId,
|
||||
transactionRun,
|
||||
onWorkflowRun,
|
||||
discardRequested,
|
||||
)
|
||||
transactionRun = terminalRun
|
||||
if (discardRequested()) return
|
||||
if (!isSuccessfulImport(terminalRun)) {
|
||||
selectionRequestRef.current = undefined
|
||||
updateSelectionUncertain(false)
|
||||
throw new Error('Source import failed')
|
||||
}
|
||||
updateSelectionUncertain(false)
|
||||
await queryClient.invalidateQueries({
|
||||
queryKey: consoleQuery.knowledgeFs.getKnowledgeSpacesByIdSources.key(),
|
||||
})
|
||||
if (discardRequested()) return
|
||||
await onSubmitted()
|
||||
router.push(newKnowledgeDetailPath(knowledgeSpaceId))
|
||||
} catch {
|
||||
setSubmitError(true)
|
||||
} finally {
|
||||
resolveTransaction(transactionRun)
|
||||
submissionPendingRef.current = false
|
||||
setSubmitting(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<form className="space-y-4" onSubmit={(event) => void submit(event)}>
|
||||
<section aria-labelledby="crawl-selection-summary">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<h3
|
||||
id="crawl-selection-summary"
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
className="min-w-0 flex-1 truncate system-xs-semibold text-text-primary"
|
||||
>
|
||||
{t(($) => $['newKnowledge.pagesCrawled'], {
|
||||
count: pages.length,
|
||||
host: new URL(rootUrl).host,
|
||||
})}
|
||||
</h3>
|
||||
<span className="system-xs-regular text-text-tertiary">
|
||||
{t(($) => $['newKnowledge.pagesSelected'], { count: selectedPageIds.size })}
|
||||
</span>
|
||||
{run.progressFailed > 0 && (
|
||||
<span className="system-xs-regular text-text-destructive">
|
||||
{run.progressFailed} {t(($) => $['newKnowledge.skippedFailed'])}
|
||||
</span>
|
||||
)}
|
||||
<Button
|
||||
type="button"
|
||||
variant="tertiary"
|
||||
size="small"
|
||||
disabled={submissionLocked}
|
||||
loading={busy}
|
||||
onClick={onRecrawl}
|
||||
>
|
||||
{t(($) => $['newKnowledge.reCrawl'])}
|
||||
</Button>
|
||||
</div>
|
||||
<div className="mt-3 overflow-hidden rounded-xl border border-divider-regular">
|
||||
<label className="flex cursor-pointer items-center gap-2.5 border-b border-divider-subtle bg-background-section px-3 py-2.5 system-xs-medium text-text-secondary">
|
||||
<Checkbox
|
||||
checked={allSelected}
|
||||
indeterminate={someSelected && !allSelected}
|
||||
onCheckedChange={toggleAll}
|
||||
disabled={!selectablePages.length || selectionLocked}
|
||||
/>
|
||||
{t(($) => $['newKnowledge.selectAll'])}
|
||||
</label>
|
||||
<ul className="max-h-[280px] divide-y divide-divider-subtle overflow-y-auto">
|
||||
{pages.map((page, index) => {
|
||||
const skipReason = pageSkipReasons.get(page.pageId)
|
||||
const selectable = !skipReason
|
||||
const selectionLimitReached =
|
||||
selectable && selectionAtLimit && !selectedPageIds.has(page.pageId)
|
||||
const titleId = `${pageDescriptionPrefixId}-title-${index}`
|
||||
const urlId = `${pageDescriptionPrefixId}-url-${index}`
|
||||
const reasonId = `${pageDescriptionPrefixId}-reason-${index}`
|
||||
return (
|
||||
<li key={page.pageId}>
|
||||
<label className="flex cursor-pointer items-center gap-2.5 px-3 py-2.5">
|
||||
<Checkbox
|
||||
checked={selectedPageIds.has(page.pageId)}
|
||||
disabled={!selectable || selectionLimitReached || selectionLocked}
|
||||
aria-labelledby={titleId}
|
||||
aria-describedby={`${urlId}${skipReason || selectionLimitReached ? ` ${reasonId}` : ''}`}
|
||||
onCheckedChange={() => togglePage(page.pageId)}
|
||||
/>
|
||||
<span className="min-w-0 flex-1">
|
||||
<span
|
||||
id={titleId}
|
||||
className="block truncate system-xs-medium text-text-primary"
|
||||
>
|
||||
{page.title || page.sourceUrl}
|
||||
</span>
|
||||
<span
|
||||
id={urlId}
|
||||
className="block truncate system-2xs-regular text-text-tertiary"
|
||||
>
|
||||
{page.sourceUrl}
|
||||
</span>
|
||||
</span>
|
||||
{(!selectable || selectionLimitReached) && (
|
||||
<span id={reasonId} className="shrink-0 system-xs-medium text-text-tertiary">
|
||||
{selectionLimitReached
|
||||
? `${t(($) => $['newKnowledge.maxPages'])}: ${MAX_SELECTED_PAGES}`
|
||||
: skipReason === 'off-domain'
|
||||
? t(($) => $['newKnowledge.skippedOffDomain'])
|
||||
: t(($) => $['newKnowledge.skippedFailed'])}
|
||||
</span>
|
||||
)}
|
||||
</label>
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ul>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<fieldset disabled={selectionLocked}>
|
||||
<label className="block">
|
||||
<span className="system-xs-medium text-text-secondary">
|
||||
{t(($) => $['newKnowledge.syncPolicy'])}
|
||||
</span>
|
||||
<select
|
||||
name="syncMode"
|
||||
value={syncMode}
|
||||
onChange={(event) => {
|
||||
setSyncMode(event.target.value as SyncMode)
|
||||
setSubmitError(false)
|
||||
}}
|
||||
className="mt-1.5 h-9 w-full rounded-lg border-0 bg-components-input-bg-normal px-3 system-sm-regular text-text-primary outline-hidden focus:ring-2 focus:ring-state-accent-solid sm:w-72"
|
||||
>
|
||||
<option value="provider">{t(($) => $['newKnowledge.syncPolicyProvider'])}</option>
|
||||
<option value="manual">{t(($) => $['newKnowledge.syncPolicyManual'])}</option>
|
||||
<option value="interval">{t(($) => $['newKnowledge.syncPolicyDaily'])}</option>
|
||||
<option value="custom">{t(($) => $['newKnowledge.syncPolicyCustom'])}</option>
|
||||
</select>
|
||||
</label>
|
||||
{syncMode === 'custom' && (
|
||||
<label className="mt-3 block sm:w-72">
|
||||
<span className="system-xs-medium text-text-secondary">
|
||||
{t(($) => $['newKnowledge.customIntervalHours'])}
|
||||
</span>
|
||||
<input
|
||||
type="number"
|
||||
name="customIntervalHours"
|
||||
min={MIN_CUSTOM_INTERVAL_HOURS}
|
||||
max={MAX_CUSTOM_INTERVAL_HOURS}
|
||||
step={1}
|
||||
value={customIntervalHours}
|
||||
aria-invalid={!customIntervalValid}
|
||||
aria-describedby={!customIntervalValid ? customIntervalErrorId : undefined}
|
||||
onChange={(event) => {
|
||||
setCustomIntervalHours(
|
||||
Number.isFinite(event.target.valueAsNumber) ? event.target.valueAsNumber : '',
|
||||
)
|
||||
setSubmitError(false)
|
||||
}}
|
||||
className="mt-1.5 h-9 w-full rounded-lg border-0 bg-components-input-bg-normal px-3 system-sm-regular text-text-primary outline-hidden focus:ring-2 focus:ring-state-accent-solid"
|
||||
/>
|
||||
{!customIntervalValid && (
|
||||
<span
|
||||
id={customIntervalErrorId}
|
||||
className="mt-1 block system-xs-regular text-text-destructive"
|
||||
>
|
||||
{t(($) => $['newKnowledge.customIntervalInvalid'])}
|
||||
</span>
|
||||
)}
|
||||
</label>
|
||||
)}
|
||||
</fieldset>
|
||||
|
||||
{submitError && (
|
||||
<p role="alert" className="system-xs-regular text-text-destructive">
|
||||
{t(($) => $['newKnowledge.addSourceFailed'])}
|
||||
</p>
|
||||
)}
|
||||
<div className="flex justify-end gap-2 border-t border-divider-subtle pt-5">
|
||||
<Button type="button" onClick={onCancel}>
|
||||
{t(($) => $['newKnowledge.cancelAddSource'])}
|
||||
</Button>
|
||||
<Button
|
||||
type="submit"
|
||||
variant="primary"
|
||||
disabled={!canSubmit || formBusy || workflowUncertain}
|
||||
loading={submitting}
|
||||
aria-describedby={!selectedPageIds.size ? 'add-source-selection-requirement' : undefined}
|
||||
>
|
||||
{t(($) => $['newKnowledge.addSource'])}
|
||||
</Button>
|
||||
{!selectedPageIds.size && (
|
||||
<span id="add-source-selection-requirement" className="sr-only">
|
||||
{t(($) => $['newKnowledge.addSourceRequiresSelection'])}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</form>
|
||||
)
|
||||
}
|
||||
|
||||
export function CrawlSelectionForm({
|
||||
busy = false,
|
||||
discardRequested,
|
||||
initialSyncMode,
|
||||
knowledgeSpaceId,
|
||||
onCancel,
|
||||
onRecrawl,
|
||||
onSubmissionUncertainChange,
|
||||
onSubmitted,
|
||||
onWorkflowPending,
|
||||
onWorkflowRun,
|
||||
pages,
|
||||
rootUrl,
|
||||
run,
|
||||
source,
|
||||
workflowUncertain = false,
|
||||
}: {
|
||||
busy?: boolean
|
||||
discardRequested: () => boolean
|
||||
initialSyncMode?: SyncMode
|
||||
knowledgeSpaceId: string
|
||||
onCancel: () => void
|
||||
onRecrawl: () => void
|
||||
onSubmissionUncertainChange: (uncertain: boolean) => void
|
||||
onSubmitted: () => Promise<void> | void
|
||||
onWorkflowPending: (request: Promise<SourceWorkflowRun | undefined>) => void
|
||||
onWorkflowRun: (run: SourceWorkflowRun) => void
|
||||
pages: PreviewPage[]
|
||||
rootUrl: string
|
||||
run: SourceWorkflowRun
|
||||
source: Source
|
||||
workflowUncertain?: boolean
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
const policyQuery = useQuery(
|
||||
consoleQuery.knowledgeFs.getKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy.queryOptions({
|
||||
context: { silent: true },
|
||||
input: { params: { id: knowledgeSpaceId, sourceId: source.id } },
|
||||
retry: false,
|
||||
}),
|
||||
)
|
||||
const policy =
|
||||
policyQuery.data ??
|
||||
(requestStatus(policyQuery.error) === 404 ? initialSyncPolicy(source) : undefined)
|
||||
|
||||
if (policyQuery.isPending) {
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<PolicyLoading />
|
||||
<div className="flex justify-end gap-2 border-t border-divider-subtle pt-5">
|
||||
<Button type="button" onClick={onCancel}>
|
||||
{t(($) => $['newKnowledge.cancelAddSource'])}
|
||||
</Button>
|
||||
<Button type="button" variant="primary" disabled>
|
||||
{t(($) => $['newKnowledge.addSource'])}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
if (!policy) {
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div role="alert" className="rounded-xl border border-divider-regular p-4">
|
||||
<p className="system-xs-regular text-text-destructive">
|
||||
{t(($) => $['newKnowledge.syncPolicyLoadFailed'])}
|
||||
</p>
|
||||
<Button className="mt-3" onClick={() => void policyQuery.refetch()}>
|
||||
{t(($) => $['newKnowledge.retrySyncPolicy'])}
|
||||
</Button>
|
||||
</div>
|
||||
<div className="flex justify-end gap-2 border-t border-divider-subtle pt-5">
|
||||
<Button type="button" onClick={onCancel}>
|
||||
{t(($) => $['newKnowledge.cancelAddSource'])}
|
||||
</Button>
|
||||
<Button type="button" variant="primary" disabled>
|
||||
{t(($) => $['newKnowledge.addSource'])}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<ReadyCrawlSelectionForm
|
||||
key={`${run.id}:${policy.revision}`}
|
||||
busy={busy}
|
||||
discardRequested={discardRequested}
|
||||
initialSyncMode={initialSyncMode}
|
||||
knowledgeSpaceId={knowledgeSpaceId}
|
||||
onCancel={onCancel}
|
||||
onRecrawl={onRecrawl}
|
||||
onSubmissionUncertainChange={onSubmissionUncertainChange}
|
||||
onSubmitted={onSubmitted}
|
||||
onWorkflowPending={onWorkflowPending}
|
||||
onWorkflowRun={onWorkflowRun}
|
||||
pages={pages}
|
||||
policy={policy}
|
||||
rootUrl={rootUrl}
|
||||
run={run}
|
||||
source={source}
|
||||
workflowUncertain={workflowUncertain}
|
||||
/>
|
||||
)
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
import type { KnowledgeSpaceCreationResponse } from '@dify/contracts/knowledge-fs/types.gen'
|
||||
import type { CreateKnowledgeExitReason } from './components/create-knowledge-exit-dialog'
|
||||
import type { KnowledgeVisibility } from './create-knowledge-workflow'
|
||||
import type { NewKnowledgeSourceDraft, NewKnowledgeStartMode } from './routes'
|
||||
import { Button } from '@langgenius/dify-ui/button'
|
||||
import {
|
||||
Dialog,
|
||||
@@ -30,12 +31,13 @@ import {
|
||||
SelectTrigger,
|
||||
} from '@langgenius/dify-ui/select'
|
||||
import { Textarea } from '@langgenius/dify-ui/textarea'
|
||||
import { toast } from '@langgenius/dify-ui/toast'
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import { useAtomValue } from 'jotai'
|
||||
import { useCallback, useEffect, useId, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { workspacePermissionKeysAtom } from '@/context/permission-state'
|
||||
import { useRouter } from '@/next/navigation'
|
||||
import { useRouter, useSearchParams } from '@/next/navigation'
|
||||
import { consoleQuery } from '@/service/client'
|
||||
import { DatasetACLPermission, hasPermission } from '@/utils/permission'
|
||||
import { KnowledgeIllustration, StartMode } from './components/create-knowledge-dialog-parts'
|
||||
@@ -47,12 +49,26 @@ import {
|
||||
KnowledgeCreationError,
|
||||
NAME_MAX_LENGTH,
|
||||
} from './create-knowledge-workflow'
|
||||
import { newKnowledgeDetailPath, newKnowledgeListPath } from './routes'
|
||||
import { CreateSourceSetup } from './create-source-setup'
|
||||
import { createRequestId } from './request-id'
|
||||
import {
|
||||
createNewKnowledgeSourceDraft,
|
||||
isValidWebsiteSourceDraft,
|
||||
newKnowledgeAddSourcePath,
|
||||
newKnowledgeDetailPath,
|
||||
newKnowledgeListPath,
|
||||
newKnowledgeSourceDraftStorageKey,
|
||||
} from './routes'
|
||||
|
||||
function normalizeStartMode(value: string | null): NewKnowledgeStartMode {
|
||||
return value === 'source' ? value : 'empty'
|
||||
}
|
||||
|
||||
export function CreateKnowledgePage() {
|
||||
const { t } = useTranslation('dataset')
|
||||
const { t: tCommon } = useTranslation('common')
|
||||
const router = useRouter()
|
||||
const searchParams = useSearchParams()
|
||||
const queryClient = useQueryClient()
|
||||
const dialogTitleId = useId()
|
||||
const permissionDescriptionId = useId()
|
||||
@@ -65,6 +81,14 @@ export function CreateKnowledgePage() {
|
||||
const [name, setName] = useState('')
|
||||
const [description, setDescription] = useState('')
|
||||
const [visibility, setVisibility] = useState<KnowledgeVisibility>(defaultVisibility)
|
||||
const initialStartMode = normalizeStartMode(searchParams.get('start'))
|
||||
const [startMode, setStartMode] = useState<NewKnowledgeStartMode>(initialStartMode)
|
||||
const [sourceDraft, setSourceDraft] = useState<NewKnowledgeSourceDraft>(() =>
|
||||
createNewKnowledgeSourceDraft('websiteCrawl'),
|
||||
)
|
||||
const sourceDraftsRef = useRef<
|
||||
Partial<Record<NewKnowledgeSourceDraft['sourceType'], NewKnowledgeSourceDraft>>
|
||||
>({})
|
||||
const [createdKnowledge, setCreatedKnowledge] = useState<KnowledgeSpaceCreationResponse>()
|
||||
const [submissionLocked, setSubmissionLocked] = useState(false)
|
||||
const [exitReason, setExitReason] = useState<CreateKnowledgeExitReason | null>(null)
|
||||
@@ -73,8 +97,25 @@ export function CreateKnowledgePage() {
|
||||
const browserBackExitRef = useRef(false)
|
||||
const pendingNavigationRef = useRef<string | undefined>(undefined)
|
||||
const createMutation = useMutation({ mutationFn: createKnowledge })
|
||||
const sourceSubmissionBlocked =
|
||||
startMode === 'source' &&
|
||||
(sourceDraft.sourceType === 'websiteCrawl'
|
||||
? !isValidWebsiteSourceDraft(sourceDraft)
|
||||
: !sourceDraft.sourceName.trim())
|
||||
const sourceDraftChanged = Object.values({
|
||||
...sourceDraftsRef.current,
|
||||
[sourceDraft.sourceType]: sourceDraft,
|
||||
}).some(
|
||||
(draft) =>
|
||||
JSON.stringify(draft) !== JSON.stringify(createNewKnowledgeSourceDraft(draft.sourceType)),
|
||||
)
|
||||
const hasUnsavedChanges = Boolean(
|
||||
name || description || visibility !== defaultVisibility || createdKnowledge,
|
||||
name ||
|
||||
description ||
|
||||
visibility !== defaultVisibility ||
|
||||
startMode !== initialStartMode ||
|
||||
sourceDraftChanged ||
|
||||
createdKnowledge,
|
||||
)
|
||||
|
||||
const armHistoryGuard = useCallback(() => {
|
||||
@@ -153,7 +194,7 @@ export function CreateKnowledgePage() {
|
||||
setExitReason('partial')
|
||||
return
|
||||
}
|
||||
if (name || description || visibility !== defaultVisibility) {
|
||||
if (hasUnsavedChanges) {
|
||||
setExitReason('discard')
|
||||
return
|
||||
}
|
||||
@@ -181,13 +222,13 @@ export function CreateKnowledgePage() {
|
||||
}
|
||||
|
||||
const handleSubmit = async () => {
|
||||
if (createMutation.isPending) return
|
||||
if (createMutation.isPending || sourceSubmissionBlocked) return
|
||||
|
||||
const normalizedName = name.trim()
|
||||
const normalizedDescription = description.trim()
|
||||
if (!normalizedName) return
|
||||
|
||||
idempotencyKeyRef.current ??= globalThis.crypto.randomUUID()
|
||||
idempotencyKeyRef.current ??= createRequestId()
|
||||
setSubmissionLocked(true)
|
||||
try {
|
||||
const created = await createMutation.mutateAsync({
|
||||
@@ -203,6 +244,21 @@ export function CreateKnowledgePage() {
|
||||
},
|
||||
visibility,
|
||||
})
|
||||
if (startMode === 'source') {
|
||||
try {
|
||||
const sourceDraftKey = createRequestId()
|
||||
globalThis.sessionStorage.setItem(
|
||||
newKnowledgeSourceDraftStorageKey(sourceDraftKey),
|
||||
JSON.stringify(sourceDraft),
|
||||
)
|
||||
replaceAfterHistoryGuard(
|
||||
newKnowledgeAddSourcePath(created.id, sourceDraft.sourceType, sourceDraftKey),
|
||||
)
|
||||
} catch {
|
||||
toast.error(t(($) => $['newKnowledge.addSourceFailed']))
|
||||
}
|
||||
return
|
||||
}
|
||||
replaceAfterHistoryGuard(newKnowledgeDetailPath(created.id))
|
||||
} catch (error) {
|
||||
if (error instanceof KnowledgeCreationError && error.createdKnowledge)
|
||||
@@ -246,7 +302,7 @@ export function CreateKnowledgePage() {
|
||||
<div className="flex min-h-0 min-w-0 flex-col items-end border-divider-subtle xl:border-r">
|
||||
<div className="min-h-6 w-full max-w-[760px] flex-1 [@media(max-height:850px)]:h-6 [@media(max-height:850px)]:flex-none" />
|
||||
<Form
|
||||
className="flex w-full max-w-[760px] shrink-0 flex-col [@media(max-height:850px)]:min-h-0 [@media(max-height:850px)]:flex-1"
|
||||
className="flex max-h-full min-h-0 w-full max-w-[760px] flex-col"
|
||||
onFormSubmit={handleSubmit}
|
||||
>
|
||||
<header className="shrink-0 px-6 pt-2 pb-6 sm:px-10">
|
||||
@@ -255,7 +311,7 @@ export function CreateKnowledgePage() {
|
||||
</DialogTitle>
|
||||
</header>
|
||||
|
||||
<div className="flex min-h-0 flex-col gap-4 px-6 sm:px-10 [@media(max-height:850px)]:flex-1 [@media(max-height:850px)]:overflow-y-auto">
|
||||
<div className="flex min-h-0 flex-col gap-4 overflow-y-auto px-6 sm:px-10">
|
||||
<div className="space-y-4">
|
||||
<Field
|
||||
name="name"
|
||||
@@ -361,25 +417,48 @@ export function CreateKnowledgePage() {
|
||||
<p className="pb-0.5 body-xs-regular text-text-tertiary">
|
||||
{t(($) => $['newKnowledge.startWithHelp'])}
|
||||
</p>
|
||||
<RadioGroup
|
||||
value="empty"
|
||||
<RadioGroup<NewKnowledgeStartMode>
|
||||
value={startMode}
|
||||
aria-label={t(($) => $['newKnowledge.startWith'])}
|
||||
className="mt-2 flex-col items-stretch gap-2"
|
||||
disabled={createMutation.isPending}
|
||||
disabled={submissionLocked}
|
||||
onValueChange={(value) => {
|
||||
setStartMode(value)
|
||||
resetUnsubmittedError()
|
||||
}}
|
||||
>
|
||||
<StartMode
|
||||
value="empty"
|
||||
icon="i-ri-folder-6-line"
|
||||
selected={startMode === 'empty'}
|
||||
title={t(($) => $['newKnowledge.startEmpty'])}
|
||||
description={t(($) => $['newKnowledge.startEmptyDescription'])}
|
||||
/>
|
||||
<StartMode
|
||||
disabled
|
||||
value="source"
|
||||
icon="i-custom-vender-solid-development-api-connection-mod"
|
||||
selected={startMode === 'source'}
|
||||
title={t(($) => $['newKnowledge.connectSource'])}
|
||||
description={t(($) => $['newKnowledge.connectSourceDescription'])}
|
||||
/>
|
||||
>
|
||||
<CreateSourceSetup
|
||||
disabled={submissionLocked}
|
||||
draft={sourceDraft}
|
||||
onDraftChange={(value) => {
|
||||
sourceDraftsRef.current[value.sourceType] = value
|
||||
setSourceDraft(value)
|
||||
resetUnsubmittedError()
|
||||
}}
|
||||
onSourceTypeChange={(value) => {
|
||||
sourceDraftsRef.current[sourceDraft.sourceType] = sourceDraft
|
||||
const nextDraft =
|
||||
sourceDraftsRef.current[value] ?? createNewKnowledgeSourceDraft(value)
|
||||
sourceDraftsRef.current[value] = nextDraft
|
||||
setSourceDraft(nextDraft)
|
||||
resetUnsubmittedError()
|
||||
}}
|
||||
/>
|
||||
</StartMode>
|
||||
<StartMode
|
||||
disabled
|
||||
value="upload"
|
||||
@@ -410,7 +489,12 @@ export function CreateKnowledgePage() {
|
||||
<Button type="button" disabled={createMutation.isPending} onClick={requestClose}>
|
||||
{tCommon(($) => $['operation.cancel'])}
|
||||
</Button>
|
||||
<Button type="submit" variant="primary" loading={createMutation.isPending}>
|
||||
<Button
|
||||
type="submit"
|
||||
variant="primary"
|
||||
loading={createMutation.isPending}
|
||||
disabled={sourceSubmissionBlocked}
|
||||
>
|
||||
{t(($) => $['newKnowledge.createTitle'])}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,391 @@
|
||||
'use client'
|
||||
|
||||
import type {
|
||||
NewKnowledgeOnlineDocumentsProvider,
|
||||
NewKnowledgeOnlineDriveProvider,
|
||||
NewKnowledgeSourceDraft,
|
||||
NewKnowledgeWebsiteProvider,
|
||||
} from './routes'
|
||||
import { Button } from '@langgenius/dify-ui/button'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import {
|
||||
isValidWebsiteSourceDraft,
|
||||
NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH,
|
||||
NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH,
|
||||
} from './routes'
|
||||
|
||||
const sourceTypes = [
|
||||
{ icon: 'i-ri-global-line', value: 'websiteCrawl' },
|
||||
{ icon: 'i-ri-file-text-line', value: 'onlineDocuments' },
|
||||
{ icon: 'i-ri-hard-drive-3-line', value: 'onlineDrive' },
|
||||
] as const
|
||||
|
||||
const providers = {
|
||||
onlineDocuments: [
|
||||
{ icon: 'i-custom-public-common-notion', label: 'Notion' },
|
||||
{ icon: 'i-ri-file-text-line', label: 'Google Docs' },
|
||||
{ icon: 'i-ri-links-line', label: 'Confluence' },
|
||||
],
|
||||
onlineDrive: [
|
||||
{ icon: 'i-custom-public-common-google-drive', label: 'Google Drive' },
|
||||
{ icon: 'i-ri-cloud-line', label: 'OneDrive' },
|
||||
{ icon: 'i-ri-box-3-line', label: 'Amazon S3' },
|
||||
],
|
||||
websiteCrawl: [
|
||||
{ icon: 'i-ri-fire-fill text-orange-500', label: 'Firecrawl', available: true },
|
||||
{ icon: 'i-custom-public-llm-jina', label: 'Jina Reader' },
|
||||
{ icon: 'i-ri-water-flash-line', label: 'WaterCrawl' },
|
||||
],
|
||||
} as const
|
||||
|
||||
function ConnectedSourceConfiguration({
|
||||
disabled,
|
||||
draft,
|
||||
onDraftChange,
|
||||
}: {
|
||||
disabled: boolean
|
||||
draft: NewKnowledgeSourceDraft
|
||||
onDraftChange: (draft: NewKnowledgeSourceDraft) => void
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
|
||||
return (
|
||||
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
|
||||
<label className="block system-xs-medium text-text-secondary">
|
||||
{t(($) => $['newKnowledge.sourceName'])}
|
||||
<span aria-hidden className="ml-0.5 text-text-destructive">
|
||||
*
|
||||
</span>
|
||||
<input
|
||||
type="text"
|
||||
autoComplete="off"
|
||||
name="sourceName"
|
||||
disabled={disabled}
|
||||
maxLength={NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH}
|
||||
value={draft.sourceName}
|
||||
placeholder={t(($) => $['newKnowledge.sourceNamePlaceholder'])}
|
||||
className="mt-1.5 h-9 w-full rounded-lg border-0 bg-components-input-bg-normal px-3 system-sm-regular text-text-primary outline-hidden placeholder:text-text-quaternary focus:ring-2 focus:ring-state-accent-solid disabled:text-text-disabled"
|
||||
onChange={(event) => onDraftChange({ ...draft, sourceName: event.target.value })}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === 'Enter') event.preventDefault()
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
<label className="block system-xs-medium text-text-secondary">
|
||||
{t(($) => $['newKnowledge.syncPolicy'])}
|
||||
<select
|
||||
name="syncPolicy"
|
||||
disabled={disabled}
|
||||
value={draft.syncPolicy}
|
||||
className="mt-1.5 h-9 w-full rounded-lg border-0 bg-components-input-bg-normal px-3 system-sm-regular text-text-primary outline-hidden focus:ring-2 focus:ring-state-accent-solid disabled:text-text-disabled"
|
||||
onChange={(event) =>
|
||||
onDraftChange({
|
||||
...draft,
|
||||
syncPolicy: event.target.value as NewKnowledgeSourceDraft['syncPolicy'],
|
||||
})
|
||||
}
|
||||
>
|
||||
<option value="provider">{t(($) => $['newKnowledge.syncPolicyProvider'])}</option>
|
||||
<option value="daily">{t(($) => $['newKnowledge.syncPolicyDaily'])}</option>
|
||||
<option value="manual">{t(($) => $['newKnowledge.syncPolicyManual'])}</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export function CreateSourceSetup({
|
||||
disabled,
|
||||
draft,
|
||||
onDraftChange,
|
||||
onSourceTypeChange,
|
||||
}: {
|
||||
disabled: boolean
|
||||
draft: NewKnowledgeSourceDraft
|
||||
onDraftChange: (draft: NewKnowledgeSourceDraft) => void
|
||||
onSourceTypeChange: (sourceType: NewKnowledgeSourceDraft['sourceType']) => void
|
||||
}) {
|
||||
const { t } = useTranslation('dataset')
|
||||
const { t: tCreation } = useTranslation('datasetCreation')
|
||||
const [optionsExpanded, setOptionsExpanded] = useState(false)
|
||||
const [backendBoundaryVisible, setBackendBoundaryVisible] = useState(false)
|
||||
const sourceType = draft.sourceType
|
||||
const availableProviders = providers[sourceType]
|
||||
const activeProvider = availableProviders.some((provider) => provider.label === draft.provider)
|
||||
? draft.provider
|
||||
: availableProviders[0].label
|
||||
const previewReady = draft.sourceType === 'websiteCrawl' && isValidWebsiteSourceDraft(draft)
|
||||
const showBackendBoundary = () => setBackendBoundaryVisible(true)
|
||||
const updateDraft = (nextDraft: NewKnowledgeSourceDraft) => {
|
||||
onDraftChange(nextDraft)
|
||||
setBackendBoundaryVisible(false)
|
||||
}
|
||||
const selectProvider = (provider: string) => {
|
||||
if (draft.sourceType === 'onlineDocuments')
|
||||
updateDraft({ ...draft, provider: provider as NewKnowledgeOnlineDocumentsProvider })
|
||||
else if (draft.sourceType === 'onlineDrive')
|
||||
updateDraft({ ...draft, provider: provider as NewKnowledgeOnlineDriveProvider })
|
||||
else updateDraft({ ...draft, provider: provider as NewKnowledgeWebsiteProvider })
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="mx-4 mb-4 space-y-4 border-t border-divider-subtle pt-4">
|
||||
<fieldset disabled={disabled}>
|
||||
<legend className="mb-1.5 system-xs-medium text-text-secondary">
|
||||
{t(($) => $['newKnowledge.sourceTypeLabel'])}
|
||||
</legend>
|
||||
<div className="grid grid-cols-1 gap-0.5 rounded-lg bg-background-default p-0.5 sm:grid-cols-3">
|
||||
{sourceTypes.map((option) => (
|
||||
<label
|
||||
key={option.value}
|
||||
className={cn(
|
||||
'relative flex min-h-8 items-center justify-center gap-1.5 rounded-md px-2 system-xs-medium outline-hidden has-focus-visible:ring-2 has-focus-visible:ring-state-accent-solid',
|
||||
sourceType === option.value
|
||||
? 'bg-components-option-card-option-selected-bg text-text-primary shadow-xs'
|
||||
: 'cursor-pointer text-text-tertiary hover:text-text-secondary',
|
||||
disabled && 'cursor-not-allowed opacity-60',
|
||||
)}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name="create-source-type"
|
||||
value={option.value}
|
||||
checked={sourceType === option.value}
|
||||
onChange={() => {
|
||||
setBackendBoundaryVisible(false)
|
||||
onSourceTypeChange(option.value)
|
||||
}}
|
||||
className="sr-only"
|
||||
/>
|
||||
<span aria-hidden className={`${option.icon} size-4`} />
|
||||
{t(($) => $[`newKnowledge.${option.value}`])}
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
</fieldset>
|
||||
|
||||
<fieldset disabled={disabled}>
|
||||
<legend className="sr-only">{tCreation(($) => $['stepOne.website.chooseProvider'])}</legend>
|
||||
<div className="mb-1.5 flex items-center justify-between gap-3">
|
||||
<span className="system-xs-medium text-text-secondary">
|
||||
{tCreation(($) => $['stepOne.website.chooseProvider'])}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
disabled={disabled}
|
||||
className="rounded-sm system-xs-medium text-text-accent outline-hidden hover:text-text-accent-secondary focus-visible:ring-2 focus-visible:ring-state-accent-solid disabled:cursor-not-allowed disabled:text-text-disabled"
|
||||
onClick={showBackendBoundary}
|
||||
>
|
||||
{t(($) => $['newKnowledge.moreProviders'])}
|
||||
</button>
|
||||
</div>
|
||||
<div className={cn('grid grid-cols-2 gap-2', 'sm:grid-cols-3')}>
|
||||
{providers[sourceType].map((provider) => {
|
||||
return (
|
||||
<label
|
||||
key={provider.label}
|
||||
className={cn(
|
||||
'flex min-h-10 items-center gap-2 rounded-lg border bg-background-default px-3 system-xs-medium outline-hidden has-focus-visible:ring-2 has-focus-visible:ring-state-accent-solid',
|
||||
activeProvider === provider.label
|
||||
? 'border-components-option-card-option-selected-border text-text-primary'
|
||||
: 'cursor-pointer border-divider-subtle text-text-secondary hover:bg-state-base-hover',
|
||||
disabled && 'cursor-not-allowed opacity-60',
|
||||
)}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name="create-source-provider"
|
||||
value={provider.label}
|
||||
checked={activeProvider === provider.label}
|
||||
disabled={disabled}
|
||||
onChange={() => selectProvider(provider.label)}
|
||||
className="sr-only"
|
||||
/>
|
||||
<span aria-hidden className={`${provider.icon} size-4 shrink-0`} />
|
||||
<span className="truncate">{provider.label}</span>
|
||||
</label>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</fieldset>
|
||||
|
||||
{draft.sourceType === 'websiteCrawl' && (
|
||||
<div className="space-y-4">
|
||||
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
|
||||
<label className="block system-xs-medium text-text-secondary">
|
||||
{t(($) => $['newKnowledge.rootUrl'])}
|
||||
<input
|
||||
type="url"
|
||||
inputMode="url"
|
||||
autoComplete="off"
|
||||
name="rootUrl"
|
||||
disabled={disabled}
|
||||
maxLength={NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH}
|
||||
value={draft.rootUrl}
|
||||
placeholder={t(($) => $['newKnowledge.rootUrlPlaceholder'])}
|
||||
className="mt-1.5 h-9 w-full rounded-lg border-0 bg-components-input-bg-normal px-3 system-sm-regular text-text-primary outline-hidden placeholder:text-text-quaternary focus:ring-2 focus:ring-state-accent-solid disabled:text-text-disabled"
|
||||
onChange={(event) => {
|
||||
updateDraft({ ...draft, rootUrl: event.target.value })
|
||||
}}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === 'Enter') event.preventDefault()
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
<label className="block system-xs-medium text-text-secondary">
|
||||
{t(($) => $['newKnowledge.sourceName'])}
|
||||
<input
|
||||
type="text"
|
||||
autoComplete="off"
|
||||
name="sourceName"
|
||||
disabled={disabled}
|
||||
maxLength={NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH}
|
||||
value={draft.sourceName}
|
||||
placeholder={t(($) => $['newKnowledge.sourceNamePlaceholder'])}
|
||||
className="mt-1.5 h-9 w-full rounded-lg border-0 bg-components-input-bg-normal px-3 system-sm-regular text-text-primary outline-hidden placeholder:text-text-quaternary focus:ring-2 focus:ring-state-accent-solid disabled:text-text-disabled"
|
||||
onChange={(event) => {
|
||||
updateDraft({ ...draft, sourceName: event.target.value })
|
||||
}}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === 'Enter') event.preventDefault()
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
<div className="overflow-hidden rounded-lg bg-background-section">
|
||||
<button
|
||||
type="button"
|
||||
aria-label={t(($) => $['newKnowledge.crawlOptions'])}
|
||||
aria-expanded={optionsExpanded}
|
||||
disabled={disabled}
|
||||
className="flex min-h-9 w-full items-center gap-2 px-3 text-left system-xs-medium text-text-secondary outline-hidden hover:bg-state-base-hover focus-visible:ring-2 focus-visible:ring-state-accent-solid disabled:cursor-not-allowed"
|
||||
onClick={() => setOptionsExpanded((value) => !value)}
|
||||
>
|
||||
<span
|
||||
aria-hidden
|
||||
className={cn(
|
||||
'size-4 transition-transform motion-reduce:transition-none',
|
||||
optionsExpanded ? 'i-ri-arrow-down-s-line' : 'i-ri-arrow-right-s-line',
|
||||
)}
|
||||
/>
|
||||
{t(($) => $['newKnowledge.crawlOptions'])}
|
||||
{!optionsExpanded && (
|
||||
<span className="ml-auto system-xs-regular text-text-tertiary">
|
||||
{t(($) => $['newKnowledge.usingDefaults'])}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
{optionsExpanded && (
|
||||
<fieldset
|
||||
disabled={disabled}
|
||||
className="grid grid-cols-1 gap-3 px-3 pb-3 sm:grid-cols-2"
|
||||
>
|
||||
<label className="flex items-center gap-2 system-xs-regular text-text-secondary">
|
||||
<input
|
||||
type="checkbox"
|
||||
name="includeSubpages"
|
||||
checked={draft.includeSubpages}
|
||||
onChange={(event) =>
|
||||
updateDraft({ ...draft, includeSubpages: event.target.checked })
|
||||
}
|
||||
/>
|
||||
{t(($) => $['newKnowledge.includeSubpages'])}
|
||||
</label>
|
||||
<label className="system-xs-medium text-text-secondary">
|
||||
{t(($) => $['newKnowledge.maxPages'])}
|
||||
<input
|
||||
type="number"
|
||||
name="maxPages"
|
||||
min={1}
|
||||
max={200}
|
||||
value={draft.maxPages}
|
||||
className="mt-1.5 h-9 w-full rounded-lg border-0 bg-components-input-bg-normal px-3 system-sm-regular text-text-primary outline-hidden focus:ring-2 focus:ring-state-accent-solid"
|
||||
onChange={(event) =>
|
||||
updateDraft({ ...draft, maxPages: event.target.valueAsNumber || 0 })
|
||||
}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === 'Enter') event.preventDefault()
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
</fieldset>
|
||||
)}
|
||||
</div>
|
||||
<Button
|
||||
type="button"
|
||||
variant="primary"
|
||||
className="w-full"
|
||||
disabled={disabled || !previewReady}
|
||||
onClick={showBackendBoundary}
|
||||
>
|
||||
{t(($) => $['newKnowledge.crawlAndPreview'])}
|
||||
</Button>
|
||||
<section
|
||||
aria-label={t(($) => $['newKnowledge.crawlPreview'])}
|
||||
className="flex min-h-40 flex-col items-center justify-center rounded-lg border border-dashed border-divider-regular px-6 text-center"
|
||||
>
|
||||
<span className="flex size-10 items-center justify-center rounded-lg bg-background-section">
|
||||
<span aria-hidden className="i-ri-global-line size-5 text-text-tertiary" />
|
||||
</span>
|
||||
<p className="mt-2 system-xs-semibold text-text-primary">
|
||||
{t(($) => $['newKnowledge.pagesAppearTitle'])}
|
||||
</p>
|
||||
<p className="mt-2 system-xs-regular text-text-tertiary">
|
||||
{t(($) => $['newKnowledge.pagesAppearDescription'])}
|
||||
</p>
|
||||
</section>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{draft.sourceType !== 'websiteCrawl' && (
|
||||
<div className="space-y-3">
|
||||
{draft.sourceType === 'onlineDocuments' && activeProvider === 'Notion' && (
|
||||
<section className="rounded-lg border border-divider-subtle bg-background-default p-4">
|
||||
<div className="flex items-start gap-3">
|
||||
<span
|
||||
aria-hidden
|
||||
className={cn(
|
||||
'mt-0.5 size-5 shrink-0',
|
||||
availableProviders.find((provider) => provider.label === activeProvider)?.icon,
|
||||
)}
|
||||
/>
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="system-sm-semibold text-text-primary">
|
||||
{t(($) => $['newKnowledge.notionNotConnected'])}
|
||||
</p>
|
||||
<p className="mt-1 system-xs-regular text-text-tertiary">
|
||||
{t(($) => $['newKnowledge.notionNotConnectedDescription'])}
|
||||
</p>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
disabled={disabled}
|
||||
className="h-8 shrink-0 rounded-lg bg-components-button-primary-bg px-3 system-xs-medium text-components-button-primary-text outline-hidden hover:bg-components-button-primary-bg-hover focus-visible:ring-2 focus-visible:ring-state-accent-solid disabled:cursor-not-allowed disabled:opacity-50"
|
||||
onClick={showBackendBoundary}
|
||||
>
|
||||
{t(($) => $['newKnowledge.connectNotion'])}
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
)}
|
||||
<ConnectedSourceConfiguration
|
||||
disabled={disabled}
|
||||
draft={draft}
|
||||
onDraftChange={updateDraft}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{backendBoundaryVisible && (
|
||||
<p
|
||||
role="alert"
|
||||
className="rounded-md bg-components-badge-status-light-warning-bg px-3 py-2 system-xs-regular text-text-warning"
|
||||
>
|
||||
{t(($) => $['newKnowledge.sourceSetupBackendDependency'])}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import { v4 as uuidV4 } from 'uuid'
|
||||
|
||||
export function createRequestId() {
|
||||
return globalThis.crypto?.randomUUID?.() ?? uuidV4()
|
||||
}
|
||||
@@ -1,4 +1,179 @@
|
||||
export type NewKnowledgeStartMode = 'empty' | 'source' | 'upload'
|
||||
export type NewKnowledgeSourceType = 'onlineDocuments' | 'onlineDrive' | 'websiteCrawl'
|
||||
type NewKnowledgeSyncPolicy = 'daily' | 'manual' | 'provider'
|
||||
export type NewKnowledgeWebsiteProvider = 'Firecrawl' | 'Jina Reader' | 'WaterCrawl'
|
||||
export type NewKnowledgeOnlineDocumentsProvider = 'Confluence' | 'Google Docs' | 'Notion'
|
||||
export type NewKnowledgeOnlineDriveProvider = 'Amazon S3' | 'Google Drive' | 'OneDrive'
|
||||
|
||||
type NewKnowledgeSourceDraftBase = {
|
||||
sourceName: string
|
||||
syncPolicy: NewKnowledgeSyncPolicy
|
||||
}
|
||||
|
||||
export type NewKnowledgeWebsiteSourceDraft = NewKnowledgeSourceDraftBase & {
|
||||
includeSubpages: boolean
|
||||
maxPages: number
|
||||
provider: NewKnowledgeWebsiteProvider
|
||||
rootUrl: string
|
||||
sourceType: 'websiteCrawl'
|
||||
}
|
||||
|
||||
export type NewKnowledgeOnlineDocumentsSourceDraft = NewKnowledgeSourceDraftBase & {
|
||||
provider: NewKnowledgeOnlineDocumentsProvider
|
||||
sourceType: 'onlineDocuments'
|
||||
}
|
||||
|
||||
export type NewKnowledgeOnlineDriveSourceDraft = NewKnowledgeSourceDraftBase & {
|
||||
provider: NewKnowledgeOnlineDriveProvider
|
||||
sourceType: 'onlineDrive'
|
||||
}
|
||||
|
||||
export type NewKnowledgeSourceDraft =
|
||||
| NewKnowledgeOnlineDocumentsSourceDraft
|
||||
| NewKnowledgeOnlineDriveSourceDraft
|
||||
| NewKnowledgeWebsiteSourceDraft
|
||||
|
||||
export const NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH = 200
|
||||
export const NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH = 2048
|
||||
const NEW_KNOWLEDGE_SOURCE_DRAFT_STORAGE_PREFIX = 'new-knowledge-source-draft:'
|
||||
|
||||
export function createNewKnowledgeSourceDraft(
|
||||
sourceType: NewKnowledgeSourceType,
|
||||
): NewKnowledgeSourceDraft {
|
||||
if (sourceType === 'onlineDocuments')
|
||||
return {
|
||||
provider: 'Notion',
|
||||
sourceName: '',
|
||||
sourceType,
|
||||
syncPolicy: 'provider',
|
||||
}
|
||||
if (sourceType === 'onlineDrive')
|
||||
return {
|
||||
provider: 'Google Drive',
|
||||
sourceName: '',
|
||||
sourceType,
|
||||
syncPolicy: 'provider',
|
||||
}
|
||||
return {
|
||||
includeSubpages: true,
|
||||
maxPages: 100,
|
||||
provider: 'Firecrawl',
|
||||
rootUrl: '',
|
||||
sourceName: '',
|
||||
sourceType,
|
||||
syncPolicy: 'provider',
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeWebsiteSourceUrl(value: string) {
|
||||
if (value.length > NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH) return undefined
|
||||
try {
|
||||
const url = new URL(value.trim())
|
||||
if (
|
||||
!['http:', 'https:'].includes(url.protocol) ||
|
||||
!url.hostname ||
|
||||
url.username ||
|
||||
url.password
|
||||
)
|
||||
return undefined
|
||||
url.hash = ''
|
||||
return url
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
export function isValidWebsiteSourceDraft(
|
||||
draft: NewKnowledgeWebsiteSourceDraft,
|
||||
{ allowEmpty = false }: { allowEmpty?: boolean } = {},
|
||||
) {
|
||||
const hasInput = Boolean(
|
||||
draft.rootUrl.length ||
|
||||
draft.sourceName.length ||
|
||||
!draft.includeSubpages ||
|
||||
draft.maxPages !== 100,
|
||||
)
|
||||
if (allowEmpty && !hasInput) return true
|
||||
const sourceName = draft.sourceName.trim()
|
||||
return Boolean(
|
||||
normalizeWebsiteSourceUrl(draft.rootUrl) &&
|
||||
sourceName &&
|
||||
sourceName.length <= NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH &&
|
||||
Number.isInteger(draft.maxPages) &&
|
||||
draft.maxPages > 0 &&
|
||||
draft.maxPages <= 200,
|
||||
)
|
||||
}
|
||||
|
||||
export function newKnowledgeSourceDraftStorageKey(draftKey: string) {
|
||||
return `${NEW_KNOWLEDGE_SOURCE_DRAFT_STORAGE_PREFIX}${draftKey}`
|
||||
}
|
||||
|
||||
export function parseNewKnowledgeSourceDraft(value: string): NewKnowledgeSourceDraft | undefined {
|
||||
try {
|
||||
const draft: unknown = JSON.parse(value)
|
||||
if (!draft || typeof draft !== 'object') return undefined
|
||||
const candidate = draft as Record<string, unknown>
|
||||
const syncPolicy = ['daily', 'manual', 'provider'].includes(String(candidate.syncPolicy))
|
||||
? (candidate.syncPolicy as NewKnowledgeSyncPolicy)
|
||||
: candidate.syncPolicy === undefined
|
||||
? 'provider'
|
||||
: undefined
|
||||
if (
|
||||
typeof candidate.sourceName !== 'string' ||
|
||||
candidate.sourceName.length > NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH ||
|
||||
!syncPolicy
|
||||
)
|
||||
return undefined
|
||||
if (candidate.sourceType === 'onlineDocuments') {
|
||||
if (!['Confluence', 'Google Docs', 'Notion'].includes(String(candidate.provider)))
|
||||
return undefined
|
||||
return {
|
||||
provider: candidate.provider as NewKnowledgeOnlineDocumentsProvider,
|
||||
sourceName: candidate.sourceName,
|
||||
sourceType: candidate.sourceType,
|
||||
syncPolicy,
|
||||
}
|
||||
}
|
||||
if (candidate.sourceType === 'onlineDrive') {
|
||||
if (!['Amazon S3', 'Google Drive', 'OneDrive'].includes(String(candidate.provider)))
|
||||
return undefined
|
||||
return {
|
||||
provider: candidate.provider as NewKnowledgeOnlineDriveProvider,
|
||||
sourceName: candidate.sourceName,
|
||||
sourceType: candidate.sourceType,
|
||||
syncPolicy,
|
||||
}
|
||||
}
|
||||
if (
|
||||
(candidate.sourceType !== undefined && candidate.sourceType !== 'websiteCrawl') ||
|
||||
!['Firecrawl', 'Jina Reader', 'WaterCrawl'].includes(String(candidate.provider)) ||
|
||||
typeof candidate.includeSubpages !== 'boolean' ||
|
||||
typeof candidate.maxPages !== 'number' ||
|
||||
!Number.isInteger(candidate.maxPages) ||
|
||||
candidate.maxPages < 1 ||
|
||||
candidate.maxPages > 200 ||
|
||||
typeof candidate.rootUrl !== 'string' ||
|
||||
candidate.rootUrl.length > NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH
|
||||
)
|
||||
return undefined
|
||||
return {
|
||||
includeSubpages: candidate.includeSubpages,
|
||||
maxPages: candidate.maxPages,
|
||||
provider: candidate.provider as NewKnowledgeWebsiteProvider,
|
||||
rootUrl: candidate.rootUrl,
|
||||
sourceName: candidate.sourceName,
|
||||
sourceType: 'websiteCrawl',
|
||||
syncPolicy,
|
||||
}
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
export function singleSearchParam(value: string | string[] | undefined) {
|
||||
return typeof value === 'string' ? value : undefined
|
||||
}
|
||||
|
||||
const newKnowledgeCreatePath = '/datasets/new/create'
|
||||
|
||||
@@ -13,5 +188,14 @@ export const newKnowledgeDetailPath = (knowledgeSpaceId: string) =>
|
||||
export const newKnowledgeDocumentsPath = (knowledgeSpaceId: string) =>
|
||||
`/datasets/new/${knowledgeSpaceId}/documents`
|
||||
|
||||
export const newKnowledgeAddSourcePath = (knowledgeSpaceId: string) =>
|
||||
`/datasets/new/${knowledgeSpaceId}/sources/new`
|
||||
export const newKnowledgeAddSourcePath = (
|
||||
knowledgeSpaceId: string,
|
||||
sourceType?: NewKnowledgeSourceType,
|
||||
draftKey?: string,
|
||||
) => {
|
||||
const searchParams = new URLSearchParams()
|
||||
if (sourceType) searchParams.set('type', sourceType)
|
||||
if (draftKey) searchParams.set('draft', draftKey)
|
||||
const query = searchParams.toString()
|
||||
return `/datasets/new/${knowledgeSpaceId}/sources/new${query ? `?${query}` : ''}`
|
||||
}
|
||||
|
||||
@@ -55,6 +55,10 @@ function metadataString(metadata: Source['metadata'], key: string) {
|
||||
return typeof value === 'string' && value.trim() ? value : undefined
|
||||
}
|
||||
|
||||
function isPreviewDraft(source: Source) {
|
||||
return source.metadata.preview === true && source.status === 'disabled'
|
||||
}
|
||||
|
||||
function createIdempotencyKey() {
|
||||
return globalThis.crypto?.randomUUID?.() ?? `${Date.now()}-${Math.random()}`
|
||||
}
|
||||
@@ -404,15 +408,15 @@ function SourcesEmpty({
|
||||
return (
|
||||
<div className="flex min-h-0 flex-1 flex-col items-center justify-center px-6 py-16 text-center">
|
||||
<div aria-hidden className="flex items-center gap-3">
|
||||
<span data-brand="firecrawl" className="i-ri-fire-fill size-8 text-orange-500" />
|
||||
<span data-brand="jina" className="i-custom-public-llm-jina size-8" />
|
||||
<span data-brand="firecrawl" className="i-custom-public-common-firecrawl size-5" />
|
||||
<span data-brand="jina" className="i-custom-public-llm-jina size-5" />
|
||||
<span
|
||||
data-brand="notion"
|
||||
className="i-custom-public-common-notion size-8 text-text-primary"
|
||||
className="i-custom-public-common-notion size-5 text-text-primary"
|
||||
/>
|
||||
<span data-brand="google-drive" className="i-custom-public-plugins-google size-8" />
|
||||
<span data-brand="confluence" className="i-custom-public-common-confluence size-8" />
|
||||
<span data-brand="more" className="i-ri-more-fill size-8 text-text-tertiary" />
|
||||
<span data-brand="google-drive" className="i-custom-public-common-google-drive size-5" />
|
||||
<span data-brand="confluence" className="i-custom-public-common-confluence size-5" />
|
||||
<span data-brand="dropbox" className="i-custom-public-common-dropbox size-5" />
|
||||
</div>
|
||||
<h2 className="mt-5 title-xl-semi-bold text-text-primary">
|
||||
{t(($) => $['newKnowledge.sourcesEmptyTitle'])}
|
||||
@@ -472,7 +476,12 @@ export function SourcesPage({ knowledgeSpaceId }: { knowledgeSpaceId: string })
|
||||
() =>
|
||||
(remoteSources ?? [])
|
||||
.filter((source) => !removedSourceIds.has(source.id))
|
||||
.map((source) => getCurrentSource(source, sourceOverrides[source.id])),
|
||||
.map((source) => getCurrentSource(source, sourceOverrides[source.id]))
|
||||
.filter((source) => !isPreviewDraft(source))
|
||||
.sort(
|
||||
(left, right) =>
|
||||
right.createdAt.localeCompare(left.createdAt) || right.id.localeCompare(left.id),
|
||||
),
|
||||
[remoteSources, removedSourceIds, sourceOverrides],
|
||||
)
|
||||
const loadedPageCount = sourcesQuery.data?.pages.length ?? 0
|
||||
@@ -492,8 +501,18 @@ export function SourcesPage({ knowledgeSpaceId }: { knowledgeSpaceId: string })
|
||||
const localTransformActive = filter !== 'all' || Boolean(search.trim()) || Boolean(sort)
|
||||
const canAutoCompleteFilteredResults =
|
||||
localTransformActive && loadedPageCount < MAX_AUTO_FILTER_PAGES
|
||||
const latestSourcePage = sourcesQuery.data?.pages.at(-1)
|
||||
const needsVisibleSource =
|
||||
latestSourcePage !== undefined &&
|
||||
latestSourcePage.items.some((source) =>
|
||||
isPreviewDraft(getCurrentSource(source, sourceOverrides[source.id])),
|
||||
) &&
|
||||
!latestSourcePage.items.some((source) => {
|
||||
if (removedSourceIds.has(source.id)) return false
|
||||
return !isPreviewDraft(getCurrentSource(source, sourceOverrides[source.id]))
|
||||
})
|
||||
const completingFilteredResults =
|
||||
canAutoCompleteFilteredResults &&
|
||||
(canAutoCompleteFilteredResults || needsVisibleSource) &&
|
||||
!sourcesQuery.isFetchNextPageError &&
|
||||
(sourcesQuery.hasNextPage || sourcesQuery.isFetchingNextPage)
|
||||
const allFilteredSourcesSelected =
|
||||
@@ -510,7 +529,7 @@ export function SourcesPage({ knowledgeSpaceId }: { knowledgeSpaceId: string })
|
||||
|
||||
useEffect(() => {
|
||||
if (
|
||||
canAutoCompleteFilteredResults &&
|
||||
(canAutoCompleteFilteredResults || needsVisibleSource) &&
|
||||
hasNextSourcePage &&
|
||||
!isFetchingNextSourcePage &&
|
||||
!sourcesQuery.isFetchNextPageError
|
||||
@@ -521,6 +540,7 @@ export function SourcesPage({ knowledgeSpaceId }: { knowledgeSpaceId: string })
|
||||
fetchNextSourcePage,
|
||||
hasNextSourcePage,
|
||||
isFetchingNextSourcePage,
|
||||
needsVisibleSource,
|
||||
sourcesQuery.isFetchNextPageError,
|
||||
])
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user