From 61e7a8906725c4d403539584dee1e582152a8596 Mon Sep 17 00:00:00 2001 From: ChengLei Shao Date: Tue, 31 Dec 2024 20:16:03 +0800 Subject: [PATCH] refactor: export action (#5665) * chore: export xlsx command * chore: container service * chore: report export progress * chore: export * chore: performInsert * chore: export command * chore: base exporter * chore: limit option * chore: export * chore: export * chore: types * chore: ws socket with auth * refactor: refactor exporter, change addRows method to handleRow method to support row-by-row processing * chore: xlsx exporter * chore: base exporter * chore: export limit * chore: import action * chore: load websocket * chore: import action * chore: object to cli args * chore: import options * chore: import action * chore: import runner * chore: import action * chore: import options * chore: import options * chore: plugin load event * chore: test * chore: i18n * chore: i18n * chore: i18n * fix: ws auth status * chore: cache in data source * chore: load datasource * chore: import with field alias * fix: build * fix: test * fix: build * fix: import schema settings * fix: import action * fix: import progress * chore: template creator * fix: import error message * fix: import error message * chore: test * chore: workflow dispatch event * chore: export alias * fix: typo * chore: error render * fix: event error * chore: send message to tags * fix: test * fix: import action setting --- .gitignore | 1 + packages/core/app/src/index.ts | 6 +- .../client/src/application/Application.tsx | 140 +++++-- .../client/src/application/PluginManager.ts | 1 + .../client/src/application/WebSocketClient.ts | 2 + packages/core/client/src/locale/zh-CN.json | 4 +- packages/core/sdk/src/APIClient.ts | 4 + packages/core/server/src/application.ts | 4 + packages/core/server/src/gateway/index.ts | 67 ++- packages/core/server/src/gateway/ws-server.ts | 58 ++- .../src/plugin-manager/plugin-manager.ts | 5 +- packages/core/server/src/service-container.ts | 15 + packages/core/utils/src/index.ts | 2 +- packages/core/utils/src/object-to-cli-args.ts | 13 + .../src/client/useExportAction.tsx | 3 + .../src/locale/zh-CN.json | 6 +- .../server/__tests__/export-to-xlsx.test.ts | 126 +++++- .../src/server/actions/export-xlsx.ts | 6 +- .../plugin-action-export/src/server/index.ts | 4 + .../src/server/services/base-exporter.ts | 172 ++++++++ .../src/server/services/xlsx-exporter.ts | 95 +++++ .../src/server/xlsx-exporter.ts | 221 ---------- .../plugin-action-import/package.json | 3 +- .../src/client/ImportAction.tsx | 130 ++++++ .../src/client/ImportActionContext.tsx | 8 + .../src/client/ImportActionInitializer.tsx | 172 ++------ .../src/client/ImportPluginProvider.tsx | 3 +- .../{schemaSettings.ts => schemaSettings.tsx} | 11 +- .../client/useDownloadXlsxTemplateAction.ts | 46 +++ .../src/client/useImportAction.ts | 49 ++- .../src/client/useShared.ts | 16 + .../plugin-action-import/src/index.ts | 1 + .../src/locale/en-US.json | 16 +- .../src/locale/zh-CN.json | 15 +- .../__tests__/download-template.test.ts | 7 +- .../server/__tests__/xlsx-importer.test.ts | 380 ++++++++++++++++-- .../server/actions/download-xlsx-template.ts | 23 +- .../plugin-action-import/src/server/errors.ts | 31 ++ .../plugin-action-import/src/server/index.ts | 45 ++- .../src/server/services/template-creator.ts | 137 ++++++- .../src/server/services/xlsx-importer.ts | 162 ++++++-- .../src/server/utils/workbook-converter.ts | 18 + .../plugin-auth/src/server/plugin.ts | 27 ++ .../src/server/models/data-source.ts | 3 + .../src/server/error-handler.ts | 18 +- .../plugin-error-handler/src/server/index.ts | 1 + .../plugin-error-handler/src/server/server.ts | 4 +- .../plugin-workflow/src/server/Plugin.ts | 5 + 48 files changed, 1740 insertions(+), 546 deletions(-) create mode 100644 packages/core/server/src/service-container.ts create mode 100644 packages/core/utils/src/object-to-cli-args.ts create mode 100644 packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts create mode 100644 packages/plugins/@nocobase/plugin-action-export/src/server/services/xlsx-exporter.ts delete mode 100644 packages/plugins/@nocobase/plugin-action-export/src/server/xlsx-exporter.ts create mode 100644 packages/plugins/@nocobase/plugin-action-import/src/client/ImportAction.tsx create mode 100644 packages/plugins/@nocobase/plugin-action-import/src/client/ImportActionContext.tsx rename packages/plugins/@nocobase/plugin-action-import/src/client/{schemaSettings.ts => schemaSettings.tsx} (87%) create mode 100644 packages/plugins/@nocobase/plugin-action-import/src/client/useDownloadXlsxTemplateAction.ts create mode 100644 packages/plugins/@nocobase/plugin-action-import/src/server/errors.ts create mode 100644 packages/plugins/@nocobase/plugin-action-import/src/server/utils/workbook-converter.ts diff --git a/.gitignore b/.gitignore index d3d8b147d09..63dd4bf4fff 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ storage/plugins storage/tar storage/tmp storage/print-templates +storage/cache storage/app.watch.ts storage/.upgrading storage/logs-e2e diff --git a/packages/core/app/src/index.ts b/packages/core/app/src/index.ts index 69f11f9d06d..484df0acfad 100644 --- a/packages/core/app/src/index.ts +++ b/packages/core/app/src/index.ts @@ -13,12 +13,10 @@ import { getConfig } from './config'; async function initializeGateway() { await runPluginStaticImports(); const config = await getConfig(); + await Gateway.getInstance().run({ mainAppOptions: config, }); } -initializeGateway().catch((e) => { - console.error(e); - process.exit(1); -}); +initializeGateway(); diff --git a/packages/core/client/src/application/Application.tsx b/packages/core/client/src/application/Application.tsx index feadcd27378..2000208d1a3 100644 --- a/packages/core/client/src/application/Application.tsx +++ b/packages/core/client/src/application/Application.tsx @@ -76,6 +76,8 @@ export interface ApplicationOptions { } export class Application { + public eventBus = new EventTarget(); + public providers: ComponentAndProps[] = []; public router: RouterManager; public scopes: Record = {}; @@ -96,13 +98,15 @@ export class Application { public schemaInitializerManager: SchemaInitializerManager; public schemaSettingsManager: SchemaSettingsManager; public dataSourceManager: DataSourceManager; - public name: string; loading = true; maintained = false; maintaining = false; error = null; + + private wsAuthorized = false; + get pm() { return this.pluginManager; } @@ -110,6 +114,14 @@ export class Application { return this.options.disableAcl; } + get isWsAuthorized() { + return this.wsAuthorized; + } + + setWsAuthorized(authorized: boolean) { + this.wsAuthorized = authorized; + } + constructor(protected options: ApplicationOptions = {}) { this.initRequireJs(); define(this, { @@ -140,6 +152,46 @@ export class Application { this.i18n.on('languageChanged', (lng) => { this.apiClient.auth.locale = lng; }); + this.initListeners(); + } + + private initListeners() { + this.eventBus.addEventListener('auth:tokenChanged', (event: CustomEvent) => { + this.setTokenInWebSocket(event.detail.token); + }); + + this.eventBus.addEventListener('maintaining:end', () => { + if (this.apiClient.auth.token) { + this.setTokenInWebSocket(this.apiClient.auth.token); + } + }); + } + + protected setTokenInWebSocket(token: string) { + if (this.maintaining) { + return; + } + + this.ws.send( + JSON.stringify({ + type: 'auth:token', + payload: { + token, + }, + }), + ); + } + + setMaintaining(maintaining: boolean) { + // if maintaining is the same, do nothing + if (this.maintaining === maintaining) { + return; + } + + this.maintaining = maintaining; + if (!maintaining) { + this.eventBus.dispatchEvent(new Event('maintaining:end')); + } } private initRequireJs() { @@ -240,40 +292,9 @@ export class Application { } async load() { - let loadFailed = false; - this.ws.on('message', (event) => { - const data = JSON.parse(event.data); - console.log(data.payload); - if (data?.payload?.refresh) { - window.location.reload(); - return; - } - if (data.type === 'notification') { - this.notification[data.payload?.type || 'info']({ message: data.payload?.message }); - return; - } - const maintaining = data.type === 'maintaining' && data.payload.code !== 'APP_RUNNING'; - if (maintaining) { - this.maintaining = true; - this.error = data.payload; - } else { - // console.log('loadFailed', loadFailed); - if (loadFailed) { - window.location.reload(); - return; - } - this.maintaining = false; - this.maintained = true; - this.error = null; - } - }); - this.ws.on('serverDown', () => { - this.maintaining = true; - this.maintained = false; - }); - this.ws.connect(); try { this.loading = true; + await this.loadWebSocket(); await this.pm.load(); } catch (error) { if (this.ws.enabled) { @@ -281,7 +302,6 @@ export class Application { setTimeout(() => resolve(null), 1000); }); } - loadFailed = true; const toError = (error) => { if (typeof error?.response?.data === 'string') { const tempElement = document.createElement('div'); @@ -305,6 +325,58 @@ export class Application { this.loading = false; } + async loadWebSocket() { + this.eventBus.addEventListener('ws:message:authorized', () => { + this.setWsAuthorized(true); + }); + + this.ws.on('message', (event) => { + const data = JSON.parse(event.data); + + if (data?.payload?.refresh) { + window.location.reload(); + return; + } + if (data.type === 'notification') { + this.notification[data.payload?.type || 'info']({ message: data.payload?.message }); + return; + } + const maintaining = data.type === 'maintaining' && data.payload.code !== 'APP_RUNNING'; + + if (maintaining) { + this.setMaintaining(true); + this.error = data.payload; + } else { + this.setMaintaining(false); + this.maintained = true; + this.error = null; + + const type = data.type; + if (!type) { + return; + } + + const eventName = `ws:message:${type}`; + this.eventBus.dispatchEvent(new CustomEvent(eventName, { detail: data.payload })); + } + }); + + this.ws.on('serverDown', () => { + this.maintaining = true; + this.maintained = false; + }); + + this.ws.on('open', () => { + const token = this.apiClient.auth.token; + + if (token) { + this.setTokenInWebSocket(token); + } + }); + + this.ws.connect(); + } + getComponent(Component: ComponentTypeAndString, isShowError = true): ComponentType | undefined { const showError = (msg: string) => isShowError && console.error(msg); if (!Component) { diff --git a/packages/core/client/src/application/PluginManager.ts b/packages/core/client/src/application/PluginManager.ts index 936fd43a191..30c23617dab 100644 --- a/packages/core/client/src/application/PluginManager.ts +++ b/packages/core/client/src/application/PluginManager.ts @@ -108,6 +108,7 @@ export class PluginManager { for (const plugin of this.pluginInstances.values()) { await plugin.load(); + this.app.eventBus.dispatchEvent(new CustomEvent(`plugin:${plugin.options.name}:loaded`, { detail: plugin })); } } } diff --git a/packages/core/client/src/application/WebSocketClient.ts b/packages/core/client/src/application/WebSocketClient.ts index a753442b2eb..edf95e8d0c2 100644 --- a/packages/core/client/src/application/WebSocketClient.ts +++ b/packages/core/client/src/application/WebSocketClient.ts @@ -108,6 +108,7 @@ export class WebSocketClient { this._reconnectTimes++; const ws = new WebSocket(this.getURL(), this.options.protocols); let pingIntervalTimer: any; + ws.onopen = () => { console.log('[nocobase-ws]: connected.'); this.serverDown = false; @@ -121,6 +122,7 @@ export class WebSocketClient { } pingIntervalTimer = setInterval(() => this.send('ping'), this.pingInterval); this.connected = true; + this.emit('open', {}); }; ws.onerror = async () => { // setTimeout(() => this.connect(), this.reconnectInterval); diff --git a/packages/core/client/src/locale/zh-CN.json b/packages/core/client/src/locale/zh-CN.json index 79095828e68..cc08747aa15 100644 --- a/packages/core/client/src/locale/zh-CN.json +++ b/packages/core/client/src/locale/zh-CN.json @@ -369,7 +369,7 @@ "Hide": "隐藏", "Enable actions": "启用操作", "Import": "导入", - "Export": "导出", + "Export": "导出记录", "Customize": "自定义", "Custom": "自定义", "Function": "Function", @@ -740,7 +740,7 @@ "Plugin starting...": "插件启动中...", "Plugin stopping...": "插件停止中...", "Are you sure to delete this plugin?": "确定要删除此插件吗？", - "Are you sure to disable this plugin?": "确定要禁用此插件吗?", + "Are you sure to disable this plugin?": "确定要禁用此插件吗？", "re-download file": "重新下载文件", "Not enabled": "未启用", "Search plugin": "搜索插件", diff --git a/packages/core/sdk/src/APIClient.ts b/packages/core/sdk/src/APIClient.ts index 32ddf1311a6..04a80cd9798 100644 --- a/packages/core/sdk/src/APIClient.ts +++ b/packages/core/sdk/src/APIClient.ts @@ -166,6 +166,10 @@ export class Auth { */ setToken(token: string) { this.setOption('token', token); + + if (this.api['app']) { + this.api['app'].eventBus.dispatchEvent(new CustomEvent('auth:tokenChanged', { detail: token })); + } } /** diff --git a/packages/core/server/src/application.ts b/packages/core/server/src/application.ts index a57311e298b..e772349e588 100644 --- a/packages/core/server/src/application.ts +++ b/packages/core/server/src/application.ts @@ -73,6 +73,7 @@ import { createPubSubManager, PubSubManager, PubSubManagerOptions } from './pub- import { SyncMessageManager } from './sync-message-manager'; import packageJson from '../package.json'; +import { ServiceContainer } from './service-container'; import { availableActions } from './acl/available-action'; import { AuditManager } from './audit-manager'; @@ -245,6 +246,7 @@ export class Application exten private _maintainingStatusBeforeCommand: MaintainingCommandStatus | null; private _actionCommand: Command; + public container = new ServiceContainer(); public lockManager: LockManager; constructor(public options: ApplicationOptions) { @@ -774,9 +776,11 @@ export class Application exten try { const commandName = options?.from === 'user' ? argv[0] : argv[2]; + if (!this.cli.hasCommand(commandName)) { await this.pm.loadCommands(); } + const command = await this.cli.parseAsync(argv, options); this.setMaintaining({ diff --git a/packages/core/server/src/gateway/index.ts b/packages/core/server/src/gateway/index.ts index 04af3e682c1..1c5fe690786 100644 --- a/packages/core/server/src/gateway/index.ts +++ b/packages/core/server/src/gateway/index.ts @@ -29,6 +29,8 @@ import { applyErrorWithArgs, getErrorWithCode } from './errors'; import { IPCSocketClient } from './ipc-socket-client'; import { IPCSocketServer } from './ipc-socket-server'; import { WSServer } from './ws-server'; +import { isMainThread, workerData } from 'node:worker_threads'; +import process from 'node:process'; const compress = promisify(compression()); @@ -346,11 +348,14 @@ export class Gateway extends EventEmitter { const mainApp = AppSupervisor.getInstance().bootMainApp(options.mainAppOptions); + let runArgs: any = [process.argv, { throwError: true, from: 'node' }]; + + if (!isMainThread) { + runArgs = [workerData.argv, { throwError: true, from: 'user' }]; + } + mainApp - .runAsCLI(process.argv, { - throwError: true, - from: 'node', - }) + .runAsCLI(...runArgs) .then(async () => { if (!isStart && !(await mainApp.isStarted())) { await mainApp.stop({ logging: false }); @@ -358,8 +363,13 @@ export class Gateway extends EventEmitter { }) .catch(async (e) => { if (e.code !== 'commander.helpDisplayed') { + if (!isMainThread) { + throw e; + } + mainApp.log.error(e); } + if (!isStart && !(await mainApp.isStarted())) { await mainApp.stop({ logging: false }); } @@ -416,6 +426,55 @@ export class Gateway extends EventEmitter { this.wsServer = new WSServer(); + this.wsServer.on('message', async ({ client, message }) => { + const app = await AppSupervisor.getInstance().getApp(client.app); + const parsedMessage = JSON.parse(message.toString()); + + if (!parsedMessage.type) { + return; + } + + // check app has event listener + + if (!app.listenerCount(`ws:setTag`)) { + app.on('ws:setTag', ({ clientId, tagKey, tagValue }) => { + this.wsServer.setClientTag(clientId, tagKey, tagValue); + }); + + app.on('ws:sendToTag', ({ tagKey, tagValue, message }) => { + this.wsServer.sendToConnectionsByTags( + [ + { tagName: tagKey, tagValue }, + { tagName: 'app', tagValue: app.name }, + ], + message, + ); + }); + + app.on('ws:sendToTags', ({ tags, message }) => { + this.wsServer.sendToConnectionsByTags(tags, message); + }); + + app.on('ws:authorized', ({ clientId, userId }) => { + this.wsServer.sendToConnectionsByTags( + [ + { tagName: 'userId', tagValue: userId }, + { tagName: 'app', tagValue: app.name }, + ], + { type: 'authorized' }, + ); + }); + } + + const eventName = `ws:message:${parsedMessage.type}`; + + app.emit(eventName, { + clientId: client.id, + tags: [...client.tags], + payload: parsedMessage.payload, + }); + }); + this.server.on('upgrade', (request, socket, head) => { const { pathname } = parse(request.url); diff --git a/packages/core/server/src/gateway/ws-server.ts b/packages/core/server/src/gateway/ws-server.ts index c118be874c1..54dad182594 100644 --- a/packages/core/server/src/gateway/ws-server.ts +++ b/packages/core/server/src/gateway/ws-server.ts @@ -8,13 +8,14 @@ */ import { Gateway, IncomingRequest } from '../gateway'; -import WebSocket, { WebSocketServer } from 'ws'; +import WebSocket, { WebSocketServer as WSS } from 'ws'; import { nanoid } from 'nanoid'; import { IncomingMessage } from 'http'; import { AppSupervisor } from '../app-supervisor'; import { applyErrorWithArgs, getErrorWithCode } from './errors'; import lodash from 'lodash'; import { Logger } from '@nocobase/logger'; +import EventEmitter from 'events'; declare class WebSocketWithId extends WebSocket { id: string; @@ -22,10 +23,11 @@ declare class WebSocketWithId extends WebSocket { interface WebSocketClient { ws: WebSocketWithId; - tags: string[]; + tags: Set; url: string; headers: any; app?: string; + id: string; } function getPayloadByErrorCode(code, options) { @@ -33,13 +35,14 @@ function getPayloadByErrorCode(code, options) { return lodash.omit(applyErrorWithArgs(error, options), ['status', 'maintaining']); } -export class WSServer { +export class WSServer extends EventEmitter { wss: WebSocket.Server; webSocketClients = new Map(); logger: Logger; constructor() { - this.wss = new WebSocketServer({ noServer: true }); + super(); + this.wss = new WSS({ noServer: true }); this.wss.on('connection', (ws: WebSocketWithId, request: IncomingMessage) => { const client = this.addNewConnection(ws, request); @@ -53,18 +56,33 @@ export class WSServer { ws.on('close', () => { this.removeConnection(ws.id); }); + + ws.on('message', (message) => { + if (message.toString() === 'ping') { + return; + } + + this.emit('message', { + client, + message, + }); + }); }); Gateway.getInstance().on('appSelectorChanged', () => { - // reset connection app tags this.loopThroughConnections(async (client) => { const handleAppName = await Gateway.getInstance().getRequestHandleAppName({ url: client.url, headers: client.headers, }); - client.tags = client.tags.filter((tag) => !tag.startsWith('app#')); - client.tags.push(`app#${handleAppName}`); + for (const tag of client.tags) { + if (tag.startsWith('app#')) { + client.tags.delete(tag); + } + } + + client.tags.add(`app#${handleAppName}`); AppSupervisor.getInstance().bootStrapApp(handleAppName); }); @@ -121,21 +139,26 @@ export class WSServer { addNewConnection(ws: WebSocketWithId, request: IncomingMessage) { const id = nanoid(); - ws.id = id; this.webSocketClients.set(id, { ws, - tags: [], + tags: new Set(), url: request.url, headers: request.headers, + id, }); this.setClientApp(this.webSocketClients.get(id)); - return this.webSocketClients.get(id); } + setClientTag(clientId: string, tagKey: string, tagValue: string) { + const client = this.webSocketClients.get(clientId); + client.tags.add(`${tagKey}#${tagValue}`); + console.log(`client tags: ${Array.from(client.tags)}`); + } + async setClientApp(client: WebSocketClient) { const req: IncomingRequest = { url: client.url, @@ -146,7 +169,7 @@ export class WSServer { client.app = handleAppName; console.log(`client tags: app#${handleAppName}`); - client.tags.push(`app#${handleAppName}`); + client.tags.add(`app#${handleAppName}`); const hasApp = AppSupervisor.getInstance().hasApp(handleAppName); @@ -191,8 +214,19 @@ export class WSServer { } sendToConnectionsByTag(tagName: string, tagValue: string, sendMessage: object) { + this.sendToConnectionsByTags([{ tagName, tagValue }], sendMessage); + } + + /** + * Send message to clients that match all the given tag conditions + * @param tags Array of tag conditions, each condition is an object with tagName and tagValue + * @param sendMessage Message to be sent + */ + sendToConnectionsByTags(tags: Array<{ tagName: string; tagValue: string }>, sendMessage: object) { this.loopThroughConnections((client: WebSocketClient) => { - if (client.tags.includes(`${tagName}#${tagValue}`)) { + const allTagsMatch = tags.every(({ tagName, tagValue }) => client.tags.has(`${tagName}#${tagValue}`)); + + if (allTagsMatch) { this.sendMessageToConnection(client, sendMessage); } }); diff --git a/packages/core/server/src/plugin-manager/plugin-manager.ts b/packages/core/server/src/plugin-manager/plugin-manager.ts index d4f044fa8b6..68113f6a515 100644 --- a/packages/core/server/src/plugin-manager/plugin-manager.ts +++ b/packages/core/server/src/plugin-manager/plugin-manager.ts @@ -395,7 +395,7 @@ export class PluginManager { const source = []; for (const packageName of packageNames) { const dirname = await getPluginBasePath(packageName); - const directory = join(dirname, 'server/commands/*.' + (basename(dirname) === 'src' ? 'ts' : 'js')); + const directory = join(dirname, 'server/commands/*.' + (basename(dirname) === 'src' ? '{ts,js}' : 'js')); source.push(directory.replaceAll(sep, '/')); } @@ -403,7 +403,7 @@ export class PluginManager { if (typeof plugin === 'string') { const { packageName } = await PluginManager.parseName(plugin); const dirname = await getPluginBasePath(packageName); - const directory = join(dirname, 'server/commands/*.' + (basename(dirname) === 'src' ? 'ts' : 'js')); + const directory = join(dirname, 'server/commands/*.' + (basename(dirname) === 'src' ? '{ts,js}' : 'js')); source.push(directory.replaceAll(sep, '/')); } } @@ -411,6 +411,7 @@ export class PluginManager { ignore: ['**/*.d.ts'], cwd: process.env.NODE_MODULES_PATH, }); + for (const file of files) { const callback = await importModule(file); callback(this.app); diff --git a/packages/core/server/src/service-container.ts b/packages/core/server/src/service-container.ts new file mode 100644 index 00000000000..51617fbfdaa --- /dev/null +++ b/packages/core/server/src/service-container.ts @@ -0,0 +1,15 @@ +export class ServiceContainer { + private services: Map = new Map(); + + public register(name: string, service: T) { + if (typeof service === 'function') { + service = service(); + } + + this.services.set(name, service); + } + + public get(name: string): T { + return this.services.get(name); + } +} diff --git a/packages/core/utils/src/index.ts b/packages/core/utils/src/index.ts index fb8a5cb4747..df78d970ae6 100644 --- a/packages/core/utils/src/index.ts +++ b/packages/core/utils/src/index.ts @@ -36,6 +36,6 @@ export * from './uid'; export * from './url'; export * from './i18n'; export * from './wrap-middleware'; - +export * from './object-to-cli-args'; export { dayjs, lodash }; export { Schema } from '@formily/json-schema'; diff --git a/packages/core/utils/src/object-to-cli-args.ts b/packages/core/utils/src/object-to-cli-args.ts new file mode 100644 index 00000000000..a60407bdc84 --- /dev/null +++ b/packages/core/utils/src/object-to-cli-args.ts @@ -0,0 +1,13 @@ +/** + * Convert object to CLI arguments array + * @param obj Object to convert + * @returns CLI arguments array + */ +export function objectToCliArgs(obj: Record): string[] { + return Object.entries(obj) + .filter(([_, value]) => value !== null && value !== undefined) + .map(([key, value]) => { + const stringValue = typeof value === 'object' ? JSON.stringify(value) : value; + return `--${key}=${stringValue}`; + }); +} diff --git a/packages/plugins/@nocobase/plugin-action-export/src/client/useExportAction.tsx b/packages/plugins/@nocobase/plugin-action-export/src/client/useExportAction.tsx index 2dda4bf97ae..7fd6fc2ee73 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/client/useExportAction.tsx +++ b/packages/plugins/@nocobase/plugin-action-export/src/client/useExportAction.tsx @@ -54,11 +54,14 @@ export const useExportAction = () => { content: t('Export warning', { limit: exportLimit }), okText: t('Start export'), }); + if (!confirmed) { return; } + field.data.loading = true; const { exportSettings } = lodash.cloneDeep(actionSchema?.['x-action-settings'] ?? {}); + exportSettings.forEach((es) => { const { uiSchema, interface: fieldInterface } = getCollectionJoinField(`${name}.${es.dataIndex.join('.')}`) ?? {}; diff --git a/packages/plugins/@nocobase/plugin-action-export/src/locale/zh-CN.json b/packages/plugins/@nocobase/plugin-action-export/src/locale/zh-CN.json index d8f5bbda9e6..efc2ba7307b 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/locale/zh-CN.json +++ b/packages/plugins/@nocobase/plugin-action-export/src/locale/zh-CN.json @@ -1,7 +1,7 @@ { - "Export warning": "每次最多导出 {{limit}} 行数据，超出的将被忽略。", - "Start export": "开始导出", - "another export action is running, please try again later.": "另一导出任务正在运行，请稍后重试。", + "Export warning": "每次最多导出记录 {{limit}} 行数据，超出的将被忽略。", + "Start export": "开始导出记录", + "another export action is running, please try again later.": "另一导出记录任务正在运行，请稍后重试。", "True": "是", "False": "否" } diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/__tests__/export-to-xlsx.test.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/__tests__/export-to-xlsx.test.ts index 2c9db76b4ab..43844ca3611 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/__tests__/export-to-xlsx.test.ts +++ b/packages/plugins/@nocobase/plugin-action-export/src/server/__tests__/export-to-xlsx.test.ts @@ -9,7 +9,7 @@ import { createMockServer, MockServer } from '@nocobase/test'; import { uid } from '@nocobase/utils'; -import XlsxExporter from '../xlsx-exporter'; +import { XlsxExporter } from '../services/xlsx-exporter'; import XLSX from 'xlsx'; import fs from 'fs'; import path from 'path'; @@ -510,6 +510,7 @@ describe('export to xlsx', () => { afterEach(async () => { await app.destroy(); + vi.unstubAllEnvs(); }); it('should throw error when export field not exists', async () => { @@ -1374,4 +1375,127 @@ describe('export to xlsx', () => { fs.unlinkSync(xlsxFilePath); } }); + + it('should respect the EXPORT_LIMIT env variable', async () => { + vi.stubEnv('EXPORT_LIMIT', '30'); // Set a small limit + + const User = app.db.collection({ + name: 'users', + fields: [ + { type: 'string', name: 'name' }, + { type: 'integer', name: 'age' }, + ], + }); + + await app.db.sync(); + + const values = Array.from({ length: 100 }).map((_, index) => { + return { + name: `user${index}`, + age: index % 100, + }; + }); + + await User.model.bulkCreate(values); + + const exporter = new XlsxExporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + chunkSize: 10, + columns: [ + { dataIndex: ['name'], defaultTitle: 'Name' }, + { dataIndex: ['age'], defaultTitle: 'Age' }, + ], + }); + + const wb = await exporter.run(); + const firstSheet = wb.Sheets[wb.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 }); + + expect(sheetData.length).toBe(31); // 30 users + 1 header + expect(sheetData[0]).toEqual(['Name', 'Age']); // header + expect(sheetData[1]).toEqual(['user0', 0]); // first user + expect(sheetData[30]).toEqual(['user29', 29]); // last user + }); + + it('should use default EXPORT_LIMIT (2000) when env not set', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { type: 'string', name: 'name' }, + { type: 'integer', name: 'age' }, + ], + }); + + await app.db.sync(); + + const values = Array.from({ length: 2500 }).map((_, index) => { + return { + name: `user${index}`, + age: index % 100, + }; + }); + + await User.model.bulkCreate(values); + + const exporter = new XlsxExporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + chunkSize: 10, + columns: [ + { dataIndex: ['name'], defaultTitle: 'Name' }, + { dataIndex: ['age'], defaultTitle: 'Age' }, + ], + }); + + const wb = await exporter.run(); + const firstSheet = wb.Sheets[wb.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 }); + + expect(sheetData.length).toBe(2001); // 2000 users + 1 header + expect(sheetData[0]).toEqual(['Name', 'Age']); // header + expect(sheetData[1]).toEqual(['user0', 0]); // first user + expect(sheetData[2000]).toEqual(['user1999', 99]); // last user + }); + + it('should respect the limit option when exporting data', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { type: 'string', name: 'name' }, + { type: 'integer', name: 'age' }, + ], + }); + + await app.db.sync(); + + const values = Array.from({ length: 100 }).map((_, index) => { + return { + name: `user${index}`, + age: index % 100, + }; + }); + + await User.model.bulkCreate(values); + + const exporter = new XlsxExporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + chunkSize: 10, + limit: 10, + columns: [ + { dataIndex: ['name'], defaultTitle: 'Name' }, + { dataIndex: ['age'], defaultTitle: 'Age' }, + ], + }); + + const wb = await exporter.run(); + const firstSheet = wb.Sheets[wb.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 }); + + expect(sheetData.length).toBe(11); // 10 users + 1 header + expect(sheetData[0]).toEqual(['Name', 'Age']); // header + expect(sheetData[1]).toEqual(['user0', 0]); // first user + expect(sheetData[10]).toEqual(['user9', 9]); // last user + }); }); diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/actions/export-xlsx.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/actions/export-xlsx.ts index a9c42d40fb2..0325e82155e 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/actions/export-xlsx.ts +++ b/packages/plugins/@nocobase/plugin-action-export/src/server/actions/export-xlsx.ts @@ -10,7 +10,7 @@ import { Context, Next } from '@nocobase/actions'; import { Repository } from '@nocobase/database'; -import XlsxExporter from '../xlsx-exporter'; +import { XlsxExporter } from '../services/xlsx-exporter'; import XLSX from 'xlsx'; import { Mutex } from 'async-mutex'; import { DataSource } from '@nocobase/data-source-manager'; @@ -54,6 +54,10 @@ async function exportXlsxAction(ctx: Context, next: Next) { } export async function exportXlsx(ctx: Context, next: Next) { + if (ctx.exportHandled) { + return await next(); + } + if (mutex.isLocked()) { throw new Error( ctx.t(`another export action is running, please try again later.`, { diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/index.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/index.ts index ed523e098fd..8095c93221b 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/index.ts +++ b/packages/plugins/@nocobase/plugin-action-export/src/server/index.ts @@ -46,9 +46,13 @@ export class PluginActionExportServer extends Plugin { dataSource.acl.setAvailableAction('export', { displayName: '{{t("Export")}}', allowConfigureFields: true, + aliases: ['export', 'exportAttachments'], }); }); } } export default PluginActionExportServer; + +export * from './services/base-exporter'; +export * from './services/xlsx-exporter'; diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts new file mode 100644 index 00000000000..a86eea3c3d6 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts @@ -0,0 +1,172 @@ +import { + FindOptions, + ICollection, + ICollectionManager, + IField, + IModel, + IRelationField, +} from '@nocobase/data-source-manager'; +import EventEmitter from 'events'; +import { deepGet } from '../utils/deep-get'; +import path from 'path'; +import os from 'os'; + +export type ExportOptions = { + collectionManager: ICollectionManager; + collection: ICollection; + repository?: any; + fields: Array>; + findOptions?: FindOptions; + chunkSize?: number; + limit?: number; +}; + +abstract class BaseExporter extends EventEmitter { + /** + * You can adjust the maximum number of exported rows based on business needs and system + * available resources. However, please note that you need to fully understand the risks + * after the modification. Increasing the maximum number of rows that can be exported may + * increase system resource usage, leading to increased processing delays for other + * requests, or even server processes being recycled by the operating system. + * + * 您可以根据业务需求和系统可用资源等参数，调整最大导出数量的限制。但请注意，您需要充分了解修改之后的风险， + * 增加最大可导出的行数可能会导致系统资源占用率升高，导致其他请求处理延迟增加、无法处理、甚至 + * 服务端进程被操作系统回收等问题。 + */ + protected limit: number; + + protected constructor(protected options: T) { + super(); + this.limit = options.limit ?? (process.env['EXPORT_LIMIT'] ? parseInt(process.env['EXPORT_LIMIT']) : 2000); + } + + abstract init(ctx?): Promise; + abstract finalize(): Promise; + abstract handleRow(row: any, ctx?): Promise; + + async run(ctx?): Promise { + await this.init(ctx); + + const { collection, chunkSize, repository } = this.options; + + const total = await (repository || collection.repository).count(this.getFindOptions()); + let current = 0; + + await (repository || collection.repository).chunk({ + ...this.getFindOptions(), + chunkSize: chunkSize || 200, + callback: async (rows, options) => { + for (const row of rows) { + await this.handleRow(row, ctx); + current += 1; + + this.emit('progress', { + total, + current, + }); + } + }, + }); + + return this.finalize(); + } + + protected getAppendOptionsFromFields() { + return this.options.fields + .map((field) => { + const fieldInstance = this.options.collection.getField(field[0]); + if (!fieldInstance) { + throw new Error(`Field "${field[0]}" not found: , please check the fields configuration.`); + } + + if (fieldInstance.isRelationField()) { + return field.join('.'); + } + + return null; + }) + .filter(Boolean); + } + + protected getFindOptions() { + const { findOptions = {} } = this.options; + + if (this.limit) { + findOptions.limit = this.limit; + } + + const appendOptions = this.getAppendOptionsFromFields(); + + if (appendOptions.length) { + return { + ...findOptions, + appends: appendOptions, + }; + } + + return findOptions; + } + + protected findFieldByDataIndex(dataIndex: Array): IField { + const { collection } = this.options; + const currentField = collection.getField(dataIndex[0]); + + if (dataIndex.length > 1) { + let targetCollection: ICollection; + + for (let i = 0; i < dataIndex.length; i++) { + const isLast = i === dataIndex.length - 1; + + if (isLast) { + return targetCollection.getField(dataIndex[i]); + } + + targetCollection = (currentField as IRelationField).targetCollection(); + } + } + + return currentField; + } + + protected renderRawValue(value) { + if (typeof value === 'object' && value !== null) { + return JSON.stringify(value); + } + + return value; + } + + protected getFieldRenderer(field?: IField, ctx?): (value) => any { + const InterfaceClass = this.options.collectionManager.getFieldInterface(field?.options?.interface); + if (!InterfaceClass) { + return this.renderRawValue; + } + const fieldInterface = new InterfaceClass(field?.options); + return (value) => fieldInterface.toString(value, ctx); + } + + protected formatValue(rowData: IModel, dataIndex: Array, ctx?) { + rowData = rowData.toJSON(); + const value = rowData[dataIndex[0]]; + const field = this.findFieldByDataIndex(dataIndex); + const render = this.getFieldRenderer(field, ctx); + + if (dataIndex.length > 1) { + const deepValue = deepGet(rowData, dataIndex); + + if (Array.isArray(deepValue)) { + return deepValue.map(render).join(','); + } + + return render(deepValue); + } + return render(value); + } + + public generateOutputPath(prefix = 'export', ext = '', destination = os.tmpdir()): string { + const fileName = `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2)}${ext}`; + return path.join(destination, fileName); + } +} + +export { BaseExporter }; diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/services/xlsx-exporter.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/services/xlsx-exporter.ts new file mode 100644 index 00000000000..900837d3fc1 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-export/src/server/services/xlsx-exporter.ts @@ -0,0 +1,95 @@ +import XLSX from 'xlsx'; +import { BaseExporter, ExportOptions } from './base-exporter'; +import { NumberField } from '@nocobase/database'; + +type ExportColumn = { + dataIndex: Array; + title?: string; + defaultTitle: string; +}; + +type XlsxExportOptions = Omit & { + columns: Array; +}; + +export class XlsxExporter extends BaseExporter> }> { + /** + * You can adjust the maximum number of exported rows based on business needs and system + * available resources. However, please note that you need to fully understand the risks + * after the modification. Increasing the maximum number of rows that can be exported may + * increase system resource usage, leading to increased processing delays for other + * requests, or even server processes being recycled by the operating system. + * + * 您可以根据业务需求和系统可用资源等参数，调整最大导出数量的限制。但请注意，您需要充分了解修改之后的风险， + * 增加最大可导出的行数可能会导致系统资源占用率升高，导致其他请求处理延迟增加、无法处理、甚至 + * 服务端进程被操作系统回收等问题。 + */ + + private workbook: XLSX.WorkBook; + private worksheet: XLSX.WorkSheet; + private startRowNumber: number; + + constructor(options: XlsxExportOptions) { + const fields = options.columns.map((col) => col.dataIndex); + super({ ...options, fields }); + } + + async init(ctx?): Promise { + this.workbook = XLSX.utils.book_new(); + this.worksheet = XLSX.utils.sheet_new(); + + // write headers + XLSX.utils.sheet_add_aoa(this.worksheet, [this.renderHeaders(this.options.columns)], { + origin: 'A1', + }); + + this.startRowNumber = 2; + } + + async handleRow(row: any, ctx?): Promise { + const rowData = [ + this.options.columns.map((col) => { + return this.formatValue(row, col.dataIndex, ctx); + }), + ]; + + XLSX.utils.sheet_add_aoa(this.worksheet, rowData, { + origin: `A${this.startRowNumber}`, + }); + + this.startRowNumber += 1; + } + + async finalize(): Promise { + for (const col of this.options.columns) { + const fieldInstance = this.findFieldByDataIndex(col.dataIndex); + if (fieldInstance instanceof NumberField) { + // set column cell type to number + const colIndex = this.options.columns.indexOf(col); + const cellRange = XLSX.utils.decode_range(this.worksheet['!ref']); + + for (let r = 1; r <= cellRange.e.r; r++) { + const cell = this.worksheet[XLSX.utils.encode_cell({ c: colIndex, r })]; + // if cell and cell.v is a number, set cell.t to 'n' + if (cell && isNumeric(cell.v)) { + cell.t = 'n'; + } + } + } + } + + XLSX.utils.book_append_sheet(this.workbook, this.worksheet, 'Data'); + return this.workbook; + } + + private renderHeaders(columns: Array) { + return columns.map((col) => { + const fieldInstance = this.findFieldByDataIndex(col.dataIndex); + return col.title || fieldInstance?.options.title || col.defaultTitle; + }); + } +} + +function isNumeric(n) { + return !isNaN(parseFloat(n)) && isFinite(n); +} diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/xlsx-exporter.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/xlsx-exporter.ts deleted file mode 100644 index 5ff47dda049..00000000000 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/xlsx-exporter.ts +++ /dev/null @@ -1,221 +0,0 @@ -/** - * This file is part of the NocoBase (R) project. - * Copyright (c) 2020-2024 NocoBase Co., Ltd. - * Authors: NocoBase Team. - * - * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. - * For more information, please refer to: https://www.nocobase.com/agreement. - */ - -import { - FindOptions, - ICollection, - ICollectionManager, - IField, - IModel, - IRelationField, -} from '@nocobase/data-source-manager'; - -import XLSX from 'xlsx'; -import { deepGet } from './utils/deep-get'; -import { NumberField } from '@nocobase/database'; - -type ExportColumn = { - dataIndex: Array; - title?: string; - defaultTitle: string; -}; - -type ExportOptions = { - collectionManager: ICollectionManager; - collection: ICollection; - repository?: any; - columns: Array; - findOptions?: FindOptions; - chunkSize?: number; -}; - -class XlsxExporter { - /** - * You can adjust the maximum number of exported rows based on business needs and system - * available resources. However, please note that you need to fully understand the risks - * after the modification. Increasing the maximum number of rows that can be exported may - * increase system resource usage, leading to increased processing delays for other - * requests, or even server processes being recycled by the operating system. - * - * 您可以根据业务需求和系统可用资源等参数，调整最大导出数量的限制。但请注意，您需要充分了解修改之后的风险， - * 增加最大可导出的行数可能会导致系统资源占用率升高，导致其他请求处理延迟增加、无法处理、甚至 - * 服务端进程被操作系统回收等问题。 - */ - limit = process.env['EXPORT_LIMIT'] ? parseInt(process.env['EXPORT_LIMIT']) : 2000; - - constructor(private options: ExportOptions) {} - - async run(ctx?): Promise { - const { collection, columns, chunkSize, repository } = this.options; - - const workbook = XLSX.utils.book_new(); - const worksheet = XLSX.utils.sheet_new(); - - // write headers - XLSX.utils.sheet_add_aoa(worksheet, [this.renderHeaders()], { - origin: 'A1', - }); - - let startRowNumber = 2; - - await (repository || collection.repository).chunk({ - ...this.getFindOptions(), - chunkSize: chunkSize || 200, - callback: async (rows, options) => { - const chunkData = rows.map((r) => { - return columns.map((col) => { - return this.renderCellValue(r, col, ctx); - }); - }); - - XLSX.utils.sheet_add_aoa(worksheet, chunkData, { - origin: `A${startRowNumber}`, - }); - - startRowNumber += rows.length; - - await new Promise((resolve) => { - setTimeout(resolve, 50); - }); - }, - }); - - for (const col of columns) { - const field = this.findFieldByDataIndex(col.dataIndex); - if (field instanceof NumberField) { - // set column cell type to number - const colIndex = columns.indexOf(col); - const cellRange = XLSX.utils.decode_range(worksheet['!ref']); - - for (let r = 1; r <= cellRange.e.r; r++) { - const cell = worksheet[XLSX.utils.encode_cell({ c: colIndex, r })]; - // if cell and cell.v is a number, set cell.t to 'n' - if (cell && isNumeric(cell.v)) { - cell.t = 'n'; - } - } - } - } - - XLSX.utils.book_append_sheet(workbook, worksheet, 'Data'); - return workbook; - } - - private getAppendOptionsFromColumns() { - return this.options.columns - .map((col) => { - if (col.dataIndex.length > 1) { - return col.dataIndex.join('.'); - } - - const field = this.options.collection.getField(col.dataIndex[0]); - if (!field) { - throw new Error(`Field "${col.dataIndex[0]}" not found: , please check the columns configuration.`); - } - - if (field.isRelationField()) { - return col.dataIndex[0]; - } - - return null; - }) - .filter(Boolean); - } - - private getFindOptions() { - const { findOptions = {} } = this.options; - - findOptions.limit = this.limit; - - const appendOptions = this.getAppendOptionsFromColumns(); - - if (appendOptions.length) { - return { - ...findOptions, - appends: appendOptions, - }; - } - - return findOptions; - } - - private findFieldByDataIndex(dataIndex: Array): IField { - const { collection } = this.options; - const currentField = collection.getField(dataIndex[0]); - - if (dataIndex.length > 1) { - let targetCollection: ICollection; - - for (let i = 0; i < dataIndex.length; i++) { - const isLast = i === dataIndex.length - 1; - - if (isLast) { - return targetCollection.getField(dataIndex[i]); - } - - targetCollection = (currentField as IRelationField).targetCollection(); - } - } - - return currentField; - } - - private renderHeaders() { - return this.options.columns.map((col) => { - const field = this.findFieldByDataIndex(col.dataIndex); - if (col.title) { - return col.title; - } - - return field?.options.title || col.defaultTitle; - }); - } - - private renderRawValue(value) { - if (typeof value === 'object' && value !== null) { - return JSON.stringify(value); - } - - return value; - } - - private getFieldRenderer(field?: IField, ctx?): (value) => any { - const InterfaceClass = this.options.collectionManager.getFieldInterface(field?.options?.interface); - if (!InterfaceClass) { - return this.renderRawValue; - } - const fieldInternface = new InterfaceClass(field?.options); - return (value) => fieldInternface.toString(value, ctx); - } - - private renderCellValue(rowData: IModel, column: ExportColumn, ctx?) { - const { dataIndex } = column; - rowData = rowData.toJSON(); - const value = rowData[dataIndex[0]]; - const field = this.findFieldByDataIndex(dataIndex); - const render = this.getFieldRenderer(field, ctx); - - if (dataIndex.length > 1) { - const deepValue = deepGet(rowData, dataIndex); - - if (Array.isArray(deepValue)) { - return deepValue.map(render).join(','); - } - - return render(deepValue); - } - return render(value); - } -} - -function isNumeric(n) { - return !isNaN(parseFloat(n)) && isFinite(n); -} - -export default XlsxExporter; diff --git a/packages/plugins/@nocobase/plugin-action-import/package.json b/packages/plugins/@nocobase/plugin-action-import/package.json index f7c9b847952..3ec994a31f4 100644 --- a/packages/plugins/@nocobase/plugin-action-import/package.json +++ b/packages/plugins/@nocobase/plugin-action-import/package.json @@ -25,7 +25,8 @@ "react": "^18.2.0", "react-dom": "^18.2.0", "react-i18next": "^11.15.1", - "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz", + "exceljs": "^4.4.0" }, "peerDependencies": { "@nocobase/actions": "1.x", diff --git a/packages/plugins/@nocobase/plugin-action-import/src/client/ImportAction.tsx b/packages/plugins/@nocobase/plugin-action-import/src/client/ImportAction.tsx new file mode 100644 index 00000000000..b278b362b12 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-import/src/client/ImportAction.tsx @@ -0,0 +1,130 @@ +import { css } from '@emotion/css'; +import { ISchema, useFieldSchema } from '@formily/react'; +import { Action, ActionContextProvider, SchemaComponent } from '@nocobase/client'; +import React, { useState } from 'react'; +import { NAMESPACE } from './constants'; +import { useTranslation } from 'react-i18next'; +import { UploadOutlined } from '@ant-design/icons'; + +const importFormSchema: ISchema = { + type: 'void', + name: 'import-modal', + title: `{{ t("Import Data", {ns: "${NAMESPACE}" }) }}`, + 'x-component': 'Action.Modal', + 'x-decorator': 'Form', + 'x-component-props': { + width: '100%', + style: { + maxWidth: '750px', + }, + className: css` + .ant-formily-item-label { + height: var(--controlHeightLG); + } + `, + }, + properties: { + formLayout: { + type: 'void', + 'x-component': 'FormLayout', + properties: { + warning: { + type: 'void', + 'x-component': 'ImportWarning', + }, + download: { + type: 'void', + title: `{{ t("Step 1: Download template", {ns: "${NAMESPACE}" }) }}`, + 'x-component': 'FormItem', + 'x-acl-ignore': true, + properties: { + tip: { + type: 'void', + 'x-component': 'DownloadTips', + }, + downloadAction: { + type: 'void', + title: `{{ t("Download template", {ns: "${NAMESPACE}" }) }}`, + 'x-component': 'Action', + 'x-component-props': { + className: css` + margin-top: 5px; + `, + useAction: '{{ useDownloadXlsxTemplateAction }}', + }, + }, + }, + }, + upload: { + type: 'array', + title: `{{ t("Step 2: Upload Excel", {ns: "${NAMESPACE}" }) }}`, + 'x-decorator': 'FormItem', + 'x-acl-ignore': true, + 'x-component': 'Upload.Dragger', + 'x-validator': '{{ uploadValidator }}', + 'x-component-props': { + action: '', + height: '150px', + tipContent: `{{ t("Upload placeholder", {ns: "${NAMESPACE}" }) }}`, + beforeUpload: '{{ beforeUploadHandler }}', + }, + }, + }, + }, + footer: { + 'x-component': 'Action.Modal.Footer', + 'x-component-props': {}, + properties: { + actions: { + type: 'void', + 'x-component': 'ActionBar', + 'x-component-props': {}, + properties: { + cancel: { + type: 'void', + title: '{{ t("Cancel") }}', + 'x-component': 'Action', + 'x-component-props': { + useAction: '{{ cm.useCancelAction }}', + }, + }, + startImport: { + type: 'void', + title: `{{ t("Start import", {ns: "${NAMESPACE}" }) }}`, + 'x-component': 'Action', + 'x-component-props': { + type: 'primary', + htmlType: 'submit', + useAction: '{{ useImportStartAction }}', + }, + 'x-reactions': { + dependencies: ['upload'], + fulfill: { + run: 'validateUpload($form, $self, $deps)', + }, + }, + }, + }, + }, + }, + }, + }, +}; + +export const ImportAction = (props) => { + const [visible, setVisible] = useState(false); + const { t } = useTranslation(NAMESPACE); + const fieldSchema = useFieldSchema(); + + return ( + + } + title={fieldSchema?.title || t('Import', { ns: 'action-import' })} + {...props} + onClick={() => setVisible(true)} + /> + + + ); +}; diff --git a/packages/plugins/@nocobase/plugin-action-import/src/client/ImportActionContext.tsx b/packages/plugins/@nocobase/plugin-action-import/src/client/ImportActionContext.tsx new file mode 100644 index 00000000000..e0ca0d3ea80 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-import/src/client/ImportActionContext.tsx @@ -0,0 +1,8 @@ +import React from 'react'; +import { Schema } from '@formily/react'; + +export const ImportActionContext = React.createContext({}); + +export const useImportActionContext = () => { + return React.useContext(ImportActionContext); +}; diff --git a/packages/plugins/@nocobase/plugin-action-import/src/client/ImportActionInitializer.tsx b/packages/plugins/@nocobase/plugin-action-import/src/client/ImportActionInitializer.tsx index ffa3c4b03e7..549184e023f 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/client/ImportActionInitializer.tsx +++ b/packages/plugins/@nocobase/plugin-action-import/src/client/ImportActionInitializer.tsx @@ -11,17 +11,17 @@ import type { ISchema } from '@formily/react'; import { Schema } from '@formily/react'; import { merge } from '@formily/shared'; import { - css, SchemaInitializerItem, useCollection_deprecated, useSchemaInitializer, useSchemaInitializerItem, } from '@nocobase/client'; -import { Alert } from 'antd'; import React from 'react'; import { NAMESPACE } from './constants'; import { useImportTranslation } from './locale'; import { useFields } from './useFields'; +import { Alert } from 'antd'; +import { lodash } from '@nocobase/utils/client'; const findSchema = (schema: Schema, key: string, action: string) => { return schema.reduceProperties((buf, s) => { @@ -36,11 +36,38 @@ const findSchema = (schema: Schema, key: string, action: string) => { }); }; -const initImportSettings = (fields) => { +export const initImportSettings = (fields) => { const importColumns = fields?.filter((f) => !f.children).map((f) => ({ dataIndex: [f.name] })); return { importColumns, explain: '' }; }; +export const ImportActionInitializer = () => { + const itemConfig = useSchemaInitializerItem(); + const { insert } = useSchemaInitializer(); + const { name } = useCollection_deprecated(); + const fields = useFields(name); + + const schema: ISchema = { + type: 'void', + title: '{{ t("Import") }}', + 'x-component': 'ImportAction', + 'x-action': 'importXlsx', + 'x-settings': 'actionSettings:import', + }; + + return ( + { + lodash.set(schema, 'x-action-settings.importSettings', initImportSettings(fields)); + const s = merge(schema || {}, itemConfig.schema || {}); + itemConfig?.schemaInitialize?.(s); + insert(s); + }} + /> + ); +}; + export const ImportWarning = () => { const { t } = useImportTranslation(); return ; @@ -50,142 +77,3 @@ export const DownloadTips = () => { const { t } = useImportTranslation(); return ; }; - -export const ImportActionInitializer = () => { - const itemConfig = useSchemaInitializerItem(); - const { insert } = useSchemaInitializer(); - const { name } = useCollection_deprecated(); - const fields = useFields(name); - const schema: ISchema = { - type: 'void', - title: '{{ t("Import") }}', - 'x-action': 'importXlsx', - 'x-action-settings': { - importSettings: { importColumns: [], explain: '' }, - }, - 'x-toolbar': 'ActionSchemaToolbar', - 'x-settings': 'actionSettings:import', - 'x-component': 'Action', - 'x-component-props': { - icon: 'CloudUploadOutlined', - openMode: 'modal', - }, - properties: { - modal: { - type: 'void', - title: `{{ t("Import Data", {ns: "${NAMESPACE}" }) }}`, - 'x-component': 'Action.Modal', - 'x-decorator': 'Form', - 'x-component-props': { - width: '100%', - style: { - maxWidth: '750px', - }, - className: css` - .ant-formily-item-label { - height: var(--controlHeightLG); - } - `, - }, - properties: { - formLayout: { - type: 'void', - 'x-component': 'FormLayout', - properties: { - warning: { - type: 'void', - 'x-component': 'ImportWarning', - }, - download: { - type: 'void', - title: `{{ t("Step 1: Download template", {ns: "${NAMESPACE}" }) }}`, - 'x-component': 'FormItem', - 'x-acl-ignore': true, - properties: { - tip: { - type: 'void', - 'x-component': 'DownloadTips', - }, - downloadAction: { - type: 'void', - title: `{{ t("Download template", {ns: "${NAMESPACE}" }) }}`, - 'x-component': 'Action', - 'x-component-props': { - className: css` - margin-top: 5px; - `, - useAction: '{{ useDownloadXlsxTemplateAction }}', - }, - }, - }, - }, - upload: { - type: 'array', - title: `{{ t("Step 2: Upload Excel", {ns: "${NAMESPACE}" }) }}`, - 'x-decorator': 'FormItem', - 'x-acl-ignore': true, - 'x-component': 'Upload.Dragger', - 'x-validator': '{{ uploadValidator }}', - 'x-component-props': { - action: '', - height: '150px', - tipContent: `{{ t("Upload placeholder", {ns: "${NAMESPACE}" }) }}`, - beforeUpload: '{{ beforeUploadHandler }}', - }, - }, - }, - }, - footer: { - 'x-component': 'Action.Modal.Footer', - 'x-component-props': {}, - properties: { - actions: { - type: 'void', - 'x-component': 'ActionBar', - 'x-component-props': {}, - properties: { - cancel: { - type: 'void', - title: '{{ t("Cancel") }}', - 'x-component': 'Action', - 'x-component-props': { - useAction: '{{ cm.useCancelAction }}', - }, - }, - startImport: { - type: 'void', - title: `{{ t("Start import", {ns: "${NAMESPACE}" }) }}`, - 'x-component': 'Action', - 'x-component-props': { - type: 'primary', - htmlType: 'submit', - useAction: '{{ useImportStartAction }}', - }, - 'x-reactions': { - dependencies: ['upload'], - fulfill: { - run: 'validateUpload($form, $self, $deps)', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }; - - return ( - { - schema['x-action-settings']['importSettings'] = initImportSettings(fields); - const s = merge(schema || {}, itemConfig.schema || {}); - itemConfig?.schemaInitialize?.(s); - insert(s); - }} - /> - ); -}; diff --git a/packages/plugins/@nocobase/plugin-action-import/src/client/ImportPluginProvider.tsx b/packages/plugins/@nocobase/plugin-action-import/src/client/ImportPluginProvider.tsx index d7e2b76e8d0..a80da79d683 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/client/ImportPluginProvider.tsx +++ b/packages/plugins/@nocobase/plugin-action-import/src/client/ImportPluginProvider.tsx @@ -15,12 +15,13 @@ import { ImportContext } from './context'; import { ImportModal, ImportStatus } from './ImportModal'; import { useDownloadXlsxTemplateAction, useImportStartAction } from './useImportAction'; import { useShared } from './useShared'; +import { ImportAction } from './ImportAction'; export const ImportPluginProvider = (props: any) => { const { uploadValidator, beforeUploadHandler, validateUpload } = useShared(); return ( { @@ -51,6 +57,7 @@ export const importActionSchemaSettings = new SchemaSettings({ .map((item) => ({ dataIndex: item.dataIndex.map((di) => di.name ?? di), title: item.title, + description: item.description, })); fieldSchema['x-action-settings']['importSettings'] = { importColumns: columns, explain }; diff --git a/packages/plugins/@nocobase/plugin-action-import/src/client/useDownloadXlsxTemplateAction.ts b/packages/plugins/@nocobase/plugin-action-import/src/client/useDownloadXlsxTemplateAction.ts new file mode 100644 index 00000000000..e95fe07b2bd --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-import/src/client/useDownloadXlsxTemplateAction.ts @@ -0,0 +1,46 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { + useAPIClient, + useRecord, + useBlockRequestContext, + useCollection_deprecated, + useCompile, +} from '@nocobase/client'; +import { saveAs } from 'file-saver'; + +export const useDownloadXlsxTemplateAction = () => { + const { resource } = useBlockRequestContext(); + const compile = useCompile(); + const record = useRecord(); + const { title } = useCollection_deprecated(); + + return { + async run() { + const { explain, importColumns } = record; + const { data } = await resource.downloadXlsxTemplate( + { + values: { + title: compile(title), + explain, + columns: compile(importColumns), + }, + }, + { + method: 'post', + responseType: 'blob', + }, + ); + + const blob = new Blob([data], { type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' }); + saveAs(blob, `${compile(title)}.xlsx`); + }, + }; +}; diff --git a/packages/plugins/@nocobase/plugin-action-import/src/client/useImportAction.ts b/packages/plugins/@nocobase/plugin-action-import/src/client/useImportAction.ts index 0ba50aa89b0..2803efafaa9 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/client/useImportAction.ts +++ b/packages/plugins/@nocobase/plugin-action-import/src/client/useImportAction.ts @@ -26,11 +26,18 @@ import { NAMESPACE } from './constants'; import { useImportContext } from './context'; import { ImportStatus } from './ImportModal'; import { useEffect } from 'react'; +import { useFields } from './useFields'; +import { initImportSettings } from './ImportActionInitializer'; +import { useImportActionContext } from './ImportActionContext'; -const useImportSchema = (s: Schema) => { - let schema = s; +const useImportSchema = () => { + const { fieldSchema: actionSchema } = useActionContext(); + const fieldSchema = useFieldSchema(); + + let schema = actionSchema || fieldSchema; while (schema && schema['x-action'] !== 'importXlsx') { schema = schema.parent; + console.log('schema', schema); } return { schema }; }; @@ -50,7 +57,8 @@ export const useDownloadXlsxTemplateAction = () => { const { getCollectionJoinField, getCollectionField } = useCollectionManager_deprecated(); const { name, title, getField } = useCollection_deprecated(); const { t } = useTranslation(NAMESPACE); - const { schema: importSchema } = useImportSchema(actionSchema); + const { schema: importSchema } = useImportSchema(); + return { async run() { const { importColumns, explain } = lodash.cloneDeep( @@ -103,18 +111,19 @@ export const useImportStartAction = () => { const { getCollectionJoinField, getCollectionField } = useCollectionManager_deprecated(); const { name, title, getField } = useCollection_deprecated(); const { t } = useTranslation(NAMESPACE); - const { schema: importSchema } = useImportSchema(actionSchema); + const { schema: importSchema } = useImportSchema(); const form = useForm(); const { setVisible, fieldSchema } = useActionContext(); const { setImportModalVisible, setImportStatus, setImportResult } = useImportContext(); const { upload } = form.values; - const dataBlockProps = useDataBlockProps() || ({} as any); + const dataBlockProps = useDataBlockProps(); const headers = useDataSourceHeaders(dataBlockProps.dataSource); const newResource = useDataBlockResource(); useEffect(() => { form.reset(); }, []); + return { async run() { const { importColumns, explain } = lodash.cloneDeep( @@ -140,11 +149,30 @@ export const useImportStartAction = () => { return column; }) .filter(Boolean); + const formData = new FormData(); + const uploadFiles = form.values.upload.map((f) => f.originFileObj); formData.append('file', uploadFiles[0]); formData.append('columns', JSON.stringify(columns)); formData.append('explain', explain); + + const { triggerWorkflow, identifyDuplicates, uniqueField, duplicateStrategy } = form.values; + + if (triggerWorkflow !== undefined) { + formData.append('triggerWorkflow', JSON.stringify(triggerWorkflow)); + } + + if (identifyDuplicates) { + formData.append( + 'duplicateOption', + JSON.stringify({ + uniqueField, + mode: duplicateStrategy, + }), + ); + } + setVisible(false); setImportModalVisible(true); setImportStatus(ImportStatus.IMPORTING); @@ -161,8 +189,15 @@ export const useImportStartAction = () => { setImportResult(data); form.reset(); - await service?.refresh?.(); - setImportStatus(ImportStatus.IMPORTED); + + if (!data.data.taskId) { + setImportResult(data); + await service?.refresh?.(); + setImportStatus(ImportStatus.IMPORTED); + } else { + setImportModalVisible(false); + setVisible(false); + } } catch (error) { setImportModalVisible(false); setVisible(true); diff --git a/packages/plugins/@nocobase/plugin-action-import/src/client/useShared.ts b/packages/plugins/@nocobase/plugin-action-import/src/client/useShared.ts index 8c826a72bca..f3dd38acc8f 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/client/useShared.ts +++ b/packages/plugins/@nocobase/plugin-action-import/src/client/useShared.ts @@ -73,6 +73,22 @@ export const useShared = () => { changeOnSelect: false, }, }, + title: { + type: 'string', + 'x-decorator': 'FormItem', + 'x-component': 'Input', + 'x-component-props': { + placeholder: '{{ t("Custom column title") }}', + }, + }, + description: { + type: 'string', + 'x-decorator': 'FormItem', + 'x-component': 'Input', + 'x-component-props': { + placeholder: `{{ t("Field description placeholder", {ns: "${NAMESPACE}"}) }}`, + }, + }, remove: { type: 'void', 'x-decorator': 'FormItem', diff --git a/packages/plugins/@nocobase/plugin-action-import/src/index.ts b/packages/plugins/@nocobase/plugin-action-import/src/index.ts index 2c62180bd72..752bff68408 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/index.ts +++ b/packages/plugins/@nocobase/plugin-action-import/src/index.ts @@ -9,6 +9,7 @@ // @ts-ignore import { name } from '../package.json'; + export * from './server'; export { default } from './server'; export const namespace = name; diff --git a/packages/plugins/@nocobase/plugin-action-import/src/locale/en-US.json b/packages/plugins/@nocobase/plugin-action-import/src/locale/en-US.json index 1cb15d0e738..fc02bec0d3f 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/locale/en-US.json +++ b/packages/plugins/@nocobase/plugin-action-import/src/locale/en-US.json @@ -8,6 +8,7 @@ "Download template": "Download template", "Step 1: Download template": "Step 1: Download template", "Step 2: Upload Excel": "Step 2: Upload Excel", + "Step 3: Import options": "Step 3: Import options", "Download tips": "- Download the template and fill in the data according to the format \r\n - Import only the first worksheet \r\n - Do not change the header of the template to prevent import failure", "Import warnings": "You can import up to {{limit}} rows of data at a time, any excess will be ignored.", "Upload placeholder": "Drag and drop the file here or click to upload, file size should not exceed 30M", @@ -24,5 +25,18 @@ "Incorrect time format": "Incorrect time format", "Incorrect date format": "Incorrect date format", "Incorrect email format": "Incorrect email format", - "Illegal percentage format": "Illegal percentage format" + "Illegal percentage format": "Illegal percentage format", + "Imported template does not match, please download again.": "Imported template does not match, please download again.", + "another import action is running, please try again later.": "another import action is running, please try again later.", + "Please select": "Please select", + "Custom column title": "Custom column title", + "Field description": "Field description", + "Field description placeholder": "Enter field description", + "Columns configuration is empty": "Columns configuration is empty", + "Field not found: {{field}}": "Field not found: {{field}}", + "Headers not found. Expected headers: {{headers}}": "Headers not found. Expected headers: {{headers}}", + "Header mismatch at column {{column}}: expected \"{{expected}}\", but got \"{{actual}}\"": "Header mismatch at column {{column}}: expected \"{{expected}}\", but got \"{{actual}}\"", + "No data to import": "No data to import", + "Failed to import row {{row}}, {{message}}, row data: {{data}}": "Failed to import row {{row}}, {{message}}, row data: {{data}}", + "import-error": "Failed to import row {{rowIndex}}, row data: {{rowData}}, cause: {{causeMessage}}" } diff --git a/packages/plugins/@nocobase/plugin-action-import/src/locale/zh-CN.json b/packages/plugins/@nocobase/plugin-action-import/src/locale/zh-CN.json index 1c5083c3ea5..b93af7092fa 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/locale/zh-CN.json +++ b/packages/plugins/@nocobase/plugin-action-import/src/locale/zh-CN.json @@ -3,11 +3,13 @@ "File size cannot exceed 10M": "文件大小不能超过10M", "Please upload the file of Excel": "请上传Excel的文件", "Import Data": "导入数据", + "Import": "导入", "Start import": "开始导入", "Import explain": "说明", "Download template": "下载模板", "Step 1: Download template": "1.下载模板", "Step 2: Upload Excel": "2.上传完善后的表格", + "Step 3: Import options": "3.导入选项", "Download tips": "- 下载模板后，按格式填写数据\r\n - 只导入第一张工作表\r\n - 请勿改模板表头，防止导入失败", "Import warnings": "每次最多导入 {{limit}} 行数据，超出的将被忽略。", "Upload placeholder": "将文件拖曳到此处或点击上传，文件大小不超过10M", @@ -18,6 +20,7 @@ "Done": "完成", "Yes": "是", "No": "否", + "Please select": "请选择", "Field {{fieldName}} does not exist": "字段 {{fieldName}} 不存在", "can not find value": "找不到对应值", "password is empty": "密码为空", @@ -26,5 +29,15 @@ "Incorrect email format": "邮箱格式不正确", "Illegal percentage format": "百分比格式有误", "Imported template does not match, please download again.": "导入模板不匹配，请检查导入文件标题行或重新下载导入模板", - "another import action is running, please try again later.": "另一导入任务正在运行，请稍后重试。" + "another import action is running, please try again later.": "另一导入任务正在运行，请稍后重试。", + "Custom column title": "自定义列标题", + "Field description": "字段说明", + "Field description placeholder": "请输入字段说明", + "Columns configuration is empty": "列配置为空", + "Field not found: {{field}}": "字段未找到：{{field}}", + "Headers not found. Expected headers: {{headers}}": "未找到表头。预期的表头：{{headers}}", + "Header mismatch at column {{column}}: expected \"{{expected}}\", but got \"{{actual}}\"": "第 {{column}} 列的表头不匹配：预期 \"{{expected}}\"，实际是 \"{{actual}}\"", + "No data to import": "没有数据可导入", + "Failed to import row {{row}}, {{message}}, row data: {{data}}": "导入第 {{row}} 行失败，{{message}}，行数据：{{data}}", + "import-error": "导入第 {{rowIndex}} 行失败，行数据：{{rowData}}, 原因：{{causeMessage}}" } diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/download-template.test.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/download-template.test.ts index 4ecbb0f5ffb..06d658d31c8 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/download-template.test.ts +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/download-template.test.ts @@ -56,7 +56,7 @@ describe('download template', () => { ], }); - const workbook = await templateCreator.run(); + const workbook = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const sheet0 = workbook.Sheets[workbook.SheetNames[0]]; const sheetData = XLSX.utils.sheet_to_json(sheet0, { header: 1, defval: null, raw: false }); @@ -64,7 +64,8 @@ describe('download template', () => { expect(explainData[0]).toEqual(explain); const headerData = sheetData[1]; - expect(headerData).toEqual(['Name', 'Email']); + expect(headerData[0]).toEqual('Name'); + expect(headerData[1]).toEqual('Email'); }); it('should render template', async () => { @@ -99,7 +100,7 @@ describe('download template', () => { ], }); - const workbook = await templateCreator.run(); + const workbook = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const sheet0 = workbook.Sheets[workbook.SheetNames[0]]; const sheetData = XLSX.utils.sheet_to_json(sheet0, { header: 1, defval: null, raw: false }); const headerData = sheetData[0]; diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/xlsx-importer.test.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/xlsx-importer.test.ts index d5bb10dd9b4..dcf022b3042 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/xlsx-importer.test.ts +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/xlsx-importer.test.ts @@ -26,6 +26,85 @@ describe('xlsx importer', () => { await app.destroy(); }); + describe('validate empty data', () => { + let User; + + beforeEach(async () => { + User = app.db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + }, + ], + }); + await app.db.sync(); + }); + + it('should throw error when file only has header row', async () => { + const templateCreator = new TemplateCreator({ + collection: User, + columns: [ + { + dataIndex: ['name'], + defaultTitle: 'Name', + }, + ], + }); + + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; + // template already has header row, no need to add data + + const importer = new XlsxImporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + columns: [ + { + dataIndex: ['name'], + defaultTitle: 'Name', + }, + ], + workbook: template, + }); + + await expect(importer.validate()).rejects.toThrow('No data to import'); + }); + + it('should pass validation when file has header and data rows', async () => { + const templateCreator = new TemplateCreator({ + collection: User, + columns: [ + { + dataIndex: ['name'], + defaultTitle: 'Name', + }, + ], + }); + + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; + const worksheet = template.Sheets[template.SheetNames[0]]; + + XLSX.utils.sheet_add_aoa(worksheet, [['Test Data 1'], ['Test Data 2']], { + origin: 'A2', + }); + + const importer = new XlsxImporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + columns: [ + { + dataIndex: ['name'], + defaultTitle: 'Name', + }, + ], + workbook: template, + }); + + await expect(importer.validate()).resolves.toBeUndefined(); + }); + }); + describe('import with date field', () => { let User; @@ -96,7 +175,7 @@ describe('xlsx importer', () => { columns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -142,7 +221,7 @@ describe('xlsx importer', () => { columns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -188,7 +267,7 @@ describe('xlsx importer', () => { columns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -234,7 +313,7 @@ describe('xlsx importer', () => { columns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -336,7 +415,7 @@ describe('xlsx importer', () => { columns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -378,7 +457,7 @@ describe('xlsx importer', () => { columns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -420,7 +499,7 @@ describe('xlsx importer', () => { columns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -524,7 +603,7 @@ describe('xlsx importer', () => { columns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -566,7 +645,7 @@ describe('xlsx importer', () => { columns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -686,7 +765,7 @@ describe('xlsx importer', () => { columns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -737,7 +816,7 @@ describe('xlsx importer', () => { columns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -780,7 +859,7 @@ describe('xlsx importer', () => { columns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -839,7 +918,7 @@ describe('xlsx importer', () => { ], }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -872,7 +951,6 @@ describe('xlsx importer', () => { } expect(error).toBeTruthy(); - console.log(error.message); }); it('should report validation error message on unique validation', async () => { @@ -907,7 +985,7 @@ describe('xlsx importer', () => { ], }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; XLSX.utils.sheet_add_aoa( @@ -946,7 +1024,6 @@ describe('xlsx importer', () => { } expect(error).toBeTruthy(); - console.log(error.message); }); it('should import china region field', async () => { @@ -980,7 +1057,7 @@ describe('xlsx importer', () => { columns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -1078,7 +1155,7 @@ describe('xlsx importer', () => { columns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -1168,7 +1245,7 @@ describe('xlsx importer', () => { ], }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -1269,7 +1346,7 @@ describe('xlsx importer', () => { ], }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -1372,7 +1449,7 @@ describe('xlsx importer', () => { let error; try { - importer.getData(); + await importer.validate(); } catch (e) { error = e; } @@ -1409,7 +1486,7 @@ describe('xlsx importer', () => { ], }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const importer = new XlsxImporter({ collectionManager: app.mainDataSource.collectionManager, @@ -1427,9 +1504,15 @@ describe('xlsx importer', () => { workbook: template, }); + // insert a row for test + const worksheet = template.Sheets[template.SheetNames[0]]; + XLSX.utils.sheet_add_aoa(worksheet, [['User1', 'test@test.com']], { + origin: 'A2', + }); + let error; try { - importer.getData(); + await importer.getData(); } catch (e) { error = e; } @@ -1567,7 +1650,7 @@ describe('xlsx importer', () => { columns: importColumns, }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -1629,7 +1712,7 @@ describe('xlsx importer', () => { ], }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -1698,7 +1781,7 @@ describe('xlsx importer', () => { ], }); - const template = await templateCreator.run(); + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; const worksheet = template.Sheets[template.SheetNames[0]]; @@ -1739,4 +1822,249 @@ describe('xlsx importer', () => { expect(await User.repository.count()).toBe(0); expect(error).toBeTruthy(); }); + + it('should import data with multiline explain and field descriptions', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + }, + { + type: 'string', + name: 'email', + }, + ], + }); + + await app.db.sync(); + + const templateCreator = new TemplateCreator({ + collection: User, + explain: '这是第一行说明\n这是第二行说明', + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + description: '请输入用户姓名', + }, + { + dataIndex: ['email'], + defaultTitle: '邮箱', + description: '请输入有效的邮箱地址', + }, + ], + }); + + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; + + const headerRowIndex = templateCreator.getHeaderRowIndex(); + + console.log({ headerRowIndex }); + const worksheet = template.Sheets[template.SheetNames[0]]; + + XLSX.utils.sheet_add_aoa( + worksheet, + [ + ['User1', 'test@test.com'], + ['User2', 'test2@test.com'], + ], + { + origin: `A${headerRowIndex + 1}`, + }, + ); + + const importer = new XlsxImporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + description: '请输入用户姓名', + }, + { + dataIndex: ['email'], + defaultTitle: '邮箱', + description: '请输入有效的邮箱地址', + }, + ], + workbook: template, + }); + + await importer.run(); + + expect(await User.repository.count()).toBe(2); + + const users = await User.repository.find(); + expect(users[0].get('name')).toBe('User1'); + expect(users[0].get('email')).toBe('test@test.com'); + expect(users[1].get('name')).toBe('User2'); + expect(users[1].get('email')).toBe('test2@test.com'); + }); + + describe('template creator', () => { + it('should create template with explain and field descriptions', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + }, + { + type: 'string', + name: 'email', + }, + ], + }); + + const templateCreator = new TemplateCreator({ + collection: User, + explain: '这是导入说明\n第二行说明', + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + description: '请输入用户姓名', + }, + { + dataIndex: ['email'], + defaultTitle: '邮箱', + description: '请输入有效的邮箱地址', + }, + ], + }); + + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; + const worksheet = template.Sheets[template.SheetNames[0]]; + const data = XLSX.utils.sheet_to_json(worksheet, { header: 1 }) as string[][]; + + // 验证说明文本 + expect(data[0][0]).toBe('这是导入说明'); + expect(data[1][0]).toBe('第二行说明'); + // 验证空行 + expect(data[2][0]).toBe(''); + // 验证字段描述 + expect(data[3][0]).toBe('姓名：请输入用户姓名'); + expect(data[4][0]).toBe('邮箱：请输入有效的邮箱地址'); + // 验证表头 + expect(data[5]).toEqual(['姓名', '邮箱']); + }); + + it('should create template with only field descriptions', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + }, + ], + }); + + const templateCreator = new TemplateCreator({ + collection: User, + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + description: '请输入用户姓名', + }, + ], + }); + + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; + const worksheet = template.Sheets[template.SheetNames[0]]; + const data = XLSX.utils.sheet_to_json(worksheet, { header: 1 }) as string[][]; + + // 验证字段描述 + expect(data[0][0]).toBe('姓名：请输入用户姓名'); + // 验证表头 + expect(data[1]).toEqual(['姓名']); + }); + + it('should create template with only explain', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + }, + ], + }); + + const templateCreator = new TemplateCreator({ + collection: User, + explain: '这是导入说明', + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + }, + ], + }); + + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; + const worksheet = template.Sheets[template.SheetNames[0]]; + const data = XLSX.utils.sheet_to_json(worksheet, { header: 1 }) as string[][]; + + // 验证说明文本 + expect(data[0][0]).toBe('这是导入说明'); + // 验证表头 + expect(data[1]).toEqual(['姓名']); + }); + }); + + it('should import data with single column', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + }, + ], + }); + + await app.db.sync(); + + const templateCreator = new TemplateCreator({ + collection: User, + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + }, + ], + }); + + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; + const worksheet = template.Sheets[template.SheetNames[0]]; + + XLSX.utils.sheet_add_aoa(worksheet, [['User1'], ['User2']], { + origin: 'A2', + }); + + const importer = new XlsxImporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + }, + ], + workbook: template, + }); + + await importer.run(); + + expect(await User.repository.count()).toBe(2); + const users = await User.repository.find(); + expect(users[0].get('name')).toBe('User1'); + expect(users[1].get('name')).toBe('User2'); + }); }); diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/actions/download-xlsx-template.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/actions/download-xlsx-template.ts index 29750be444c..88b125db457 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/server/actions/download-xlsx-template.ts +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/actions/download-xlsx-template.ts @@ -9,31 +9,24 @@ import { Context, Next } from '@nocobase/actions'; import { TemplateCreator } from '../services/template-creator'; -import XLSX from 'xlsx'; +import { Workbook } from 'exceljs'; export async function downloadXlsxTemplate(ctx: Context, next: Next) { - let { columns } = ctx.request.body as any; - const { explain, title } = ctx.request.body as any; - if (typeof columns === 'string') { - columns = JSON.parse(columns); - } + const { resourceName, values = {} } = ctx.action.params; + const { collection } = ctx.db; const templateCreator = new TemplateCreator({ - explain, - title, - columns, + collection, + ...values, }); const workbook = await templateCreator.run(); - const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' }); + const buffer = await (workbook as Workbook).xlsx.writeBuffer(); + ctx.set('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'); + ctx.set('Content-Disposition', `attachment; filename="${resourceName}-import-template.xlsx"`); ctx.body = buffer; - ctx.set({ - 'Content-Type': 'application/octet-stream', - 'Content-Disposition': `attachment; filename=${encodeURIComponent(title)}.xlsx`, - }); - await next(); } diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/errors.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/errors.ts new file mode 100644 index 00000000000..c28df4c4609 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/errors.ts @@ -0,0 +1,31 @@ +export class ImportValidationError extends Error { + code: string; + params?: Record; + + constructor(code: string, params?: Record) { + super(code); + this.code = code; + this.params = params; + this.name = 'ImportValidationError'; + } +} + +export interface ImportErrorOptions { + rowIndex: number; + rowData: any; + cause?: Error; +} + +export class ImportError extends Error { + rowIndex: number; + rowData: any; + cause?: Error; + + constructor(message: string, options: ImportErrorOptions) { + super(message); + this.name = 'ImportError'; + this.rowIndex = options.rowIndex; + this.rowData = options.rowData; + this.cause = options.cause; + } +} diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/index.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/index.ts index 9613e9e6f2f..2e8c5795074 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/server/index.ts +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/index.ts @@ -10,7 +10,7 @@ import { Plugin } from '@nocobase/server'; import { downloadXlsxTemplate, importXlsx } from './actions'; import { importMiddleware } from './middleware'; - +import { ImportError, ImportValidationError } from './errors'; export class PluginActionImportServer extends Plugin { beforeLoad() { this.app.on('afterInstall', async () => { @@ -55,7 +55,50 @@ export class PluginActionImportServer extends Plugin { dataSource.acl.allow('*', 'downloadXlsxTemplate', 'loggedIn'); }); + + const errorHandlerPlugin = this.app.getPlugin('error-handler'); + + errorHandlerPlugin.errorHandler.register( + (err) => err instanceof ImportValidationError, + (err: ImportValidationError, ctx) => { + ctx.status = 400; + ctx.body = { + errors: [ + { + message: ctx.i18n.t(err.code, { + ...err.params, + ns: 'action-import', + }), + }, + ], + }; + }, + ); + + errorHandlerPlugin.errorHandler.register( + (err) => err.name === 'ImportError', + (err: ImportError, ctx) => { + ctx.status = 400; + const causeError = err.cause; + errorHandlerPlugin.errorHandler.renderError(causeError, ctx); + + ctx.body = { + errors: [ + { + message: ctx.i18n.t('import-error', { + ns: 'action-import', + rowData: err.rowData, + rowIndex: err.rowIndex, + causeMessage: ctx.body.errors[0].message, + }), + }, + ], + }; + }, + ); } } export default PluginActionImportServer; +export * from './services/xlsx-importer'; +export * from './services/template-creator'; diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/services/template-creator.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/services/template-creator.ts index 7d682790069..91656b8793d 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/server/services/template-creator.ts +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/services/template-creator.ts @@ -9,40 +9,153 @@ import { ICollection } from '@nocobase/data-source-manager'; import { ImportColumn } from './xlsx-importer'; -import XLSX, { WorkBook } from 'xlsx'; +import { WorkbookConverter } from '../utils/workbook-converter'; +import * as XLSX from 'xlsx'; +import { Workbook as ExcelJSWorkbook } from 'exceljs'; -type TemplateCreatorOptions = { +export type TemplateCreatorOptions = { collection?: ICollection; title?: string; explain?: string; columns: Array; }; +export type TemplateResult = { + workbook: XLSX.WorkBook | ExcelJSWorkbook; + headerRowIndex: number; +}; + export class TemplateCreator { + private headerRowIndex: number; + constructor(private options: TemplateCreatorOptions) {} - async run(): Promise { - const workbook = XLSX.utils.book_new(); - const worksheet = XLSX.utils.sheet_new(); + getHeaderRowIndex() { + return this.headerRowIndex; + } - const data = [this.renderHeaders()]; + async run(options?: any): Promise { + const workbook = new ExcelJSWorkbook(); + const worksheet = workbook.addWorksheet('Sheet 1'); + + const headers = this.renderHeaders(); + let currentRow = 1; + + let explainText = ''; if (this.options.explain && this.options.explain?.trim() !== '') { - data.unshift([this.options.explain]); + explainText = this.options.explain; } - // write headers - XLSX.utils.sheet_add_aoa(worksheet, data, { - origin: 'A1', + const fieldDescriptions = this.options.columns + .filter((col) => col.description) + .map((col) => `${col.title || col.defaultTitle}：${col.description}`); + + if (fieldDescriptions.length > 0) { + if (explainText) { + explainText += '\n\n'; + } + explainText += fieldDescriptions.join('\n'); + } + + if (explainText.trim() !== '') { + const lines = explainText.split('\n'); + const EXPLAIN_MERGE_COLUMNS = 5; // Default merge 5 columns + + // Pre-set the required number of columns and their widths + const columnsNeeded = Math.max(EXPLAIN_MERGE_COLUMNS, headers.length); + for (let i = 1; i <= columnsNeeded; i++) { + const col = worksheet.getColumn(i); + // Set first column wider + col.width = i === 1 ? 60 : 30; + } + + lines.forEach((line, index) => { + const row = worksheet.getRow(index + 1); + // Merge first 5 cells for explanation text + worksheet.mergeCells(index + 1, 1, index + 1, EXPLAIN_MERGE_COLUMNS); + + const cell = row.getCell(1); + cell.value = line; + + cell.alignment = { + vertical: 'middle', + horizontal: 'left', + indent: 1, + wrapText: true, + }; + }); + + currentRow = lines.length + 1; + } + + // Write headers and set styles + const headerRow = worksheet.getRow(currentRow); + headerRow.height = 25; + headers.forEach((header, index) => { + const cell = headerRow.getCell(index + 1); + cell.value = header; + cell.font = { + bold: true, + size: 10, + name: 'Arial', + }; + cell.border = { + top: { style: 'thin' }, + bottom: { style: 'thin' }, + left: { style: 'thin' }, + right: { style: 'thin' }, + }; + cell.fill = { + type: 'pattern', + pattern: 'solid', + fgColor: { argb: 'FFE9ECEF' }, // Darker gray background for headers + }; + cell.alignment = { + vertical: 'middle', + horizontal: 'left', + indent: 1, + wrapText: true, + }; + + // 如果是第一列，恢复到正常宽度 + if (index === 0) { + worksheet.getColumn(1).width = 30; + } }); - XLSX.utils.book_append_sheet(workbook, worksheet, 'Sheet 1'); + // Set column widths + headers.forEach((_, index) => { + const col = worksheet.getColumn(index + 1); + col.width = 30; + + // Set style for the first column + if (index === 0) { + col.eachCell({ includeEmpty: false }, (cell, rowNumber) => { + // Only set background color for headers, not for other rows + if (rowNumber === currentRow) { + cell.fill = { + type: 'pattern', + pattern: 'solid', + fgColor: { argb: 'FFE9ECEF' }, + }; + } + }); + } + }); + + this.headerRowIndex = currentRow; + + if (options?.returnXLSXWorkbook) { + return await WorkbookConverter.excelJSToXLSX(workbook); + } + return workbook; } renderHeaders() { return this.options.columns.map((col) => { - return col.defaultTitle; + return col.title || col.defaultTitle; }); } } diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/services/xlsx-importer.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/services/xlsx-importer.ts index e5d8a129e8f..bb435490c8c 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/server/services/xlsx-importer.ts +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/services/xlsx-importer.ts @@ -7,29 +7,32 @@ * For more information, please refer to: https://www.nocobase.com/agreement. */ -import XLSX, { WorkBook } from 'xlsx'; +import * as XLSX from 'xlsx'; import lodash from 'lodash'; import { ICollection, ICollectionManager, IRelationField } from '@nocobase/data-source-manager'; import { Collection as DBCollection, Database } from '@nocobase/database'; import { Transaction } from 'sequelize'; import EventEmitter from 'events'; +import { ImportValidationError, ImportError } from '../errors'; export type ImportColumn = { dataIndex: Array; defaultTitle: string; + title?: string; + description?: string; }; -type ImporterOptions = { +export type ImporterOptions = { collectionManager: ICollectionManager; collection: ICollection; columns: Array; - workbook: WorkBook; + workbook: any; chunkSize?: number; explain?: string; repository?: any; }; -type RunOptions = { +export type RunOptions = { transaction?: Transaction; context?: any; }; @@ -37,9 +40,13 @@ type RunOptions = { export class XlsxImporter extends EventEmitter { private repository; - constructor(private options: ImporterOptions) { + constructor(protected options: ImporterOptions) { super(); + if (typeof options.columns === 'string') { + options.columns = JSON.parse(options.columns); + } + if (options.columns.length == 0) { throw new Error(`columns is empty`); } @@ -47,6 +54,21 @@ export class XlsxImporter extends EventEmitter { this.repository = options.repository ? options.repository : options.collection.repository; } + async validate() { + if (this.options.columns.length == 0) { + throw new ImportValidationError('Columns configuration is empty'); + } + + for (const column of this.options.columns) { + const field = this.options.collection.getField(column.dataIndex[0]); + if (!field) { + throw new ImportValidationError('Field not found: {{field}}', { field: column.dataIndex[0] }); + } + } + + await this.getData(); + } + async run(options: RunOptions = {}) { let transaction = options.transaction; @@ -57,6 +79,7 @@ export class XlsxImporter extends EventEmitter { } try { + await this.validate(); const imported = await this.performImport(options); // @ts-ignore @@ -69,7 +92,6 @@ export class XlsxImporter extends EventEmitter { return imported; } catch (error) { transaction && (await transaction.rollback()); - throw error; } } @@ -127,19 +149,21 @@ export class XlsxImporter extends EventEmitter { this.emit('seqReset', { maxVal, seqName: autoIncrInfo.seqName }); } - async performImport(options?: RunOptions) { + async performImport(options?: RunOptions): Promise { const transaction = options?.transaction; - const rows = this.getData(); - const chunks = lodash.chunk(rows, this.options.chunkSize || 200); + const data = await this.getData(); + const chunks = lodash.chunk(data.slice(1), this.options.chunkSize || 200); let handingRowIndex = 1; + let imported = 0; + + // Calculate total rows to be imported + const total = data.length - 1; // Subtract header row if (this.options.explain) { handingRowIndex += 1; } - let imported = 0; - for (const chunkRows of chunks) { for (const row of chunkRows) { const rowValues = {}; @@ -151,7 +175,9 @@ export class XlsxImporter extends EventEmitter { const field = this.options.collection.getField(column.dataIndex[0]); if (!field) { - throw new Error(`Field not found: ${column.dataIndex[0]}`); + throw new ImportValidationError('Import validation.Field not found', { + field: column.dataIndex[0], + }); } const str = row[index]; @@ -185,32 +211,49 @@ export class XlsxImporter extends EventEmitter { rowValues[dataKey] = await interfaceInstance.toValue(this.trimString(str), ctx); } - await this.repository.create({ + await this.performInsert({ values: rowValues, - context: options?.context, transaction, + context: options?.context, }); imported += 1; + // Emit progress event + this.emit('progress', { + total, + current: imported, + }); + await new Promise((resolve) => setTimeout(resolve, 5)); } catch (error) { - throw new Error( - `failed to import row ${handingRowIndex}, ${this.renderErrorMessage(error)}, rowData: ${JSON.stringify( - rowValues, - )}`, - { cause: error }, - ); + throw new ImportError(`Import failed at row ${handingRowIndex}`, { + rowIndex: handingRowIndex, + rowData: Object.entries(rowValues) + .map(([key, value]) => `${key}: ${value}`) + .join(', '), + cause: error, + }); } } - // await to prevent high cpu usage await new Promise((resolve) => setTimeout(resolve, 50)); } return imported; } + async performInsert(insertOptions: { values: any; transaction: Transaction; context: any; hooks?: boolean }) { + const { values, transaction, context } = insertOptions; + + return this.repository.create({ + values, + context, + transaction, + hooks: insertOptions.hooks == undefined ? true : insertOptions.hooks, + }); + } + renderErrorMessage(error) { let message = error.message; if (error.parent) { @@ -227,33 +270,70 @@ export class XlsxImporter extends EventEmitter { return str; } - getData() { - const firstSheet = this.firstSheet(); - const rows = XLSX.utils.sheet_to_json(firstSheet, { header: 1, defval: null }); + private getExpectedHeaders(): string[] { + return this.options.columns.map((col) => col.title || col.defaultTitle); + } - if (this.options.explain) { - rows.shift(); + async getData() { + const workbook = this.options.workbook; + const worksheet = workbook.Sheets[workbook.SheetNames[0]]; + + const data = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: null }) as string[][]; + + // Find and validate header row + const expectedHeaders = this.getExpectedHeaders(); + const { headerRowIndex, headers } = this.findAndValidateHeaders(data); + if (headerRowIndex === -1) { + throw new ImportValidationError('Headers not found. Expected headers: {{headers}}', { + headers: expectedHeaders.join(', '), + }); } - const headers = rows[0]; + // Extract data rows + const rows = data.slice(headerRowIndex + 1); - const columns = this.options.columns; + // if no data rows, throw error + if (rows.length === 0) { + throw new ImportValidationError('No data to import'); + } - // validate headers - for (let i = 0; i < columns.length; i++) { - const column = columns[i]; - if (column.defaultTitle !== headers[i]) { - throw new Error(`Invalid header: ${column.defaultTitle} !== ${headers[i]}`); + return [headers, ...rows]; + } + + private findAndValidateHeaders(data: string[][]): { headerRowIndex: number; headers: string[] } { + const expectedHeaders = this.getExpectedHeaders(); + + // Find header row and validate + for (let rowIndex = 0; rowIndex < data.length; rowIndex++) { + const row = data[rowIndex]; + const actualHeaders = row.filter((cell) => cell !== null && cell !== ''); + + const allHeadersFound = expectedHeaders.every((header) => actualHeaders.includes(header)); + const noExtraHeaders = actualHeaders.length === expectedHeaders.length; + + if (allHeadersFound && noExtraHeaders) { + const mismatchIndex = expectedHeaders.findIndex((title, index) => actualHeaders[index] !== title); + + if (mismatchIndex === -1) { + // All headers match + return { headerRowIndex: rowIndex, headers: actualHeaders }; + } else { + // Found potential header row but with mismatch + throw new ImportValidationError( + 'Header mismatch at column {{column}}: expected "{{expected}}", but got "{{actual}}"', + { + column: mismatchIndex + 1, + expected: expectedHeaders[mismatchIndex], + actual: actualHeaders[mismatchIndex] || 'empty', + }, + ); + } } } - // remove header - rows.shift(); - - return rows; - } - - firstSheet() { - return this.options.workbook.Sheets[this.options.workbook.SheetNames[0]]; + // No row with matching headers found + throw new ImportValidationError('Headers not found. Expected headers: {{headers}}', { + headers: expectedHeaders.join(', '), + }); } } diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/utils/workbook-converter.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/utils/workbook-converter.ts new file mode 100644 index 00000000000..e2ac029c238 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/utils/workbook-converter.ts @@ -0,0 +1,18 @@ +import * as XLSX from 'xlsx'; +import { Workbook as ExcelJSWorkbook } from 'exceljs'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { promisify } from 'util'; + +export class WorkbookConverter { + /** + * Convert ExcelJS Workbook to XLSX Workbook + */ + static async excelJSToXLSX(workbook: ExcelJSWorkbook): Promise { + // Convert ExcelJS workbook to buffer in memory + const buffer = await workbook.xlsx.writeBuffer(); + // Convert buffer to XLSX workbook + return XLSX.read(buffer, { type: 'buffer' }); + } +} diff --git a/packages/plugins/@nocobase/plugin-auth/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-auth/src/server/plugin.ts index 84db4cf684f..201c4ca684e 100644 --- a/packages/plugins/@nocobase/plugin-auth/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-auth/src/server/plugin.ts @@ -112,6 +112,33 @@ export class PluginAuthServer extends Plugin { await this.cache.del(`auth:${userId}`); }); + this.app.on('ws:message:auth:token', async ({ clientId, payload }) => { + const auth = await this.app.authManager.get('basic', { + getBearerToken: () => payload.token, + app: this.app, + db: this.app.db, + cache: this.app.cache, + logger: this.app.logger, + } as any); + + const user = await auth.check(); + + if (!user) { + this.app.logger.error(`Invalid token: ${payload.token}`); + return; + } + + this.app.emit(`ws:setTag`, { + clientId, + tagKey: 'userId', + tagValue: user.id, + }); + + this.app.emit(`ws:authorized`, { + clientId, + userId: user.id, + }); + }); this.app.auditManager.registerActions([ { name: 'auth:signIn', diff --git a/packages/plugins/@nocobase/plugin-data-source-manager/src/server/models/data-source.ts b/packages/plugins/@nocobase/plugin-data-source-manager/src/server/models/data-source.ts index 28854c58a9a..29c27fb7c05 100644 --- a/packages/plugins/@nocobase/plugin-data-source-manager/src/server/models/data-source.ts +++ b/packages/plugins/@nocobase/plugin-data-source-manager/src/server/models/data-source.ts @@ -13,6 +13,7 @@ import { setCurrentRole } from '@nocobase/plugin-acl'; import { ACL, AvailableActionOptions } from '@nocobase/acl'; import { DataSourcesRolesModel } from './data-sources-roles-model'; import PluginDataSourceManagerServer from '../plugin'; +import * as path from 'path'; const availableActions: { [key: string]: AvailableActionOptions; @@ -98,6 +99,8 @@ export class DataSourceModel extends Model { name: dataSourceKey, logger: app.logger.child({ dataSourceKey }), sqlLogger: app.sqlLogger.child({ dataSourceKey }), + cache: app.cache, + storagePath: path.join(process.cwd(), 'storage', 'cache', 'apps', app.name), }); dataSource.on('loadingProgress', (progress) => { diff --git a/packages/plugins/@nocobase/plugin-error-handler/src/server/error-handler.ts b/packages/plugins/@nocobase/plugin-error-handler/src/server/error-handler.ts index f7d5c796844..45d27d7a05a 100644 --- a/packages/plugins/@nocobase/plugin-error-handler/src/server/error-handler.ts +++ b/packages/plugins/@nocobase/plugin-error-handler/src/server/error-handler.ts @@ -36,6 +36,16 @@ export class ErrorHandler { }; } + renderError(err, ctx) { + for (const handler of this.handlers) { + if (handler.guard(err)) { + return handler.render(err, ctx); + } + } + + this.defaultHandler(err, ctx); + } + middleware() { const self = this; return async function errorHandler(ctx, next) { @@ -48,13 +58,7 @@ export class ErrorHandler { ctx.status = err.statusCode; } - for (const handler of self.handlers) { - if (handler.guard(err)) { - return handler.render(err, ctx); - } - } - - self.defaultHandler(err, ctx); + self.renderError(err, ctx); } }; } diff --git a/packages/plugins/@nocobase/plugin-error-handler/src/server/index.ts b/packages/plugins/@nocobase/plugin-error-handler/src/server/index.ts index 52bf01e67bd..0f1ff19189e 100644 --- a/packages/plugins/@nocobase/plugin-error-handler/src/server/index.ts +++ b/packages/plugins/@nocobase/plugin-error-handler/src/server/index.ts @@ -8,3 +8,4 @@ */ export { PluginErrorHandlerServer as default } from './server'; +export { ErrorHandler } from './error-handler'; diff --git a/packages/plugins/@nocobase/plugin-error-handler/src/server/server.ts b/packages/plugins/@nocobase/plugin-error-handler/src/server/server.ts index 5fd05fa7cf4..accb880caff 100644 --- a/packages/plugins/@nocobase/plugin-error-handler/src/server/server.ts +++ b/packages/plugins/@nocobase/plugin-error-handler/src/server/server.ts @@ -45,7 +45,9 @@ export class PluginErrorHandlerServer extends Plugin { }; this.errorHandler.register( - (err) => err?.errors?.length && err instanceof BaseError, + (err) => + err?.errors?.length && + (err.name === 'SequelizeValidationError' || err.name === 'SequelizeUniqueConstraintError'), (err, ctx) => { ctx.body = { errors: err.errors.map((err) => { diff --git a/packages/plugins/@nocobase/plugin-workflow/src/server/Plugin.ts b/packages/plugins/@nocobase/plugin-workflow/src/server/Plugin.ts index 11c623ead22..b261a819375 100644 --- a/packages/plugins/@nocobase/plugin-workflow/src/server/Plugin.ts +++ b/packages/plugins/@nocobase/plugin-workflow/src/server/Plugin.ts @@ -301,6 +301,11 @@ export default class PluginWorkflowServer extends Plugin { this.dispatch(); }, 300_000); + this.app.on('workflow:dispatch', () => { + this.app.logger.info('workflow:dispatch'); + this.dispatch(); + }); + // check for not started executions this.dispatch(); });