From 32b4e4ec1942524ee6e205ad98017e8f3a0ba27d Mon Sep 17 00:00:00 2001 From: alex <48489896+devnaumov@users.noreply.github.com> Date: Tue, 2 Apr 2024 18:51:27 +0200 Subject: [PATCH] CB-3330 add data import plugin (#2506) * CB-3330 add data import plugin * CB-3330 add mock api * CB-3330 pass processor id * CB-3330 remove extra code * CB-3330 simplify logic, allow import only one file * CB-3330. Implemented import data to table * CB-3330 replace mocks with api * CB-3330 add references * CB-3330. Parsing request, added xml, xls * CB-3330 support new settings * CB-3330 use getter * CB-3330 add svg icons * CB-3330 pass arguments in handleImport fn * CB-3330 remove extra fn wrapper * CB-3330. Fix build * CB-3330. Fix build * CB-3330. Fix import * CB-3330. Fixed import * CB-3300 fix: frontend can build * CB-3330. Refactor after review * CB-3330. Refactor after review, delete temp file after import * CB-3330. Refactor after review * CB-3330. Refactor after review * CB-3330. Added log message * CB-3330. Added response for task * CB-3330 fix icon and button shadow * CB-3330. Refactor response value * CB-3330. Refactor response value * CB-3330. delete unused imports * CB-3330 run additional task after file import * CB-3330 always allow data import for admins * CB-3330 import resource as module --------- Co-authored-by: DenisSinelnikov <142215442+DenisSinelnikov@users.noreply.github.com> Co-authored-by: denis.sinelnikov Co-authored-by: sergeyteleshev Co-authored-by: Evgenia Bezborodova <139753579+EvgeniaBzzz@users.noreply.github.com> --- .../src/io/cloudbeaver/server/CBPlatform.java | 4 +- .../events/WSDeleteTempFileHandler.java | 13 +- .../schema/service.data.transfer.graphqls | 3 + .../data/transfer/DBWServiceDataTransfer.java | 13 ++ .../WebServiceBindingDataTransfer.java | 8 ++ .../impl/WebDataTransferImportServlet.java | 123 ++++++++++++++++++ .../transfer/impl/WebServiceDataTransfer.java | 105 ++++++++++++++- .../Extensions/uploadResultDataExtension.ts | 45 +++++++ .../packages/core-sdk/src/GraphQLService.ts | 2 + .../getDataTransferImportProcessors.gql | 14 ++ .../core-utils/src/getProgressPercent.ts | 11 ++ webapp/packages/core-utils/src/index.ts | 1 + .../src/Action/Actions/ACTION_IMPORT.ts | 13 ++ webapp/packages/core-view/src/index.ts | 1 + webapp/packages/plugin-data-import/.gitignore | 17 +++ .../packages/plugin-data-import/package.json | 49 +++++++ .../public/icons/data-import.svg | 19 +++ .../public/icons/data-import_m.svg | 19 +++ .../public/icons/data-import_sm.svg | 19 +++ .../src/DataImportBootstrap.ts | 119 +++++++++++++++++ .../src/DataImportDialog/DataImportDialog.tsx | 85 ++++++++++++ .../DataImportDialog/DataImportDialogLazy.ts | 10 ++ .../DataImportDialog/DataImportFileItem.m.css | 12 ++ .../DataImportDialog/DataImportFileItem.tsx | 35 +++++ .../DataImportFileSelector.m.css | 12 ++ .../DataImportFileSelector.tsx | 51 ++++++++ .../DataImportDialog/EDataImportDialogStep.ts | 11 ++ .../IDataImportDialogState.ts | 16 +++ .../ImportProcessorList.m.css | 11 ++ .../DataImportDialog/ImportProcessorList.tsx | 38 ++++++ .../DataImportDialog/useDataImportDialog.ts | 59 +++++++++ .../src/DataImportProcessorsResource.ts | 38 ++++++ .../src/DataImportService.ts | 74 +++++++++++ .../src/DataImportSettingsService.ts | 28 ++++ .../plugin-data-import/src/LocaleService.ts | 35 +++++ .../packages/plugin-data-import/src/index.ts | 3 + .../plugin-data-import/src/locales/en.ts | 7 + .../plugin-data-import/src/locales/it.ts | 7 + .../plugin-data-import/src/locales/ru.ts | 7 + .../plugin-data-import/src/locales/zh.ts | 7 + .../plugin-data-import/src/manifest.ts | 22 ++++ .../packages/plugin-data-import/tsconfig.json | 76 +++++++++++ webapp/packages/product-default/package.json | 1 + webapp/packages/product-default/src/index.ts | 4 +- webapp/packages/product-default/tsconfig.json | 3 + 45 files changed, 1240 insertions(+), 10 deletions(-) create mode 100644 server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferImportServlet.java create mode 100644 webapp/packages/core-sdk/src/Extensions/uploadResultDataExtension.ts create mode 100644 webapp/packages/core-sdk/src/queries/data-transfer/getDataTransferImportProcessors.gql create mode 100644 webapp/packages/core-utils/src/getProgressPercent.ts create mode 100644 webapp/packages/core-view/src/Action/Actions/ACTION_IMPORT.ts create mode 100644 webapp/packages/plugin-data-import/.gitignore create mode 100644 webapp/packages/plugin-data-import/package.json create mode 100644 webapp/packages/plugin-data-import/public/icons/data-import.svg create mode 100644 webapp/packages/plugin-data-import/public/icons/data-import_m.svg create mode 100644 webapp/packages/plugin-data-import/public/icons/data-import_sm.svg create mode 100644 webapp/packages/plugin-data-import/src/DataImportBootstrap.ts create mode 100644 webapp/packages/plugin-data-import/src/DataImportDialog/DataImportDialog.tsx create mode 100644 webapp/packages/plugin-data-import/src/DataImportDialog/DataImportDialogLazy.ts create mode 100644 webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileItem.m.css create mode 100644 webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileItem.tsx create mode 100644 webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileSelector.m.css create mode 100644 webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileSelector.tsx create mode 100644 webapp/packages/plugin-data-import/src/DataImportDialog/EDataImportDialogStep.ts create mode 100644 webapp/packages/plugin-data-import/src/DataImportDialog/IDataImportDialogState.ts create mode 100644 webapp/packages/plugin-data-import/src/DataImportDialog/ImportProcessorList.m.css create mode 100644 webapp/packages/plugin-data-import/src/DataImportDialog/ImportProcessorList.tsx create mode 100644 webapp/packages/plugin-data-import/src/DataImportDialog/useDataImportDialog.ts create mode 100644 webapp/packages/plugin-data-import/src/DataImportProcessorsResource.ts create mode 100644 webapp/packages/plugin-data-import/src/DataImportService.ts create mode 100644 webapp/packages/plugin-data-import/src/DataImportSettingsService.ts create mode 100644 webapp/packages/plugin-data-import/src/LocaleService.ts create mode 100644 webapp/packages/plugin-data-import/src/index.ts create mode 100644 webapp/packages/plugin-data-import/src/locales/en.ts create mode 100644 webapp/packages/plugin-data-import/src/locales/it.ts create mode 100644 webapp/packages/plugin-data-import/src/locales/ru.ts create mode 100644 webapp/packages/plugin-data-import/src/locales/zh.ts create mode 100644 webapp/packages/plugin-data-import/src/manifest.ts create mode 100644 webapp/packages/plugin-data-import/tsconfig.json diff --git a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/CBPlatform.java b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/CBPlatform.java index 9983f9f844..5054d6b275 100644 --- a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/CBPlatform.java +++ b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/CBPlatform.java @@ -68,7 +68,8 @@ public class CBPlatform extends BasePlatformImpl { public static final String PLUGIN_ID = "io.cloudbeaver.server"; //$NON-NLS-1$ private static final Log log = Log.getLog(CBPlatform.class); - private static final String TEMP_FILE_FOLDER = "temp-sql-upload-files"; + public static final String TEMP_FILE_FOLDER = "temp-sql-upload-files"; + public static final String TEMP_FILE_IMPORT_FOLDER = "temp-import-files"; public static final String WORK_DATA_FOLDER_NAME = ".work-data"; @@ -133,6 +134,7 @@ public class CBPlatform extends BasePlatformImpl { protected IStatus run(DBRProgressMonitor monitor) { try { IOUtils.deleteDirectory(getTempFolder(monitor, TEMP_FILE_FOLDER)); + IOUtils.deleteDirectory(getTempFolder(monitor, TEMP_FILE_IMPORT_FOLDER)); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/events/WSDeleteTempFileHandler.java b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/events/WSDeleteTempFileHandler.java index 1a39f719c7..9c0085985c 100644 --- a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/events/WSDeleteTempFileHandler.java +++ b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/events/WSDeleteTempFileHandler.java @@ -31,11 +31,20 @@ import java.nio.file.Path; public class WSDeleteTempFileHandler implements WSEventHandler { private static final Log log = Log.getLog(WSDeleteTempFileHandler.class); - private static final String TEMP_FILE_FOLDER = "temp-sql-upload-files"; public void resetTempFolder(String sessionId) { Path path = CBPlatform.getInstance() - .getTempFolder(new VoidProgressMonitor(), TEMP_FILE_FOLDER) + .getTempFolder(new VoidProgressMonitor(), CBPlatform.TEMP_FILE_FOLDER) + .resolve(sessionId); + if (Files.exists(path)) { + try { + IOUtils.deleteDirectory(path); + } catch (IOException e) { + log.error("Error deleting temp path", e); + } + } + path = CBPlatform.getInstance() + .getTempFolder(new VoidProgressMonitor(), CBPlatform.TEMP_FILE_IMPORT_FOLDER) .resolve(sessionId); if (Files.exists(path)) { try { diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/schema/service.data.transfer.graphqls b/server/bundles/io.cloudbeaver.service.data.transfer/schema/service.data.transfer.graphqls index f2d0b80b61..805f1641ea 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/schema/service.data.transfer.graphqls +++ b/server/bundles/io.cloudbeaver.service.data.transfer/schema/service.data.transfer.graphqls @@ -53,6 +53,9 @@ extend type Query { # Available transfer processors dataTransferAvailableStreamProcessors: [ DataTransferProcessorInfo! ]! + # Available transfer processors for import file + dataTransferAvailableImportStreamProcessors: [ DataTransferProcessorInfo! ]! + dataTransferDefaultExportSettings: DataTransferDefaultExportSettings! dataTransferExportDataFromContainer( diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/DBWServiceDataTransfer.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/DBWServiceDataTransfer.java index d7a1c9e79e..61b80c8dd5 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/DBWServiceDataTransfer.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/DBWServiceDataTransfer.java @@ -26,7 +26,10 @@ import io.cloudbeaver.service.data.transfer.impl.WebDataTransferParameters; import io.cloudbeaver.service.data.transfer.impl.WebDataTransferStreamProcessor; import io.cloudbeaver.service.sql.WebSQLContextInfo; import io.cloudbeaver.service.sql.WebSQLProcessor; +import io.cloudbeaver.service.sql.WebSQLResultsInfo; +import org.jkiss.code.NotNull; +import java.nio.file.Path; import java.util.List; /** @@ -37,12 +40,22 @@ public interface DBWServiceDataTransfer extends DBWService { @WebAction List getAvailableStreamProcessors(WebSession session) throws DBWebException; + @WebAction + List getAvailableImportStreamProcessors(WebSession session) throws DBWebException; + @WebAction WebAsyncTaskInfo dataTransferExportDataFromContainer( WebSQLProcessor sqlProcessor, String containerNodePath, WebDataTransferParameters parameters) throws DBWebException; + @WebAction + WebAsyncTaskInfo asyncImportDataContainer( + @NotNull String processorId, + @NotNull Path path, + @NotNull WebSQLResultsInfo webSQLResultsInfo, + @NotNull WebSession webSession) throws DBWebException; + @WebAction WebAsyncTaskInfo dataTransferExportDataFromResults( WebSQLContextInfo sqlContextInfo, diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebServiceBindingDataTransfer.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebServiceBindingDataTransfer.java index 4dc33894ad..5c6daa5356 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebServiceBindingDataTransfer.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebServiceBindingDataTransfer.java @@ -24,6 +24,7 @@ import io.cloudbeaver.service.WebServiceBindingBase; import io.cloudbeaver.service.data.transfer.impl.WebDataTransferParameters; import io.cloudbeaver.service.data.transfer.impl.WebDataTransferServlet; import io.cloudbeaver.service.data.transfer.impl.WebServiceDataTransfer; +import io.cloudbeaver.service.data.transfer.impl.WebDataTransferImportServlet; import io.cloudbeaver.service.sql.WebServiceBindingSQL; import org.jkiss.dbeaver.DBException; @@ -42,6 +43,8 @@ public class WebServiceBindingDataTransfer extends WebServiceBindingBase getService(env).getAvailableStreamProcessors(getWebSession(env))) + .dataFetcher("dataTransferAvailableImportStreamProcessors", + env -> getService(env).getAvailableImportStreamProcessors(getWebSession(env))) .dataFetcher("dataTransferExportDataFromContainer", env -> getService(env).dataTransferExportDataFromContainer( WebServiceBindingSQL.getSQLProcessor(env), env.getArgument("containerNodePath"), @@ -68,5 +71,10 @@ public class WebServiceBindingDataTransfer extends WebServiceBindingBase variables = getVariables(request); + + String projectId = JSONUtils.getString(variables, "projectId"); + String connectionId = JSONUtils.getString(variables, "connectionId"); + String contextId = JSONUtils.getString(variables, "contextId"); + String resultId = JSONUtils.getString(variables, "resultsId"); + String processorId = JSONUtils.getString(variables, "processorId"); + + if (projectId == null || connectionId == null || contextId == null || resultId == null || processorId == null) { + throw new IllegalArgumentException("Missing required parameters"); + } + + WebConnectionInfo webConnectionInfo = session.getWebConnectionInfo(projectId, connectionId); + WebSQLProcessor processor = WebServiceBindingSQL.getSQLProcessor(webConnectionInfo); + WebSQLContextInfo webSQLContextInfo = processor.getContext(contextId); + + if (webSQLContextInfo == null) { + throw new DBWebException("Context is empty"); + } + + WebSQLResultsInfo webSQLResultsInfo = webSQLContextInfo.getResults(resultId); + Path filePath; + + try { + InputStream file = request.getPart("fileData").getInputStream(); + filePath = tempFolder.resolve(UUID.randomUUID().toString()); + Files.write(filePath, file.readAllBytes()); + } catch (ServletException e) { + throw new DBWebException(e.getMessage()); + } + + WebAsyncTaskInfo asyncImportDataContainer = + dbwServiceDataTransfer.asyncImportDataContainer(processorId, filePath, webSQLResultsInfo, session); + response.setContentType(CBConstants.APPLICATION_JSON); + Map parameters = new LinkedHashMap<>(); + parameters.put("id", asyncImportDataContainer.getId()); + parameters.put("name", asyncImportDataContainer.getName()); + parameters.put("running", asyncImportDataContainer.isRunning()); + parameters.put("status", asyncImportDataContainer.getStatus()); + parameters.put("error", asyncImportDataContainer.getError()); + parameters.put("taskResult", asyncImportDataContainer.getTaskResult()); + try (JsonWriter writer = new JsonWriter(response.getWriter())) { + JSONUtils.serializeMap(writer, parameters); + } + } + } +} \ No newline at end of file diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebServiceDataTransfer.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebServiceDataTransfer.java index d747e5d6ba..b363879cc8 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebServiceDataTransfer.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebServiceDataTransfer.java @@ -37,24 +37,27 @@ import org.jkiss.dbeaver.model.preferences.DBPPropertyDescriptor; import org.jkiss.dbeaver.model.runtime.DBRProgressMonitor; import org.jkiss.dbeaver.model.runtime.VoidProgressMonitor; import org.jkiss.dbeaver.model.sql.DBQuotaException; +import org.jkiss.dbeaver.model.sql.DBSQLException; import org.jkiss.dbeaver.model.struct.DBSDataContainer; +import org.jkiss.dbeaver.model.struct.DBSDataManipulator; import org.jkiss.dbeaver.model.struct.DBSEntity; +import org.jkiss.dbeaver.model.struct.DBSObjectContainer; import org.jkiss.dbeaver.tools.transfer.IDataTransferConsumer; import org.jkiss.dbeaver.tools.transfer.IDataTransferProcessor; -import org.jkiss.dbeaver.tools.transfer.database.DatabaseProducerSettings; -import org.jkiss.dbeaver.tools.transfer.database.DatabaseTransferProducer; +import org.jkiss.dbeaver.tools.transfer.database.*; import org.jkiss.dbeaver.tools.transfer.registry.DataTransferProcessorDescriptor; import org.jkiss.dbeaver.tools.transfer.registry.DataTransferRegistry; -import org.jkiss.dbeaver.tools.transfer.stream.IStreamDataExporter; -import org.jkiss.dbeaver.tools.transfer.stream.StreamConsumerSettings; -import org.jkiss.dbeaver.tools.transfer.stream.StreamTransferConsumer; +import org.jkiss.dbeaver.tools.transfer.stream.*; import org.jkiss.dbeaver.utils.ContentUtils; import org.jkiss.utils.CommonUtils; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; import java.nio.file.Path; +import java.sql.BatchUpdateException; import java.util.*; import java.util.stream.Collectors; @@ -82,7 +85,19 @@ public class WebServiceDataTransfer implements DBWServiceDataTransfer { @Override public List getAvailableStreamProcessors(WebSession session) { - List processors = DataTransferRegistry.getInstance().getAvailableProcessors(StreamTransferConsumer.class, DBSEntity.class); + List processors = DataTransferRegistry.getInstance() + .getAvailableProcessors(StreamTransferConsumer.class, DBSEntity.class); + if (CommonUtils.isEmpty(processors)) { + return Collections.emptyList(); + } + + return processors.stream().map(x -> new WebDataTransferStreamProcessor(session, x)).collect(Collectors.toList()); + } + + @Override + public List getAvailableImportStreamProcessors(WebSession session) { + List processors = + DataTransferRegistry.getInstance().getAvailableProcessors(StreamTransferProducer.class, DBSEntity.class); if (CommonUtils.isEmpty(processors)) { return Collections.emptyList(); } @@ -205,6 +220,43 @@ public class WebServiceDataTransfer implements DBWServiceDataTransfer { return sqlProcessor.getWebSession().createAndRunAsyncTask("Data export", runnable); } + public WebAsyncTaskInfo asyncImportDataContainer(@NotNull String processorId, + @NotNull Path path, + @NotNull WebSQLResultsInfo sqlContext, + @NotNull WebSession webSession) throws DBWebException { + webSession.addInfoMessage("Import data"); + DataTransferProcessorDescriptor processor = DataTransferRegistry.getInstance().getProcessor(processorId); + + DBSDataContainer dataContainer = sqlContext.getDataContainer(); + WebAsyncTaskProcessor runnable = new WebAsyncTaskProcessor<>() { + @Override + public void run(DBRProgressMonitor monitor) throws InvocationTargetException { + monitor.beginTask("Import data", 1); + try { + monitor.subTask("Import data using " + processor.getName()); + try { + importData(monitor, processor, (DBSDataManipulator) dataContainer, path); + } catch (Exception e) { + if (e instanceof DBException) { + throw e; + } + throw new DBException("Error importing data", e); + } + } catch (Throwable e) { + throw new InvocationTargetException(e); + } finally { + try { + Files.deleteIfExists(path); + } catch (IOException e) { + log.error("Failed to delete file: " + e.getMessage(), e); + } + monitor.done(); + } + } + }; + return webSession.createAndRunAsyncTask("Data import", runnable); + } + private void exportData( DBRProgressMonitor monitor, DataTransferProcessorDescriptor processor, @@ -282,4 +334,45 @@ public class WebServiceDataTransfer implements DBWServiceDataTransfer { consumer.finishTransfer(monitor, false); } + private void importData( + DBRProgressMonitor monitor, + DataTransferProcessorDescriptor processor, + @NotNull DBSDataManipulator dataContainer, + Path path) throws DBException + { + IDataTransferProcessor processorInstance = processor.getInstance(); + + if (dataContainer.getDataSource() != null) { + DatabaseTransferConsumer consumer = new DatabaseTransferConsumer(dataContainer); + + DatabaseConsumerSettings databaseConsumerSettings = new DatabaseConsumerSettings(); + databaseConsumerSettings.setContainer((DBSObjectContainer) dataContainer.getDataSource()); + consumer.setSettings(databaseConsumerSettings); + DatabaseMappingContainer databaseMappingContainer = new DatabaseMappingContainer(databaseConsumerSettings, dataContainer); + databaseMappingContainer.getAttributeMappings(monitor); + databaseMappingContainer.setTarget(dataContainer); + consumer.setContainerMapping(databaseMappingContainer); + + StreamTransferProducer producer = new StreamTransferProducer(new StreamEntityMapping(path), processor); + + StreamProducerSettings producerSettings = new StreamProducerSettings(); + Map properties = new HashMap<>(); + for (DBPPropertyDescriptor prop : processor.getProperties()) { + properties.put(prop.getId(), prop.getDefaultValue()); + } + producerSettings.setProcessorProperties(properties); + producerSettings.updateProducerSettingsFromStream( + monitor, + producer, + processorInstance, + properties); + + try { + producer.transferData(monitor, consumer, processorInstance, producerSettings, null); + } catch (DBException e) { + throw new DBWebException("Import failed cause: " + e.getMessage()); + } + } + } + } diff --git a/webapp/packages/core-sdk/src/Extensions/uploadResultDataExtension.ts b/webapp/packages/core-sdk/src/Extensions/uploadResultDataExtension.ts new file mode 100644 index 0000000000..961e6c3aa9 --- /dev/null +++ b/webapp/packages/core-sdk/src/Extensions/uploadResultDataExtension.ts @@ -0,0 +1,45 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import { GlobalConstants } from '@cloudbeaver/core-utils'; + +import type { CustomGraphQLClient, UploadProgressEvent } from '../CustomGraphQLClient'; +import type { AsyncTaskInfo } from '../sdk'; + +export interface IUploadResultDataExtension { + uploadResultData: ( + connectionId: string, + contextId: string, + projectId: string, + resultsId: string, + processorId: string, + file: File, + onUploadProgress?: (event: UploadProgressEvent) => void, + ) => Promise; +} + +export function uploadResultDataExtension(client: CustomGraphQLClient): IUploadResultDataExtension { + return { + uploadResultData( + connectionId: string, + contextId: string, + projectId: string, + resultsId: string, + processorId: string, + file: File, + onUploadProgress?: (event: UploadProgressEvent) => void, + ): Promise { + return client.uploadFile( + GlobalConstants.absoluteServiceUrl('data', 'import'), + file, + undefined, + { connectionId, contextId, projectId, resultsId, processorId }, + onUploadProgress, + ); + }, + }; +} diff --git a/webapp/packages/core-sdk/src/GraphQLService.ts b/webapp/packages/core-sdk/src/GraphQLService.ts index cdf0632277..5a035f21b1 100644 --- a/webapp/packages/core-sdk/src/GraphQLService.ts +++ b/webapp/packages/core-sdk/src/GraphQLService.ts @@ -11,6 +11,7 @@ import { CustomGraphQLClient } from './CustomGraphQLClient'; import { EnvironmentService } from './EnvironmentService'; import { uploadBlobResultSetExtension } from './Extensions/uploadBlobResultSetExtension'; import { uploadDriverLibraryExtension } from './Extensions/uploadDriverLibraryExtension'; +import { uploadResultDataExtension } from './Extensions/uploadResultDataExtension'; import type { IResponseInterceptor } from './IResponseInterceptor'; import { getSdk } from './sdk'; @@ -21,6 +22,7 @@ function extendedSDK(client: CustomGraphQLClient) { ...sdk, ...uploadDriverLibraryExtension(client), ...uploadBlobResultSetExtension(client), + ...uploadResultDataExtension(client), }; } diff --git a/webapp/packages/core-sdk/src/queries/data-transfer/getDataTransferImportProcessors.gql b/webapp/packages/core-sdk/src/queries/data-transfer/getDataTransferImportProcessors.gql new file mode 100644 index 0000000000..21e695c375 --- /dev/null +++ b/webapp/packages/core-sdk/src/queries/data-transfer/getDataTransferImportProcessors.gql @@ -0,0 +1,14 @@ +query getDataTransferImportProcessors { + processors: dataTransferAvailableImportStreamProcessors { + id + name + description + fileExtension + appFileExtension + appName + order + icon + isBinary + isHTML + } +} diff --git a/webapp/packages/core-utils/src/getProgressPercent.ts b/webapp/packages/core-utils/src/getProgressPercent.ts new file mode 100644 index 0000000000..ddd3809128 --- /dev/null +++ b/webapp/packages/core-utils/src/getProgressPercent.ts @@ -0,0 +1,11 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +export function getProgressPercent(done: number, total: number) { + const percentCompleted = Math.round((done * 100) / total); + return Math.min(100, Math.max(0, percentCompleted)); +} diff --git a/webapp/packages/core-utils/src/index.ts b/webapp/packages/core-utils/src/index.ts index c8da180740..ba4b96a22c 100644 --- a/webapp/packages/core-utils/src/index.ts +++ b/webapp/packages/core-utils/src/index.ts @@ -79,3 +79,4 @@ export * from './replaceSubstring'; export * from './formatNumber'; export * from './withTimestamp'; export * from './toSafeHtmlString'; +export * from './getProgressPercent'; diff --git a/webapp/packages/core-view/src/Action/Actions/ACTION_IMPORT.ts b/webapp/packages/core-view/src/Action/Actions/ACTION_IMPORT.ts new file mode 100644 index 0000000000..3f2501fbcc --- /dev/null +++ b/webapp/packages/core-view/src/Action/Actions/ACTION_IMPORT.ts @@ -0,0 +1,13 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import { createAction } from '../createAction'; + +export const ACTION_IMPORT = createAction('import', { + label: 'ui_import', + tooltip: 'ui_import', +}); diff --git a/webapp/packages/core-view/src/index.ts b/webapp/packages/core-view/src/index.ts index ce0e214a88..3fd533f3b8 100644 --- a/webapp/packages/core-view/src/index.ts +++ b/webapp/packages/core-view/src/index.ts @@ -18,6 +18,7 @@ export * from './Action/Actions/ACTION_ZOOM_IN'; export * from './Action/Actions/ACTION_ZOOM_OUT'; export * from './Action/Actions/ACTION_DOWNLOAD'; export * from './Action/Actions/ACTION_UPLOAD'; +export * from './Action/Actions/ACTION_IMPORT'; export * from './Action/KeyBinding/Bindings/KEY_BINDING_OPEN_IN_TAB'; export * from './Action/KeyBinding/Bindings/KEY_BINDING_REDO'; export * from './Action/KeyBinding/Bindings/KEY_BINDING_UNDO'; diff --git a/webapp/packages/plugin-data-import/.gitignore b/webapp/packages/plugin-data-import/.gitignore new file mode 100644 index 0000000000..15bc16c7c3 --- /dev/null +++ b/webapp/packages/plugin-data-import/.gitignore @@ -0,0 +1,17 @@ +# dependencies +/node_modules + +# testing +/coverage + +# production +/lib + +# misc +.DS_Store +.env* + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* diff --git a/webapp/packages/plugin-data-import/package.json b/webapp/packages/plugin-data-import/package.json new file mode 100644 index 0000000000..80869e3205 --- /dev/null +++ b/webapp/packages/plugin-data-import/package.json @@ -0,0 +1,49 @@ +{ + "name": "@cloudbeaver/plugin-data-import", + "sideEffects": [ + "src/**/*.css", + "src/**/*.scss", + "public/**/*" + ], + "version": "0.1.0", + "description": "", + "license": "Apache-2.0", + "main": "dist/index.js", + "scripts": { + "build": "tsc -b", + "lint": "eslint ./src/ --ext .ts,.tsx", + "lint-fix": "eslint ./src/ --ext .ts,.tsx --fix", + "test": "yarn build && core-cli-test", + "validate-dependencies": "core-cli-validate-dependencies", + "update-ts-references": "rimraf --glob dist && typescript-resolve-references" + }, + "dependencies": { + "@cloudbeaver/core-blocks": "~0.1.0", + "@cloudbeaver/core-connections": "~0.1.0", + "@cloudbeaver/core-di": "~0.1.0", + "@cloudbeaver/core-dialogs": "~0.1.0", + "@cloudbeaver/core-events": "~0.1.0", + "@cloudbeaver/core-localization": "~0.1.0", + "@cloudbeaver/core-navigation-tree": "~0.1.0", + "@cloudbeaver/core-resource": "~0.1.0", + "@cloudbeaver/core-root": "~0.1.0", + "@cloudbeaver/core-sdk": "~0.1.0", + "@cloudbeaver/core-settings": "~0.1.0", + "@cloudbeaver/core-ui": "~0.1.0", + "@cloudbeaver/core-utils": "~0.1.0", + "@cloudbeaver/core-view": "~0.1.0", + "@cloudbeaver/plugin-data-viewer": "~0.1.0", + "@cloudbeaver/plugin-datasource-context-switch": "~0.1.0", + "@cloudbeaver/plugin-sql-editor": "~0.1.0", + "@cloudbeaver/plugin-top-app-bar": "~0.1.0", + "mobx": "^6.12.0", + "mobx-react-lite": "^4.0.5", + "react": "^18.2.0" + }, + "peerDependencies": {}, + "devDependencies": { + "@types/react": "^18.2.63", + "typescript": "^5.3.3", + "typescript-plugin-css-modules": "^5.1.0" + } +} diff --git a/webapp/packages/plugin-data-import/public/icons/data-import.svg b/webapp/packages/plugin-data-import/public/icons/data-import.svg new file mode 100644 index 0000000000..5fa8bd2841 --- /dev/null +++ b/webapp/packages/plugin-data-import/public/icons/data-import.svg @@ -0,0 +1,19 @@ + + + + + + + + + + \ No newline at end of file diff --git a/webapp/packages/plugin-data-import/public/icons/data-import_m.svg b/webapp/packages/plugin-data-import/public/icons/data-import_m.svg new file mode 100644 index 0000000000..ca1b92e769 --- /dev/null +++ b/webapp/packages/plugin-data-import/public/icons/data-import_m.svg @@ -0,0 +1,19 @@ + + + + + + + + + + \ No newline at end of file diff --git a/webapp/packages/plugin-data-import/public/icons/data-import_sm.svg b/webapp/packages/plugin-data-import/public/icons/data-import_sm.svg new file mode 100644 index 0000000000..d634271a82 --- /dev/null +++ b/webapp/packages/plugin-data-import/public/icons/data-import_sm.svg @@ -0,0 +1,19 @@ + + + + + + + + + + \ No newline at end of file diff --git a/webapp/packages/plugin-data-import/src/DataImportBootstrap.ts b/webapp/packages/plugin-data-import/src/DataImportBootstrap.ts new file mode 100644 index 0000000000..5d9e9408e6 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportBootstrap.ts @@ -0,0 +1,119 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import { Bootstrap, injectable } from '@cloudbeaver/core-di'; +import { CommonDialogService, DialogueStateResult } from '@cloudbeaver/core-dialogs'; +import { ACTION_IMPORT, ActionService, DATA_CONTEXT_MENU, menuExtractItems, MenuService } from '@cloudbeaver/core-view'; +import { + DATA_CONTEXT_DV_DDM, + DATA_CONTEXT_DV_DDM_RESULT_INDEX, + DATA_VIEWER_DATA_MODEL_ACTIONS_MENU, + DatabaseEditAction, +} from '@cloudbeaver/plugin-data-viewer'; + +import { DataImportDialogLazy } from './DataImportDialog/DataImportDialogLazy'; +import { DataImportService } from './DataImportService'; + +@injectable() +export class DataImportBootstrap extends Bootstrap { + constructor( + private readonly menuService: MenuService, + private readonly actionService: ActionService, + private readonly commonDialogService: CommonDialogService, + private readonly dataImportService: DataImportService, + ) { + super(); + } + + register() { + this.actionService.addHandler({ + id: 'data-import-base-handler', + isActionApplicable(context, action) { + const menu = context.hasValue(DATA_CONTEXT_MENU, DATA_VIEWER_DATA_MODEL_ACTIONS_MENU); + const model = context.tryGet(DATA_CONTEXT_DV_DDM); + const resultIndex = context.tryGet(DATA_CONTEXT_DV_DDM_RESULT_INDEX); + + if (!menu || !model || resultIndex === undefined) { + return false; + } + + if (action === ACTION_IMPORT) { + if (model.isReadonly(resultIndex)) { + return false; + } + + const editor = model.source.getActionImplementation(resultIndex, DatabaseEditAction); + return editor?.hasFeature('add') === true; + } + + return [ACTION_IMPORT].includes(action); + }, + isDisabled(context) { + const model = context.get(DATA_CONTEXT_DV_DDM); + const resultIndex = context.get(DATA_CONTEXT_DV_DDM_RESULT_INDEX); + + return model.isLoading() || model.isDisabled(resultIndex) || !model.getResult(resultIndex); + }, + getActionInfo(_, action) { + if (action === ACTION_IMPORT) { + return { ...action.info, icon: '/icons/data-import.svg' }; + } + + return action.info; + }, + handler: async (context, action) => { + const model = context.get(DATA_CONTEXT_DV_DDM); + const resultIndex = context.get(DATA_CONTEXT_DV_DDM_RESULT_INDEX); + + if (action === ACTION_IMPORT) { + const result = model.getResult(resultIndex); + + if (!result?.id) { + throw new Error('Result must be provided'); + } + + const executionContext = model.source.executionContext?.context; + + if (!executionContext) { + throw new Error('Execution context must be provided'); + } + + const state = await this.commonDialogService.open(DataImportDialogLazy, { tableName: model.name ?? model.id }); + + if (state === DialogueStateResult.Rejected || state === DialogueStateResult.Resolved) { + return; + } + + const success = await this.dataImportService.importData( + executionContext.connectionId, + executionContext.id, + executionContext.projectId, + result.id, + state.processorId, + state.file, + ); + + if (success) { + await model.refresh(); + } + } + }, + }); + + this.menuService.addCreator({ + menus: [DATA_VIEWER_DATA_MODEL_ACTIONS_MENU], + isApplicable: () => !this.dataImportService.disabled, + getItems(_, items) { + return [...items, ACTION_IMPORT]; + }, + orderItems(_, items) { + const extracted = menuExtractItems(items, [ACTION_IMPORT]); + return [...items, ...extracted]; + }, + }); + } +} diff --git a/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportDialog.tsx b/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportDialog.tsx new file mode 100644 index 0000000000..db530889e3 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportDialog.tsx @@ -0,0 +1,85 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import { observer } from 'mobx-react-lite'; + +import { + Button, + CommonDialogBody, + CommonDialogFooter, + CommonDialogHeader, + CommonDialogWrapper, + Container, + Fill, + useTranslate, +} from '@cloudbeaver/core-blocks'; +import type { DialogComponent } from '@cloudbeaver/core-dialogs'; + +import { DataImportFileSelector } from './DataImportFileSelector'; +import { EDataImportDialogStep } from './EDataImportDialogStep'; +import type { IDataImportDialogState } from './IDataImportDialogState'; +import { ImportProcessorList } from './ImportProcessorList'; +import { useDataImportDialog } from './useDataImportDialog'; + +export interface IDataImportDialogResult { + file: File; + processorId: string; +} + +export interface IDataImportDialogPayload { + tableName: string; + initialState?: IDataImportDialogState; +} + +export const DataImportDialog: DialogComponent = observer(function DataImportDialog({ + payload, + resolveDialog, + rejectDialog, +}) { + const translate = useTranslate(); + const dialog = useDataImportDialog(payload.initialState); + + let title = translate('plugin_data_import_title'); + let icon = '/icons/data-import.svg'; + + if (dialog.state.step === EDataImportDialogStep.File && dialog.state.selectedProcessor) { + title += ` (${dialog.state.selectedProcessor.name ?? dialog.state.selectedProcessor.id})`; + icon = dialog.state.selectedProcessor.icon ?? icon; + } + + return ( + + + + {dialog.state.step === EDataImportDialogStep.Processor && } + {dialog.state.step === EDataImportDialogStep.File && } + + + + + + {dialog.state.step === EDataImportDialogStep.File && ( + + + + + )} + + + ); +}); diff --git a/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportDialogLazy.ts b/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportDialogLazy.ts new file mode 100644 index 0000000000..c795b46a93 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportDialogLazy.ts @@ -0,0 +1,10 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import { importLazyComponent } from '@cloudbeaver/core-blocks'; + +export const DataImportDialogLazy = importLazyComponent(() => import('./DataImportDialog').then(m => m.DataImportDialog)); diff --git a/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileItem.m.css b/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileItem.m.css new file mode 100644 index 0000000000..55d70afa22 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileItem.m.css @@ -0,0 +1,12 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ + +.tableColumnValue { + height: 36px; + padding: 0 24px; +} diff --git a/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileItem.tsx b/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileItem.tsx new file mode 100644 index 0000000000..c3a2603a54 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileItem.tsx @@ -0,0 +1,35 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import { observer } from 'mobx-react-lite'; + +import { ActionIconButton, Container, s, TableColumnValue, TableItem, useS } from '@cloudbeaver/core-blocks'; + +import classes from './DataImportFileItem.m.css'; + +interface Props { + id: string; + name: string; + disabled?: boolean; + tooltip?: string; + className?: string; + onDelete: (id: string) => void; +} + +export const DataImportFileItem = observer(function DataImportFileItem({ id, name, tooltip, disabled, className, onDelete }) { + const styles = useS(classes); + + return ( + + {name} + + + onDelete(id)} /> + + + ); +}); diff --git a/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileSelector.m.css b/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileSelector.m.css new file mode 100644 index 0000000000..86224cdd0c --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileSelector.m.css @@ -0,0 +1,12 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ + +.columnHeader { + padding-left: 24px; + padding-right: 24px; +} diff --git a/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileSelector.tsx b/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileSelector.tsx new file mode 100644 index 0000000000..ed3d7e92d1 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportDialog/DataImportFileSelector.tsx @@ -0,0 +1,51 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import { observer } from 'mobx-react-lite'; + +import { Container, Group, InputFiles, s, Table, TableBody, TableColumnHeader, TableHeader, useS, useTranslate } from '@cloudbeaver/core-blocks'; + +import { DataImportFileItem } from './DataImportFileItem'; +import classes from './DataImportFileSelector.m.css'; +import type { IDataImportDialogState } from './IDataImportDialogState'; + +interface Props { + state: IDataImportDialogState; + onDelete: () => void; +} + +export const DataImportFileSelector = observer(function DataImportFileSelector({ state, onDelete }) { + const translate = useTranslate(); + const style = useS(classes); + + function handleFileSelect(value: FileList | null) { + if (value) { + state.file = value[0]; + } + } + + const extension = state.selectedProcessor?.fileExtension ? `.${state.selectedProcessor.fileExtension}` : undefined; + + return ( + + + + + {translate('ui_name')} + + + + + + + + {state.file && } + +
+ + ); +}); diff --git a/webapp/packages/plugin-data-import/src/DataImportDialog/EDataImportDialogStep.ts b/webapp/packages/plugin-data-import/src/DataImportDialog/EDataImportDialogStep.ts new file mode 100644 index 0000000000..e4385692bd --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportDialog/EDataImportDialogStep.ts @@ -0,0 +1,11 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +export enum EDataImportDialogStep { + Processor, + File, +} diff --git a/webapp/packages/plugin-data-import/src/DataImportDialog/IDataImportDialogState.ts b/webapp/packages/plugin-data-import/src/DataImportDialog/IDataImportDialogState.ts new file mode 100644 index 0000000000..23d75a7bad --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportDialog/IDataImportDialogState.ts @@ -0,0 +1,16 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import type { DataTransferProcessorInfo } from '@cloudbeaver/core-sdk'; + +import type { EDataImportDialogStep } from './EDataImportDialogStep'; + +export interface IDataImportDialogState { + step: EDataImportDialogStep; + file: File | null; + selectedProcessor: DataTransferProcessorInfo | null; +} diff --git a/webapp/packages/plugin-data-import/src/DataImportDialog/ImportProcessorList.m.css b/webapp/packages/plugin-data-import/src/DataImportDialog/ImportProcessorList.m.css new file mode 100644 index 0000000000..bd43f5c8d6 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportDialog/ImportProcessorList.m.css @@ -0,0 +1,11 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +.staticImage { + display: flex; + width: 24px; +} diff --git a/webapp/packages/plugin-data-import/src/DataImportDialog/ImportProcessorList.tsx b/webapp/packages/plugin-data-import/src/DataImportDialog/ImportProcessorList.tsx new file mode 100644 index 0000000000..61ce76dd07 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportDialog/ImportProcessorList.tsx @@ -0,0 +1,38 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import { observer } from 'mobx-react-lite'; + +import { ItemList, ListItem, ListItemDescription, ListItemIcon, ListItemName, StaticImage, useResource } from '@cloudbeaver/core-blocks'; +import { CachedMapAllKey } from '@cloudbeaver/core-resource'; +import type { DataTransferProcessorInfo } from '@cloudbeaver/core-sdk'; + +import { DataImportProcessorsResource } from '../DataImportProcessorsResource'; +import classes from './ImportProcessorList.m.css'; + +interface Props { + onSelect: (processor: DataTransferProcessorInfo) => void; + className?: string; +} + +export const ImportProcessorList = observer(function ImportProcessorList({ onSelect, className }) { + const dataImportProcessorsResource = useResource(ImportProcessorList, DataImportProcessorsResource, CachedMapAllKey, { forceSuspense: true }); + + return ( + + {dataImportProcessorsResource.resource.values.map(processor => ( + onSelect(processor)}> + + + + {processor.name} + {processor.description} + + ))} + + ); +}); diff --git a/webapp/packages/plugin-data-import/src/DataImportDialog/useDataImportDialog.ts b/webapp/packages/plugin-data-import/src/DataImportDialog/useDataImportDialog.ts new file mode 100644 index 0000000000..eefdde64fd --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportDialog/useDataImportDialog.ts @@ -0,0 +1,59 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import { action, observable } from 'mobx'; + +import { useObservableRef } from '@cloudbeaver/core-blocks'; +import type { DataTransferProcessorInfo } from '@cloudbeaver/core-sdk'; + +import { EDataImportDialogStep } from './EDataImportDialogStep'; +import type { IDataImportDialogState } from './IDataImportDialogState'; + +interface IDialog { + state: IDataImportDialogState; + stepBack: () => void; + selectProcessor: (processor: DataTransferProcessorInfo) => void; + deleteFile: () => void; + reset: () => void; +} + +const DEFAULT_STATE_GETTER: () => IDataImportDialogState = () => ({ + step: EDataImportDialogStep.Processor, + file: null, + selectedProcessor: null, +}); + +export function useDataImportDialog(initialState?: IDataImportDialogState) { + const dialog = useObservableRef( + () => ({ + state: initialState ?? DEFAULT_STATE_GETTER(), + stepBack() { + if (this.state.step === EDataImportDialogStep.File) { + this.state.step = EDataImportDialogStep.Processor; + } + }, + selectProcessor(processor: DataTransferProcessorInfo) { + if (this.state.selectedProcessor && this.state.selectedProcessor.id !== processor.id) { + this.reset(); + } + + this.state.selectedProcessor = processor; + this.state.step = EDataImportDialogStep.File; + }, + deleteFile() { + this.state.file = null; + }, + reset() { + this.state = DEFAULT_STATE_GETTER(); + }, + }), + { state: observable, stepBack: action.bound, selectProcessor: action.bound, deleteFile: action.bound, reset: action.bound }, + false, + ); + + return dialog; +} diff --git a/webapp/packages/plugin-data-import/src/DataImportProcessorsResource.ts b/webapp/packages/plugin-data-import/src/DataImportProcessorsResource.ts new file mode 100644 index 0000000000..5b8f376a83 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportProcessorsResource.ts @@ -0,0 +1,38 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import { injectable } from '@cloudbeaver/core-di'; +import { CachedMapAllKey, CachedMapResource, resourceKeyList } from '@cloudbeaver/core-resource'; +import { ServerConfigResource } from '@cloudbeaver/core-root'; +import { DataTransferProcessorInfo, GraphQLService } from '@cloudbeaver/core-sdk'; + +@injectable() +export class DataImportProcessorsResource extends CachedMapResource { + constructor( + private readonly graphQLService: GraphQLService, + serverConfigResource: ServerConfigResource, + ) { + super(() => new Map()); + this.sync( + serverConfigResource, + () => {}, + () => CachedMapAllKey, + ); + } + + protected async loader(): Promise> { + const { processors } = await this.graphQLService.sdk.getDataTransferImportProcessors(); + + this.replace(resourceKeyList(processors.map(processor => processor.id)), processors); + + return this.data; + } + + protected validateKey(key: string): boolean { + return typeof key === 'string'; + } +} diff --git a/webapp/packages/plugin-data-import/src/DataImportService.ts b/webapp/packages/plugin-data-import/src/DataImportService.ts new file mode 100644 index 0000000000..4dacb4e678 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportService.ts @@ -0,0 +1,74 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import { computed, makeObservable } from 'mobx'; + +import { ProcessSnackbar } from '@cloudbeaver/core-blocks'; +import { injectable } from '@cloudbeaver/core-di'; +import { NotificationService } from '@cloudbeaver/core-events'; +import { EAdminPermission, SessionPermissionsResource } from '@cloudbeaver/core-root'; +import { AsyncTaskInfoService, GraphQLService } from '@cloudbeaver/core-sdk'; +import { getProgressPercent } from '@cloudbeaver/core-utils'; + +import { DataImportSettingsService } from './DataImportSettingsService'; + +@injectable() +export class DataImportService { + get disabled() { + if (this.sessionPermissionsResource.has(EAdminPermission.admin)) { + return false; + } + + return this.dataImportSettingsService.disabled; + } + + constructor( + private readonly dataImportSettingsService: DataImportSettingsService, + private readonly notificationService: NotificationService, + private readonly graphQLService: GraphQLService, + private readonly asyncTaskInfoService: AsyncTaskInfoService, + private readonly sessionPermissionsResource: SessionPermissionsResource, + ) { + makeObservable(this, { + disabled: computed, + }); + } + + async importData(connectionId: string, contextId: string, projectId: string, resultsId: string, processorId: string, file: File) { + const { controller, notification } = this.notificationService.processNotification( + () => ProcessSnackbar, + {}, + { title: 'plugin_data_import_process_title', message: file.name }, + ); + + try { + const result = await this.graphQLService.sdk.uploadResultData(connectionId, contextId, projectId, resultsId, processorId, file, event => { + if (event.total !== undefined) { + const percentCompleted = getProgressPercent(event.loaded, event.total); + + if (notification.message) { + controller.setMessage(`${percentCompleted}%\n${notification.message}`); + } + } + }); + + const task = this.asyncTaskInfoService.create(async () => { + const { taskInfo } = await this.graphQLService.sdk.getAsyncTaskInfo({ taskId: result.id, removeOnFinish: false }); + return taskInfo; + }); + + controller.setMessage('plugin_data_import_process_file_processing_step_message'); + await this.asyncTaskInfoService.run(task); + + controller.resolve('plugin_data_import_process_success'); + return true; + } catch (exception: any) { + controller.reject(exception, 'plugin_data_import_process_fail'); + return false; + } + } +} diff --git a/webapp/packages/plugin-data-import/src/DataImportSettingsService.ts b/webapp/packages/plugin-data-import/src/DataImportSettingsService.ts new file mode 100644 index 0000000000..3ddf7579d7 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/DataImportSettingsService.ts @@ -0,0 +1,28 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import { injectable } from '@cloudbeaver/core-di'; +import { SettingsProvider, SettingsProviderService } from '@cloudbeaver/core-settings'; +import { schema, schemaExtra } from '@cloudbeaver/core-utils'; + +const defaultSettings = schema.object({ + 'plugin.data-import.disabled': schemaExtra.stringedBoolean().default(false), +}); + +export type DataImportSettings = schema.infer; + +@injectable() +export class DataImportSettingsService { + get disabled(): boolean { + return this.settings.getValue('plugin.data-import.disabled'); + } + readonly settings: SettingsProvider; + + constructor(private readonly settingsProviderService: SettingsProviderService) { + this.settings = this.settingsProviderService.createSettings(defaultSettings); + } +} diff --git a/webapp/packages/plugin-data-import/src/LocaleService.ts b/webapp/packages/plugin-data-import/src/LocaleService.ts new file mode 100644 index 0000000000..edeb82d255 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/LocaleService.ts @@ -0,0 +1,35 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import { Bootstrap, injectable } from '@cloudbeaver/core-di'; +import { LocalizationService } from '@cloudbeaver/core-localization'; + +@injectable() +export class LocaleService extends Bootstrap { + constructor(private readonly localizationService: LocalizationService) { + super(); + } + + register(): void | Promise { + this.localizationService.addProvider(this.provider.bind(this)); + } + + load(): void | Promise {} + + private async provider(locale: string) { + switch (locale) { + case 'ru': + return (await import('./locales/ru')).default; + case 'it': + return (await import('./locales/it')).default; + case 'zh': + return (await import('./locales/zh')).default; + default: + return (await import('./locales/en')).default; + } + } +} diff --git a/webapp/packages/plugin-data-import/src/index.ts b/webapp/packages/plugin-data-import/src/index.ts new file mode 100644 index 0000000000..b259ce9204 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/index.ts @@ -0,0 +1,3 @@ +export { dataImportPluginManifest } from './manifest'; + +export * from './DataImportSettingsService'; diff --git a/webapp/packages/plugin-data-import/src/locales/en.ts b/webapp/packages/plugin-data-import/src/locales/en.ts new file mode 100644 index 0000000000..3c505d522f --- /dev/null +++ b/webapp/packages/plugin-data-import/src/locales/en.ts @@ -0,0 +1,7 @@ +export default [ + ['plugin_data_import_title', 'Data import'], + ['plugin_data_import_process_title', 'Importing data...'], + ['plugin_data_import_process_success', 'Data imported successfully'], + ['plugin_data_import_process_fail', 'Data import failed'], + ['plugin_data_import_process_file_processing_step_message', 'File uploaded, processing...'], +]; diff --git a/webapp/packages/plugin-data-import/src/locales/it.ts b/webapp/packages/plugin-data-import/src/locales/it.ts new file mode 100644 index 0000000000..3c505d522f --- /dev/null +++ b/webapp/packages/plugin-data-import/src/locales/it.ts @@ -0,0 +1,7 @@ +export default [ + ['plugin_data_import_title', 'Data import'], + ['plugin_data_import_process_title', 'Importing data...'], + ['plugin_data_import_process_success', 'Data imported successfully'], + ['plugin_data_import_process_fail', 'Data import failed'], + ['plugin_data_import_process_file_processing_step_message', 'File uploaded, processing...'], +]; diff --git a/webapp/packages/plugin-data-import/src/locales/ru.ts b/webapp/packages/plugin-data-import/src/locales/ru.ts new file mode 100644 index 0000000000..7e08b40f55 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/locales/ru.ts @@ -0,0 +1,7 @@ +export default [ + ['plugin_data_import_title', 'Импорт данных'], + ['plugin_data_import_process_title', 'Импорт данных...'], + ['plugin_data_import_process_success', 'Данные успешно импортированы'], + ['plugin_data_import_process_fail', 'Ошибка импорта данных'], + ['plugin_data_import_process_file_processing_step_message', 'Файл загружен, обработка...'], +]; diff --git a/webapp/packages/plugin-data-import/src/locales/zh.ts b/webapp/packages/plugin-data-import/src/locales/zh.ts new file mode 100644 index 0000000000..3c505d522f --- /dev/null +++ b/webapp/packages/plugin-data-import/src/locales/zh.ts @@ -0,0 +1,7 @@ +export default [ + ['plugin_data_import_title', 'Data import'], + ['plugin_data_import_process_title', 'Importing data...'], + ['plugin_data_import_process_success', 'Data imported successfully'], + ['plugin_data_import_process_fail', 'Data import failed'], + ['plugin_data_import_process_file_processing_step_message', 'File uploaded, processing...'], +]; diff --git a/webapp/packages/plugin-data-import/src/manifest.ts b/webapp/packages/plugin-data-import/src/manifest.ts new file mode 100644 index 0000000000..72d8c6a278 --- /dev/null +++ b/webapp/packages/plugin-data-import/src/manifest.ts @@ -0,0 +1,22 @@ +/* + * CloudBeaver - Cloud Database Manager + * Copyright (C) 2020-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0. + * you may not use this file except in compliance with the License. + */ +import type { PluginManifest } from '@cloudbeaver/core-di'; + +import { DataImportBootstrap } from './DataImportBootstrap'; +import { DataImportProcessorsResource } from './DataImportProcessorsResource'; +import { DataImportService } from './DataImportService'; +import { DataImportSettingsService } from './DataImportSettingsService'; +import { LocaleService } from './LocaleService'; + +export const dataImportPluginManifest: PluginManifest = { + info: { + name: 'Data Import Plugin', + }, + + providers: [LocaleService, DataImportSettingsService, DataImportBootstrap, DataImportService, DataImportProcessorsResource], +}; diff --git a/webapp/packages/plugin-data-import/tsconfig.json b/webapp/packages/plugin-data-import/tsconfig.json new file mode 100644 index 0000000000..7e5478bb22 --- /dev/null +++ b/webapp/packages/plugin-data-import/tsconfig.json @@ -0,0 +1,76 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo" + }, + "references": [ + { + "path": "../core-blocks/tsconfig.json" + }, + { + "path": "../core-connections/tsconfig.json" + }, + { + "path": "../core-di/tsconfig.json" + }, + { + "path": "../core-dialogs/tsconfig.json" + }, + { + "path": "../core-events/tsconfig.json" + }, + { + "path": "../core-localization/tsconfig.json" + }, + { + "path": "../core-navigation-tree/tsconfig.json" + }, + { + "path": "../core-resource/tsconfig.json" + }, + { + "path": "../core-root/tsconfig.json" + }, + { + "path": "../core-sdk/tsconfig.json" + }, + { + "path": "../core-settings/tsconfig.json" + }, + { + "path": "../core-ui/tsconfig.json" + }, + { + "path": "../core-utils/tsconfig.json" + }, + { + "path": "../core-view/tsconfig.json" + }, + { + "path": "../plugin-data-viewer/tsconfig.json" + }, + { + "path": "../plugin-datasource-context-switch/tsconfig.json" + }, + { + "path": "../plugin-sql-editor/tsconfig.json" + }, + { + "path": "../plugin-top-app-bar/tsconfig.json" + } + ], + "include": [ + "__custom_mocks__/**/*", + "src/**/*", + "src/**/*.json", + "src/**/*.css", + "src/**/*.scss" + ], + "exclude": [ + "**/node_modules", + "lib/**/*", + "dist/**/*" + ] +} diff --git a/webapp/packages/product-default/package.json b/webapp/packages/product-default/package.json index 7d386f3798..b9d11e1f3d 100644 --- a/webapp/packages/product-default/package.json +++ b/webapp/packages/product-default/package.json @@ -41,6 +41,7 @@ "@cloudbeaver/plugin-connections": "~0.1.0", "@cloudbeaver/plugin-connections-administration": "~0.1.0", "@cloudbeaver/plugin-data-export": "~0.1.0", + "@cloudbeaver/plugin-data-import": "~0.1.0", "@cloudbeaver/plugin-data-spreadsheet-new": "~0.1.0", "@cloudbeaver/plugin-data-viewer": "~0.1.0", "@cloudbeaver/plugin-data-viewer-result-set-grouping": "~0.1.0", diff --git a/webapp/packages/product-default/src/index.ts b/webapp/packages/product-default/src/index.ts index 2fe35f24ab..0d40e727c7 100644 --- a/webapp/packages/product-default/src/index.ts +++ b/webapp/packages/product-default/src/index.ts @@ -13,6 +13,7 @@ import connectionTemplate from '@cloudbeaver/plugin-connection-template'; import connectionPlugin from '@cloudbeaver/plugin-connections'; import connectionAdministration from '@cloudbeaver/plugin-connections-administration'; import { dataExportManifest } from '@cloudbeaver/plugin-data-export'; +import { dataImportPluginManifest } from '@cloudbeaver/plugin-data-import'; import { dataSpreadsheetNewManifest } from '@cloudbeaver/plugin-data-spreadsheet-new'; import { dataViewerManifest } from '@cloudbeaver/plugin-data-viewer'; import { dvResultSetGroupingPlugin } from '@cloudbeaver/plugin-data-viewer-result-set-grouping'; @@ -54,9 +55,9 @@ import topAppBar from '@cloudbeaver/plugin-top-app-bar'; import { administrationTopAppBarPlugin } from '@cloudbeaver/plugin-top-app-bar-administration'; import userProfile from '@cloudbeaver/plugin-user-profile'; import userProfileAdministration from '@cloudbeaver/plugin-user-profile-administration'; +import { userProfileSettingsPlugin } from '@cloudbeaver/plugin-user-profile-settings'; import version from '@cloudbeaver/plugin-version'; import versionUpdate from '@cloudbeaver/plugin-version-update-administration'; -import { userProfileSettingsPlugin } from '@cloudbeaver/plugin-user-profile-settings'; import { defaultProductManifest } from './manifest'; @@ -73,6 +74,7 @@ const PLUGINS: PluginManifest[] = [ connectionTemplate, connectionSearch, dataExportManifest, + dataImportPluginManifest, dataViewerManifest, dvResultSetGroupingPlugin, gisViewer, diff --git a/webapp/packages/product-default/tsconfig.json b/webapp/packages/product-default/tsconfig.json index b5eb58afc1..bce1c9bef9 100644 --- a/webapp/packages/product-default/tsconfig.json +++ b/webapp/packages/product-default/tsconfig.json @@ -54,6 +54,9 @@ { "path": "../plugin-data-export/tsconfig.json" }, + { + "path": "../plugin-data-import/tsconfig.json" + }, { "path": "../plugin-data-spreadsheet-new/tsconfig.json" },