From 8bc4a683481f915ebadca5fbcdd933e9b4cf88e2 Mon Sep 17 00:00:00 2001 From: Aleksandr Skoblikov Date: Tue, 13 Sep 2022 14:44:05 +0300 Subject: [PATCH] CB-2328 data export output params --- .../schema/service.data.transfer.graphqls | 21 +++++++++++ .../data/transfer/DBWServiceDataTransfer.java | 2 ++ .../WebServiceBindingDataTransfer.java | 1 + .../WebDataTransferDefaultExportSettings.java | 29 +++++++++++++++ .../impl/WebDataTransferOutputSettings.java | 35 +++++++++++++++++++ .../impl/WebDataTransferParameters.java | 20 +++++++---- .../transfer/impl/WebServiceDataTransfer.java | 17 +++++++-- 7 files changed, 117 insertions(+), 8 deletions(-) create mode 100644 server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferDefaultExportSettings.java create mode 100644 server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferOutputSettings.java diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/schema/service.data.transfer.graphqls b/server/bundles/io.cloudbeaver.service.data.transfer/schema/service.data.transfer.graphqls index f4ee2c59a5..773d47932b 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/schema/service.data.transfer.graphqls +++ b/server/bundles/io.cloudbeaver.service.data.transfer/schema/service.data.transfer.graphqls @@ -14,6 +14,23 @@ type DataTransferProcessorInfo { isHTML: Boolean } +input DataTransferOutputSettingsInput { + insertBom: Boolean + encoding: Boolean + timestampPattern: Boolean +} + +type DataTransferOutputSettings { + insertBom: Boolean! + encoding: Boolean! + timestampPattern: Boolean! +} + +type DataTransferDefaultExportSettings { + outputSettings : DataTransferOutputSettings! + supportedEncodings: [String!]! +} + input DataTransferParameters { # Processor ID processorId: ID! @@ -22,6 +39,8 @@ input DataTransferParameters { settings: Object # Processor properties. See DataTransferProcessorInfo.properties processorProperties: Object! + # Consumer properties. See StreamConsumerSettings + outputSettings: DataTransferOutputSettingsInput # Data filter settings filter: SQLDataFilter } @@ -31,6 +50,8 @@ extend type Query { # Available transfer processors dataTransferAvailableStreamProcessors: [ DataTransferProcessorInfo! ]! + dataTransferDefaultExportSettings: DataTransferDefaultExportSettings! + dataTransferExportDataFromContainer( projectId: ID, connectionId: ID!, diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/DBWServiceDataTransfer.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/DBWServiceDataTransfer.java index 813433facc..4f158e3cc0 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/DBWServiceDataTransfer.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/DBWServiceDataTransfer.java @@ -21,6 +21,7 @@ import io.cloudbeaver.DBWebException; import io.cloudbeaver.WebAction; import io.cloudbeaver.model.WebAsyncTaskInfo; import io.cloudbeaver.model.session.WebSession; +import io.cloudbeaver.service.data.transfer.impl.WebDataTransferDefaultExportSettings; import io.cloudbeaver.service.data.transfer.impl.WebDataTransferParameters; import io.cloudbeaver.service.data.transfer.impl.WebDataTransferStreamProcessor; import io.cloudbeaver.service.sql.WebSQLContextInfo; @@ -51,4 +52,5 @@ public interface DBWServiceDataTransfer extends DBWService { @WebAction Boolean dataTransferRemoveDataFile(WebSession session, String dataFileId) throws DBWebException; + WebDataTransferDefaultExportSettings defaultExportSettings(); } diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebServiceBindingDataTransfer.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebServiceBindingDataTransfer.java index 2a552c9138..c87a5ad872 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebServiceBindingDataTransfer.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebServiceBindingDataTransfer.java @@ -56,6 +56,7 @@ public class WebServiceBindingDataTransfer extends WebServiceBindingBase getService(env).defaultExportSettings()) ; } diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferDefaultExportSettings.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferDefaultExportSettings.java new file mode 100644 index 0000000000..0c0b6567eb --- /dev/null +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferDefaultExportSettings.java @@ -0,0 +1,29 @@ +package io.cloudbeaver.service.data.transfer.impl; + +import org.jkiss.dbeaver.tools.transfer.stream.StreamConsumerSettings; + +import java.nio.charset.Charset; +import java.util.Set; + +public class WebDataTransferDefaultExportSettings { + private final WebDataTransferOutputSettings outputSettings; + private final Set supportedEncodings; + + public WebDataTransferDefaultExportSettings() { + var defConsumerSettings = new StreamConsumerSettings(); + this.outputSettings = new WebDataTransferOutputSettings( + false, + defConsumerSettings.getOutputEncoding(), + defConsumerSettings.getOutputTimestampPattern() + ); + this.supportedEncodings = Charset.availableCharsets().keySet(); + } + + public WebDataTransferOutputSettings getOutputSettings() { + return outputSettings; + } + + public Set getSupportedEncodings() { + return supportedEncodings; + } +} diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferOutputSettings.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferOutputSettings.java new file mode 100644 index 0000000000..71ff9b9b63 --- /dev/null +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferOutputSettings.java @@ -0,0 +1,35 @@ +package io.cloudbeaver.service.data.transfer.impl; + +import org.jkiss.dbeaver.model.data.json.JSONUtils; + +import java.util.Map; + +public class WebDataTransferOutputSettings { + private final boolean insertBom; + private final String encoding; + private final String timestampPattern; + + public WebDataTransferOutputSettings(Map outputSettings) { + this.insertBom = JSONUtils.getBoolean(outputSettings, "insertBom", false); + this.encoding = JSONUtils.getString(outputSettings, "encoding"); + this.timestampPattern = JSONUtils.getString(outputSettings, "timestampPattern"); + } + + public WebDataTransferOutputSettings(boolean insertBom, String encoding, String timestampPattern) { + this.insertBom = insertBom; + this.encoding = encoding; + this.timestampPattern = timestampPattern; + } + + public boolean isInsertBom() { + return insertBom; + } + + public String getEncoding() { + return encoding; + } + + public String getTimestampPattern() { + return timestampPattern; + } +} diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferParameters.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferParameters.java index d04362dbac..aba200b50c 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferParameters.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferParameters.java @@ -2,24 +2,28 @@ package io.cloudbeaver.service.data.transfer.impl; import io.cloudbeaver.service.sql.WebSQLDataFilter; import org.jkiss.dbeaver.model.data.json.JSONUtils; +import org.jkiss.dbeaver.tools.transfer.stream.StreamConsumerSettings; import java.util.Map; public class WebDataTransferParameters { private String processorId; - private Map settings; + private Map dbProducerSettings; private Map processorProperties; private WebSQLDataFilter filter; + private WebDataTransferOutputSettings outputSettings; public WebDataTransferParameters() { } public WebDataTransferParameters(Map params) { this.processorId = JSONUtils.getString(params, "processorId"); - this.settings = JSONUtils.getObject(params, "settings"); + this.dbProducerSettings = JSONUtils.getObject(params, "settings"); this.processorProperties = JSONUtils.getObject(params, "processorProperties"); this.filter = new WebSQLDataFilter(JSONUtils.getObject(params, "filter")); + + } public String getProcessorId() { @@ -30,12 +34,12 @@ public class WebDataTransferParameters { this.processorId = processorId; } - public Map getSettings() { - return settings; + public Map getDbProducerSettings() { + return dbProducerSettings; } - public void setSettings(Map settings) { - this.settings = settings; + public void setDbProducerSettings(Map dbProducerSettings) { + this.dbProducerSettings = dbProducerSettings; } public Map getProcessorProperties() { @@ -53,4 +57,8 @@ public class WebDataTransferParameters { public void setFilter(WebSQLDataFilter filter) { this.filter = filter; } + + public WebDataTransferOutputSettings getOutputSettings() { + return outputSettings; + } } \ No newline at end of file diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebServiceDataTransfer.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebServiceDataTransfer.java index 81fc63357f..42bf3dfb54 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebServiceDataTransfer.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebServiceDataTransfer.java @@ -142,6 +142,11 @@ public class WebServiceDataTransfer implements DBWServiceDataTransfer { return true; } + @Override + public WebDataTransferDefaultExportSettings defaultExportSettings() { + return new WebDataTransferDefaultExportSettings(); + } + private WebAsyncTaskInfo asyncExportFromDataContainer(WebSQLProcessor sqlProcessor, WebDataTransferParameters parameters, DBSDataContainer dataContainer, @Nullable WebSQLResultsInfo resultsInfo) { sqlProcessor.getWebSession().addInfoMessage("Export data"); @@ -213,10 +218,18 @@ public class WebServiceDataTransfer implements DBWServiceDataTransfer { StreamConsumerSettings settings = new StreamConsumerSettings(); - settings.setOutputEncodingBOM(false); settings.setOutputFolder(exportFile.getParent().toAbsolutePath().toString()); settings.setOutputFilePattern(exportFile.getFileName().toString()); + WebDataTransferOutputSettings outputSettings = parameters.getOutputSettings(); + settings.setOutputEncodingBOM(outputSettings.isInsertBom()); + if (!CommonUtils.isEmpty(outputSettings.getEncoding())) { + settings.setOutputEncoding(outputSettings.getEncoding()); + } + if (!CommonUtils.isEmpty(outputSettings.getTimestampPattern())) { + settings.setOutputTimestampPattern(outputSettings.getTimestampPattern()); + } + Map properties = new HashMap<>(); Map processorProperties = parameters.getProcessorProperties(); @@ -241,7 +254,7 @@ public class WebServiceDataTransfer implements DBWServiceDataTransfer { DatabaseProducerSettings producerSettings = new DatabaseProducerSettings(); producerSettings.setExtractType(DatabaseProducerSettings.ExtractType.SINGLE_QUERY); producerSettings.setQueryRowCount(false); - producerSettings.setOpenNewConnections(CommonUtils.getOption(parameters.getSettings(), "openNewConnection")); + producerSettings.setOpenNewConnections(CommonUtils.getOption(parameters.getDbProducerSettings(), "openNewConnection")); producer.transferData(monitor, consumer, null, producerSettings, null);