dbeaver-corp/dbeaver-web#620 Data transfer: dt processors + async api

samples/sample-databases/cloudbeaver-sample.json

@@ -1,20 +1,7 @@
 {
   // Server configuration
-  "server": {
-    "serverName": "Cloudbeaver Sample Server",
-    "port": 8978,
-    "workspaceLocation": "workspace",
-    "contentRoot": "web",
-    "driversLocation": "drivers",
-    "develMode": false
-  },
-
-  "ui": {
-    "plugins": {
-      "logView": {
-        "logRefreshInterval": 3,
-        "logBatchSize": 1000
-      }
-    }
+  "logView": {
+    "logRefreshInterval": 3,
+    "logBatchSize": 1000
   }
 }

server/bundles/io.cloudbeaver.server/schema/schema.graphqls

@@ -20,6 +20,9 @@ type AsyncTaskInfo {
     error: ServerError
 
     result: SQLExecuteInfo
+    # Task result.
+    # Can be some kind of identifier to obtain real result using another API function
+    taskResult: Object
 }
 
 # Various server errors descriptor

server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/graphql/GraphQLEndpoint.java

@@ -234,7 +234,9 @@ public class GraphQLEndpoint extends HttpServlet {
 //                    apiCall += " (" + variables + ")";
 //                }
 //            }
-            log.debug("API > " + apiCall);
+            if (apiCall != null) {
+                log.debug("API > " + apiCall);
+            }
         }
         ExecutionInput executionInput = contextBuilder.build();
         ExecutionResult executionResult = graphQL.execute(executionInput);

server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/model/WebAsyncTaskInfo.java

@@ -61,6 +61,10 @@ public class WebAsyncTaskInfo {
         this.running = running;
     }
 
+    public Object getTaskResult() {
+        return result;
+    }
+
     public Object getResult() {
         return result;
     }

server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/model/WebPropertyInfo.java

@@ -44,6 +44,11 @@ public class WebPropertyInfo {
         this.propertySource = propertySource;
     }
 
+    public WebPropertyInfo(WebSession session, DBPPropertyDescriptor property) {
+        this.session = session;
+        this.property = property;
+    }
+
     ///////////////////////////////////
     // General properties
     ///////////////////////////////////
@@ -94,14 +99,15 @@ public class WebPropertyInfo {
 
     @Property
     public Object getValue() throws DBException {
-        Object value = propertySource.getPropertyValue(session.getProgressMonitor(), property.getId());
+        Object value = propertySource == null ? null : propertySource.getPropertyValue(session.getProgressMonitor(), property.getId());
         return value == null ? null : makePropertyValue(value);
     }
 
     @Property
     public Object[] getValidValues() {
         if (property instanceof IPropertyValueListProvider) {
-            Object[] possibleValues = ((IPropertyValueListProvider) property).getPossibleValues(propertySource.getEditableValue());
+            Object[] possibleValues = ((IPropertyValueListProvider) property).getPossibleValues(
+                propertySource == null ? null : propertySource.getEditableValue());
             if (possibleValues != null) {
                 Object[] validValues = new Object[possibleValues.length];
                 for (int i = 0; i < possibleValues.length; i++) {

server/bundles/io.cloudbeaver.service.data.transfer/META-INF/MANIFEST.MF

@@ -14,6 +14,7 @@ Require-Bundle: org.eclipse.core.runtime,
  org.eclipse.core.commands,
  org.eclipse.core.resources,
  io.cloudbeaver.server,
+ org.jkiss.dbeaver.data.transfer,
  org.jkiss.dbeaver.model,
  org.jkiss.dbeaver.model.sql
 Bundle-Localization: OSGI-INF/l10n/bundle

server/bundles/io.cloudbeaver.service.data.transfer/schema/service.data.transfer.graphqls

@@ -1,9 +1,10 @@
 # Data transfer queries
 
 type DataTransferProcessorInfo {
-    id: String!
+    id: ID!
     name: String
     description: String
+    fileExtension: String
     appFileExtension: String
     appName: String
     order: Int!
@@ -13,23 +14,36 @@ type DataTransferProcessorInfo {
     isHTML: Boolean
 }
 
+input DataTransferParameters {
+    # Processor ID
+    processorId: ID!
+    # General settings:
+    #   - openNewConnection: opens new database connection for data transfer task
+    settings: Object
+    # Processor properties. See DataTransferProcessorInfo.properties
+    processorProperties: Object
+    # Data filter settings
+    filter: SQLDataFilter
+}
 
 extend type Query {
 
     # Available transfer processors
     dataTransferAvailableStreamProcessors: [ DataTransferProcessorInfo ]
 
-    asyncDataTransferExportDataFromContainer(
+    dataTransferExportDataFromContainer(
         connectionId: ID!,
-        contextId: ID!,
         containerNodePath: ID!,
-        filter: SQLDataFilter
+        parameters: DataTransferParameters!
     ): AsyncTaskInfo!
 
-    asyncDataTransferExportDataFromResults(
+    dataTransferExportDataFromResults(
         connectionId: ID!,
         contextId: ID!,
-        resultsId: ID!
+        resultsId: ID!,
+        parameters: DataTransferParameters!
     ): AsyncTaskInfo!
 
+    dataTransferRemoveDataFile(dataFileId: String!): Boolean
+
 }

server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebDataTransferManager.java

@@ -0,0 +1,84 @@
+/*
+ * DBeaver - Universal Database Manager
+ * Copyright (C) 2010-2020 DBeaver Corp and others
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.cloudbeaver.service.data.transfer;
+
+import io.cloudbeaver.api.DBWModel;
+import io.cloudbeaver.server.model.WebAsyncTaskInfo;
+import io.cloudbeaver.server.model.session.WebSession;
+import org.jkiss.dbeaver.model.meta.RuntimeAction;
+import org.jkiss.dbeaver.model.struct.DBSEntity;
+import org.jkiss.dbeaver.tools.transfer.registry.DataTransferProcessorDescriptor;
+import org.jkiss.dbeaver.tools.transfer.registry.DataTransferRegistry;
+import org.jkiss.dbeaver.tools.transfer.stream.StreamTransferConsumer;
+import org.jkiss.utils.CommonUtils;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Web service implementation
+ */
+public class WebDataTransferManager {
+
+    private final DBWModel model;
+
+    public WebDataTransferManager(DBWModel model) {
+        this.model = model;
+    }
+
+    @RuntimeAction
+    public List<WebDataTransferStreamProcessor> getAvailableStreamProcessors(WebSession session) {
+        List<DataTransferProcessorDescriptor> processors = DataTransferRegistry.getInstance().getAvailableProcessors(StreamTransferConsumer.class, DBSEntity.class);
+        if (CommonUtils.isEmpty(processors)) {
+            return Collections.emptyList();
+        }
+
+        return processors.stream().map(x -> new WebDataTransferStreamProcessor(session, x)).collect(Collectors.toList());
+    }
+
+    @RuntimeAction
+    public WebAsyncTaskInfo dataTransferExportDataFromContainer(
+        WebSession session,
+        String connectionId,
+        String containerNodePath,
+        WebDataTransferParameters parameters)
+    {
+
+        return null;
+    }
+
+    @RuntimeAction
+    public WebAsyncTaskInfo dataTransferExportDataFromResults(
+        WebSession session,
+        String connectionId,
+        String contextId,
+        String resultsId,
+        WebDataTransferParameters parameters)
+    {
+        return null;
+    }
+
+    @RuntimeAction
+    public Boolean dataTransferRemoveDataFile(WebSession session, String dataFileId) {
+        return true;
+    }
+
+    private WebDataTransferSessionConfig getConfig(WebSession session) {
+        return session.getAttribute("dataTransfer", x -> new WebDataTransferSessionConfig());
+    }
+}

server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebDataTransferParameters.java

@@ -0,0 +1,45 @@
+package io.cloudbeaver.service.data.transfer;
+
+import io.cloudbeaver.server.model.sql.WebSQLDataFilter;
+
+import java.util.Map;
+
+public class WebDataTransferParameters {
+
+    private String processorId;
+    private Map<String, Object> settings;
+    private Map<String, Object> processorProperties;
+    private WebSQLDataFilter filter;
+
+    public String getProcessorId() {
+        return processorId;
+    }
+
+    public void setProcessorId(String processorId) {
+        this.processorId = processorId;
+    }
+
+    public Map<String, Object> getSettings() {
+        return settings;
+    }
+
+    public void setSettings(Map<String, Object> settings) {
+        this.settings = settings;
+    }
+
+    public Map<String, Object> getProcessorProperties() {
+        return processorProperties;
+    }
+
+    public void setProcessorProperties(Map<String, Object> processorProperties) {
+        this.processorProperties = processorProperties;
+    }
+
+    public WebSQLDataFilter getFilter() {
+        return filter;
+    }
+
+    public void setFilter(WebSQLDataFilter filter) {
+        this.filter = filter;
+    }
+}

server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebDataTransferSessionConfig.java

@@ -0,0 +1,12 @@
+package io.cloudbeaver.service.data.transfer;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class WebDataTransferSessionConfig {
+
+    private Map<String, WebDataTransferTaskConfig> tasks = new HashMap<>();
+
+
+
+}

server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebDataTransferStreamProcessor.java

@@ -0,0 +1,82 @@
+package io.cloudbeaver.service.data.transfer;
+
+import io.cloudbeaver.server.model.WebPropertyInfo;
+import io.cloudbeaver.server.model.session.WebSession;
+import org.jkiss.code.NotNull;
+import org.jkiss.dbeaver.model.meta.Property;
+import org.jkiss.dbeaver.model.preferences.DBPPropertyDescriptor;
+import org.jkiss.dbeaver.tools.transfer.registry.DataTransferProcessorDescriptor;
+import org.jkiss.utils.CommonUtils;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class WebDataTransferStreamProcessor {
+
+    private final WebSession session;
+    private final DataTransferProcessorDescriptor processor;
+
+    public WebDataTransferStreamProcessor(WebSession session, DataTransferProcessorDescriptor processor) {
+        this.session = session;
+        this.processor = processor;
+    }
+
+    @Property
+    public String getId() {
+        return processor.getId();
+    }
+
+    @Property
+    public String getName() {
+        return processor.getName();
+    }
+
+    @Property
+    public String getDescription() {
+        return processor.getDescription();
+    }
+
+    @Property
+    public String getFileExtension() {
+        DBPPropertyDescriptor extProperty = processor.getProperty("extension");
+        return extProperty == null ? getAppFileExtension() : CommonUtils.toString(extProperty.getDefaultValue(), null);
+    }
+
+    @Property
+    public String getAppFileExtension() {
+        return processor.getAppFileExtension();
+    }
+
+    @Property
+    public String getAppName() {
+        return processor.getAppName();
+    }
+
+    @Property
+    public int getOrder() {
+        return processor.getOrder();
+    }
+
+    @Property
+    @NotNull
+    public String getIcon() {
+        return processor.getIcon().getLocation();
+    }
+
+    @Property
+    public List<WebPropertyInfo> getProperties() {
+        return Arrays.stream(processor.getProperties()).map(x -> new WebPropertyInfo(session, x)).collect(Collectors.toList());
+    }
+
+    @Property
+    public boolean isBinaryFormat() {
+        return processor.isBinaryFormat();
+    }
+
+    @Property
+    public boolean isHTMLFormat() {
+        return processor.isHTMLFormat();
+    }
+
+}

server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebDataTransferTaskConfig.java

@@ -0,0 +1,14 @@
+package io.cloudbeaver.service.data.transfer;
+
+import org.jkiss.dbeaver.tools.transfer.registry.DataTransferProcessorDescriptor;
+
+import java.util.Map;
+
+public class WebDataTransferTaskConfig {
+
+    private String dataFileId;
+    private DataTransferProcessorDescriptor processor;
+    private Map<String, Object> settings;
+    private Map<String, Object> processorProperties;
+
+}

server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebServiceDataTransfer.java

@@ -16,13 +16,16 @@
  */
 package io.cloudbeaver.service.data.transfer;
 
+import graphql.schema.DataFetchingEnvironment;
 import graphql.schema.idl.TypeDefinitionRegistry;
 import io.cloudbeaver.DBWebException;
 import io.cloudbeaver.WebServiceUtils;
 import io.cloudbeaver.api.DBWModel;
 import io.cloudbeaver.api.DBWServiceGraphQL;
 import io.cloudbeaver.api.DBWServiceServlet;
+import io.cloudbeaver.api.DBWUtils;
 import io.cloudbeaver.server.CloudbeaverApplication;
+import io.cloudbeaver.server.model.session.WebSession;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 
@@ -33,37 +36,42 @@ public class WebServiceDataTransfer implements DBWServiceGraphQL, DBWServiceServ
 
     private static final String DT_SCHEMA_FILE_NAME = "schema/service.data.transfer.graphqls";
 
+    private WebDataTransferManager dtManager;
+
     @Override
     public TypeDefinitionRegistry getTypeDefinition() throws DBWebException {
         return WebServiceUtils.loadSchemaDefinition(getClass(), DT_SCHEMA_FILE_NAME);
     }
 
     @Override
-    public void bindWiring(DBWModel model) throws DBWebException {
-        model.getQueryType().dataFetcher("data.dtGetAvailableStreamFormats", env -> {
-            return null;
+    public void bindWiring(DBWModel model) {
+        dtManager = new WebDataTransferManager(model);
+        model.getQueryType()
+            .dataFetcher("dataTransferAvailableStreamProcessors",
+                env -> dtManager.getAvailableStreamProcessors(getWebSession(model, env)))
+            .dataFetcher("dataTransferExportDataFromContainer", env -> dtManager.dataTransferExportDataFromContainer(
+                getWebSession(model, env),
+                env.getArgument("connectionId"),
+                env.getArgument("containerNodePath"),
+                env.getArgument("parameters")
+            ))
+            .dataFetcher("dataTransferExportDataFromResults", env -> dtManager.dataTransferExportDataFromResults(
+                getWebSession(model, env),
+                env.getArgument("connectionId"),
+                env.getArgument("contextId"),
+                env.getArgument("resultsId"),
+                env.getArgument("parameters")
+            ))
+            .dataFetcher("dataTransferRemoveDataFile", env -> dtManager.dataTransferRemoveDataFile(
+                getWebSession(model, env),
+                env.getArgument("dataFileId")
+            ))
+        ;
 
-/*
-            WebSession webSession = model.getSessionManager().getWebSession(DBWUtils.getServletRequest(env));
-            WebNavigatorNodeInfo node = webSession.getNavigatorNodeInfo(env.getArgument("nodeId"));
-            DBNNode dbNode = node.getNode();
-            if (dbNode instanceof DBNDatabaseNode) {
-                DBSObject object = ((DBNDatabaseNode) dbNode).getObject();
-                if (object instanceof DBPScriptObject) {
-                    Map<String, Object> options = env.getArgument("options");
-                    if (options == null) {
-                        options = new LinkedHashMap<>();
-                    }
-                    return ((DBPScriptObject) object).getObjectDefinitionText(webSession.getProgressMonitor(), options);
-                } else {
-                    throw new DBWebException("Object '" + node.getId() + "' doesn't support DDL");
-                }
-            } else {
-                throw new DBWebException("Node '" + node.getId() + "' is not database node");
-            }
-*/
-        });
+    }
 
+    private WebSession getWebSession(DBWModel model, DataFetchingEnvironment env) throws DBWebException {
+        return model.getSessionManager().getWebSession(DBWUtils.getServletRequest(env));
     }
 
     @Override