From 0198df5406b385c80b0536b0905fee3f0719f635 Mon Sep 17 00:00:00 2001
From: vahid <jalili.vahid@gmail.com>
Date: Tue, 13 Feb 2018 12:34:04 -0800
Subject: [PATCH 01/41] Implement overall structure.

---
 lib/galaxy/managers/cloud_storages.py         | 22 +++++++
 .../webapps/galaxy/api/cloud_storage.py       | 64 +++++++++++++++++++
 2 files changed, 86 insertions(+)
 create mode 100644 lib/galaxy/managers/cloud_storages.py
 create mode 100644 lib/galaxy/webapps/galaxy/api/cloud_storage.py

diff --git a/lib/galaxy/managers/cloud_storages.py b/lib/galaxy/managers/cloud_storages.py
new file mode 100644
index 00000000000..787c1d4a158
--- /dev/null
+++ b/lib/galaxy/managers/cloud_storages.py
@@ -0,0 +1,22 @@
+"""
+Manager and serializer for cloud-based storages.
+"""
+
+from galaxy.managers import sharable
+
+import logging
+log = logging.getLogger(__name__)
+
+
+class CloudStoragesManager(sharable.SharableModelManager):
+
+    def __init__(self, app, *args, **kwargs):
+        super(CloudStoragesManager, self).__init__(app, *args, **kwargs)
+
+    def download(self, provider, container, object):
+        # TODO: implement the download logic.
+        pass
+
+    def upload(self, dataset, provider, container, object):
+        # TODO: implement the upload logic.
+        pass
diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
new file mode 100644
index 00000000000..9e95b6a5d91
--- /dev/null
+++ b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
@@ -0,0 +1,64 @@
+"""
+API operations on Cloud-based storages, such as Amazon Simple Storage Service (S3).
+"""
+
+import logging
+
+from galaxy import web
+from galaxy.web.base.controller import BaseAPIController
+from galaxy.managers import cloud_storages
+
+log = logging.getLogger(__name__)
+
+
+class CloudStorageController(BaseAPIController):
+    """
+    RESTfull controller for interaction with Amazon S3.
+    """
+
+    def __init__(self, app):
+        super(CloudStorageController, self).__init__(app)
+
+    @web.expose_api
+    def index(self, trans, **kwargs):
+        """
+        * GET /api/cloud_storage
+            Lists cloud-based containers (e.g., S3 bucket, Azure blob) user has defined.
+        :param trans:
+        :param kwargs:
+        :return: A list of cloud-based containers user has defined.
+        """
+        # TODO: This can be implemented leveraging PluggedMedia objects (part of the user-based object store project)
+        trans.response.status = 501
+        return 'Not Implemented'
+
+    @web.expose_api
+    def download(self, trans, payload, **kwargs):
+        """
+        * POST /api/cloud_storage/download
+            Downloads a given object from a given cloud-based container.
+        :type  trans: galaxy.web.framework.webapp.GalaxyWebTransaction
+        :param trans: Galaxy web transaction
+
+        :type  payload: dict
+        :param payload: A dictionary structure containing the following keys:
+
+
+        :param kwargs:
+
+        :rtype:  boolean
+        :return: True/False if the given object is successfully downloaded from the cloud-based storage.
+        """
+        pass
+
+    @web.expose_api
+    def upload(self, trans, payload, **kwargs):
+        """
+        * POST /api/cloud_storage/upload
+            Uploads a given dataset to a given cloud-based container.
+        :param trans:
+        :param payload:
+        :param kwargs:
+        :return:
+        """
+        pass
\ No newline at end of file

From bdc98d5569dabcf6b82b2d447e75821adb080d03 Mon Sep 17 00:00:00 2001
From: vahid <jalili.vahid@gmail.com>
Date: Tue, 27 Feb 2018 14:50:35 -0800
Subject: [PATCH 02/41] Add cloud storage api mapper.

---
 lib/galaxy/webapps/galaxy/buildapp.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py
index c86579acd67..3171bef42a1 100644
--- a/lib/galaxy/webapps/galaxy/buildapp.py
+++ b/lib/galaxy/webapps/galaxy/buildapp.py
@@ -226,6 +226,14 @@ def populate_api_routes(webapp, app):
                               path_prefix='/api/histories/{history_id}/contents/{history_content_id}')
     webapp.mapper.connect('/api/histories/published', action='published', controller="histories", conditions=dict(method=["GET"]))
     webapp.mapper.connect('/api/histories/shared_with_me', action='shared_with_me', controller="histories")
+
+    webapp.mapper.resource('cloud_storage', 'cloud_storage', path_prefix='/api')
+    webapp.mapper.connect('cloud_storage_download',
+                          '/api/cloud_storage/download',
+                          controller='cloud_storage',
+                          action='download',
+                          conditions=dict(method=["POST"]))
+
     _add_item_tags_controller(webapp,
                               name_prefix="history_",
                               path_prefix='/api/histories/{history_id}')

From 645f74b6b6ae9ded06f8f073c08bae0519ed38c3 Mon Sep 17 00:00:00 2001
From: vahid <jalili.vahid@gmail.com>
Date: Tue, 27 Feb 2018 14:56:10 -0800
Subject: [PATCH 03/41] Capture invalid type of cloud storage api payload, &
 raise exception.

---
 lib/galaxy/webapps/galaxy/api/cloud_storage.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
index 9e95b6a5d91..1c63f89f0e8 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud_storage.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
@@ -49,7 +49,11 @@ class CloudStorageController(BaseAPIController):
         :rtype:  boolean
         :return: True/False if the given object is successfully downloaded from the cloud-based storage.
         """
-        pass
+        if not isinstance(payload, dict):
+            trans.response.status = 400
+            return "Invalid payload data type. The payload is expected to be a dictionary, " \
+                   "but received data of type `%s`." % str(type(payload))
+
 
     @web.expose_api
     def upload(self, trans, payload, **kwargs):

From 140a1f7a8ea8455e4a96fa6e2dbb5148ebb3fe0e Mon Sep 17 00:00:00 2001
From: vahid <jalili.vahid@gmail.com>
Date: Tue, 27 Feb 2018 15:20:51 -0800
Subject: [PATCH 04/41] Unbox payload & check if it contains all data required
 for cloud storage

---
 .../webapps/galaxy/api/cloud_storage.py       | 20 +++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
index 1c63f89f0e8..d9c1fff9f28 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud_storage.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
@@ -51,8 +51,24 @@ class CloudStorageController(BaseAPIController):
         """
         if not isinstance(payload, dict):
             trans.response.status = 400
-            return "Invalid payload data type. The payload is expected to be a dictionary, " \
-                   "but received data of type `%s`." % str(type(payload))
+            return {'status': 'error',
+                    'message': 'Invalid payload data type. The payload is expected to be a dictionary, '
+                               'but received data of type `%s`.' % str(type(payload))}
+
+        missing_arguments = []
+        bucket = payload.get("bucket", None)
+        if bucket is None:
+            missing_arguments.append("bucket")
+
+        obj = payload.get("object", None)
+        if obj is None:
+            missing_arguments.append("object")
+
+        if len(missing_arguments) > 0:
+            trans.response.status = 400
+            return {'status': 'error',
+                    'message': "The following required arguments are missing in the payload: %s" % missing_arguments}
+
 
 
     @web.expose_api

From 15293d652c724635f294376cb5a23bf4e6d3419f Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Tue, 13 Mar 2018 14:54:50 -0700
Subject: [PATCH 05/41] Some refactoring: "Storages" -> "Storage" & "object" ->
 "obj"

---
 .../managers/{cloud_storages.py => cloud_storage.py}      | 8 ++++----
 lib/galaxy/webapps/galaxy/api/cloud_storage.py            | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)
 rename lib/galaxy/managers/{cloud_storages.py => cloud_storage.py} (56%)

diff --git a/lib/galaxy/managers/cloud_storages.py b/lib/galaxy/managers/cloud_storage.py
similarity index 56%
rename from lib/galaxy/managers/cloud_storages.py
rename to lib/galaxy/managers/cloud_storage.py
index 787c1d4a158..5e61c2a67f2 100644
--- a/lib/galaxy/managers/cloud_storages.py
+++ b/lib/galaxy/managers/cloud_storage.py
@@ -8,15 +8,15 @@ import logging
 log = logging.getLogger(__name__)
 
 
-class CloudStoragesManager(sharable.SharableModelManager):
+class CloudStorageManager(sharable.SharableModelManager):
 
     def __init__(self, app, *args, **kwargs):
-        super(CloudStoragesManager, self).__init__(app, *args, **kwargs)
+        super(CloudStorageManager, self).__init__(app, *args, **kwargs)
 
-    def download(self, provider, container, object):
+    def download(self, provider, container, obj):
         # TODO: implement the download logic.
         pass
 
-    def upload(self, dataset, provider, container, object):
+    def upload(self, dataset, provider, container, obj):
         # TODO: implement the upload logic.
         pass
diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
index d9c1fff9f28..56b626ece9b 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud_storage.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
@@ -6,7 +6,7 @@ import logging
 
 from galaxy import web
 from galaxy.web.base.controller import BaseAPIController
-from galaxy.managers import cloud_storages
+from galaxy.managers import cloud_storage
 
 log = logging.getLogger(__name__)
 

From ca0bb57add1c66237e3e6ceff3a0afbac91ba26f Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Tue, 13 Mar 2018 15:34:46 -0700
Subject: [PATCH 06/41] Add provider as a payload requirement to the cloud
 storage API.

---
 lib/galaxy/webapps/galaxy/api/cloud_storage.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
index 56b626ece9b..633a1b734b9 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud_storage.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
@@ -56,6 +56,10 @@ class CloudStorageController(BaseAPIController):
                                'but received data of type `%s`.' % str(type(payload))}
 
         missing_arguments = []
+        provider = payload.get("provider", None)
+        if provider is None:
+            missing_arguments.append("provider")
+
         bucket = payload.get("bucket", None)
         if bucket is None:
             missing_arguments.append("bucket")

From 164f8d8b1f6a3697ed18ba9b01c188ec688e72c9 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Tue, 13 Mar 2018 15:35:59 -0700
Subject: [PATCH 07/41] Renamed bucket to container to use more generic
 terminology.

---
 lib/galaxy/webapps/galaxy/api/cloud_storage.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
index 633a1b734b9..32e349e15bf 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud_storage.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
@@ -60,9 +60,9 @@ class CloudStorageController(BaseAPIController):
         if provider is None:
             missing_arguments.append("provider")
 
-        bucket = payload.get("bucket", None)
-        if bucket is None:
-            missing_arguments.append("bucket")
+        container = payload.get("container", None)
+        if container is None:
+            missing_arguments.append("container")
 
         obj = payload.get("object", None)
         if obj is None:

From 76aa47f69be61c77a7322a91824675999617f345 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Tue, 13 Mar 2018 16:16:41 -0700
Subject: [PATCH 08/41] Cloud storage api now sends download request to cloud
 storage manager.

---
 lib/galaxy/webapps/galaxy/api/cloud_storage.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
index 32e349e15bf..f1a728bb8f9 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud_storage.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
@@ -18,6 +18,7 @@ class CloudStorageController(BaseAPIController):
 
     def __init__(self, app):
         super(CloudStorageController, self).__init__(app)
+        self.cloud_storage_manager = cloud_storage.CloudStorageManager(app)
 
     @web.expose_api
     def index(self, trans, **kwargs):
@@ -73,7 +74,9 @@ class CloudStorageController(BaseAPIController):
             return {'status': 'error',
                     'message': "The following required arguments are missing in the payload: %s" % missing_arguments}
 
-
+        status, message = self.cloud_storage_manager.download(provider=provider, container=container, obj=obj)
+        trans.response.status = 200 if status == 'ok' else trans.response.status = 500
+        return {'status': status, 'message': message}
 
     @web.expose_api
     def upload(self, trans, payload, **kwargs):

From 2e1fbedc7f692fd23eea24e646c6b0b1caa851a2 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Tue, 13 Mar 2018 20:03:43 -0700
Subject: [PATCH 09/41] Fix a conditional assignment bug.

---
 lib/galaxy/webapps/galaxy/api/cloud_storage.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
index f1a728bb8f9..99afd812776 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud_storage.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
@@ -75,7 +75,7 @@ class CloudStorageController(BaseAPIController):
                     'message': "The following required arguments are missing in the payload: %s" % missing_arguments}
 
         status, message = self.cloud_storage_manager.download(provider=provider, container=container, obj=obj)
-        trans.response.status = 200 if status == 'ok' else trans.response.status = 500
+        trans.response.status = 200 if status == 'ok' else 500
         return {'status': status, 'message': message}
 
     @web.expose_api

From c725b33ec25d67be1ca2b51faeec5a07186dcf36 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Tue, 27 Mar 2018 12:03:14 -0700
Subject: [PATCH 10/41] Pass trans from cloud_storage api to
 cloud_storage_manager.

---
 lib/galaxy/webapps/galaxy/api/cloud_storage.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
index 99afd812776..52db5d4ef11 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud_storage.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
@@ -74,7 +74,7 @@ class CloudStorageController(BaseAPIController):
             return {'status': 'error',
                     'message': "The following required arguments are missing in the payload: %s" % missing_arguments}
 
-        status, message = self.cloud_storage_manager.download(provider=provider, container=container, obj=obj)
+        status, message = self.cloud_storage_manager.download(trans=trans, provider=provider, container=container, obj=obj)
         trans.response.status = 200 if status == 'ok' else 500
         return {'status': status, 'message': message}
 

From 6856b81db0f2f6913c7b1f98c4d54ccaaf022480 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Wed, 28 Mar 2018 10:55:13 -0700
Subject: [PATCH 11/41] In cloud_storage, receive credentials and pass them to
 the manager.

---
 lib/galaxy/webapps/galaxy/api/cloud_storage.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
index 52db5d4ef11..8e4637c807f 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud_storage.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
@@ -69,12 +69,20 @@ class CloudStorageController(BaseAPIController):
         if obj is None:
             missing_arguments.append("object")
 
+        credentials = payload.get("credentials", None)
+        if credentials is None:
+            missing_arguments.append("credentials")
+
         if len(missing_arguments) > 0:
             trans.response.status = 400
             return {'status': 'error',
                     'message': "The following required arguments are missing in the payload: %s" % missing_arguments}
 
-        status, message = self.cloud_storage_manager.download(trans=trans, provider=provider, container=container, obj=obj)
+        status, message = self.cloud_storage_manager.download(trans=trans,
+                                                              provider=provider,
+                                                              container=container,
+                                                              obj=obj,
+                                                              credentials=credentials)
         trans.response.status = 200 if status == 'ok' else 500
         return {'status': status, 'message': message}
 

From dc17e0285a6460c1ded477a976aaa316897caa13 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Wed, 28 Mar 2018 11:00:28 -0700
Subject: [PATCH 12/41] Download data from AWS S3 in cloud_storage manager.

---
 lib/galaxy/managers/cloud_storage.py | 41 ++++++++++++++++++++++++++--
 1 file changed, 38 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/managers/cloud_storage.py b/lib/galaxy/managers/cloud_storage.py
index 5e61c2a67f2..1f200ac82f2 100644
--- a/lib/galaxy/managers/cloud_storage.py
+++ b/lib/galaxy/managers/cloud_storage.py
@@ -2,19 +2,54 @@
 Manager and serializer for cloud-based storages.
 """
 
+import string
+import random
+import logging
+import os
 from galaxy.managers import sharable
 
-import logging
+try:
+    from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList
+except ImportError:
+    CloudProviderFactory = None
+    ProviderList = None
+
 log = logging.getLogger(__name__)
 
+NO_CLOUDBRIDGE_ERROR_MESSAGE = (
+    "Cloud ObjectStore is configured, but no CloudBridge dependency available."
+    "Please install CloudBridge or modify ObjectStore configuration."
+)
+
 
 class CloudStorageManager(sharable.SharableModelManager):
 
     def __init__(self, app, *args, **kwargs):
         super(CloudStorageManager, self).__init__(app, *args, **kwargs)
 
-    def download(self, provider, container, obj):
-        # TODO: implement the download logic.
+    def download(self, trans, provider, container, obj, credentials):
+        if CloudProviderFactory is None:
+            raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE)
+
+        aws_config = {'aws_access_key': credentials.get('access_key'),
+                      'aws_secret_key': credentials.get('secret_key')}
+        connection = CloudProviderFactory().create_provider(ProviderList.AWS, aws_config)
+
+        try:
+            container_obj = connection.object_store.get(container)
+            if container_obj is None:
+                return 400, "The container `{}` not found.".format(container)
+        except Exception:
+            msg = "Could not get the container `{}`".format(container)
+            log.exception(msg)
+            return 400, msg
+
+        key = container_obj.get(obj)
+        staging_file_name = os.path.abspath(os.path.join(
+            trans.app.config.new_file_path,
+            "cd_" + ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(11))))
+        staging_file = open(staging_file_name, "w+")
+        key.save_content(staging_file)
         pass
 
     def upload(self, dataset, provider, container, obj):

From f56e4abb800d09956294323025827ee149e08ae2 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Thu, 29 Mar 2018 15:00:14 -0700
Subject: [PATCH 13/41] Call upload tool to persist the dataset copied from
 cloud-based storage.

---
 lib/galaxy/managers/cloud_storage.py | 34 ++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/managers/cloud_storage.py b/lib/galaxy/managers/cloud_storage.py
index 1f200ac82f2..af167e93449 100644
--- a/lib/galaxy/managers/cloud_storage.py
+++ b/lib/galaxy/managers/cloud_storage.py
@@ -7,6 +7,8 @@ import random
 import logging
 import os
 from galaxy.managers import sharable
+from galaxy.util import Params
+from cgi import FieldStorage
 
 try:
     from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList
@@ -34,7 +36,6 @@ class CloudStorageManager(sharable.SharableModelManager):
         aws_config = {'aws_access_key': credentials.get('access_key'),
                       'aws_secret_key': credentials.get('secret_key')}
         connection = CloudProviderFactory().create_provider(ProviderList.AWS, aws_config)
-
         try:
             container_obj = connection.object_store.get(container)
             if container_obj is None:
@@ -50,7 +51,36 @@ class CloudStorageManager(sharable.SharableModelManager):
             "cd_" + ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(11))))
         staging_file = open(staging_file_name, "w+")
         key.save_content(staging_file)
-        pass
+
+        with open(staging_file_name, "r") as f:
+            content = f.read()
+            headers = {'content-disposition': 'form-data; name="{}"; filename="{}"'.format(
+                'files_0|file_data', staging_file), }
+
+            input_file = FieldStorage(headers=headers)
+            input_file.file = input_file.make_file()
+            input_file.file.write(content)
+
+            inputs = {
+                'dbkey': '?',
+                'file_type': 'auto',
+                'files_0|type': 'upload_dataset',
+                'files_0|space_to_tab': None,
+                'files_0|to_posix_lines': 'Yes',
+                'files_0|file_data': input_file,
+            }
+
+            params = Params(inputs, sanitize=False)
+            incoming = params.__dict__
+            upload_tool = trans.app.toolbox.get_tool('upload1')
+            output = upload_tool.handle_input(trans, incoming, history=None)
+
+            hids = {}
+            job_errors = output.get('job_errors', [])
+            if job_errors:
+                raise ValueError('Cannot upload a dataset.')
+            else:
+                hids.update({staging_file: output['out_data'][0][1].hid})
 
     def upload(self, dataset, provider, container, obj):
         # TODO: implement the upload logic.

From 8ceaae7a89457c6688375ddb324c0ecc2893c31c Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Fri, 30 Mar 2018 14:19:51 -0400
Subject: [PATCH 14/41] Linting fixes while reviewing cloud_storage

---
 lib/galaxy/managers/cloud_storage.py           | 7 ++++---
 lib/galaxy/webapps/galaxy/api/cloud_storage.py | 4 ++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/managers/cloud_storage.py b/lib/galaxy/managers/cloud_storage.py
index af167e93449..58c4003fc2e 100644
--- a/lib/galaxy/managers/cloud_storage.py
+++ b/lib/galaxy/managers/cloud_storage.py
@@ -2,13 +2,14 @@
 Manager and serializer for cloud-based storages.
 """
 
-import string
-import random
 import logging
 import os
+import random
+import string
+from cgi import FieldStorage
+
 from galaxy.managers import sharable
 from galaxy.util import Params
-from cgi import FieldStorage
 
 try:
     from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList
diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
index 8e4637c807f..dbb9c486ed3 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud_storage.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
@@ -5,8 +5,8 @@ API operations on Cloud-based storages, such as Amazon Simple Storage Service (S
 import logging
 
 from galaxy import web
-from galaxy.web.base.controller import BaseAPIController
 from galaxy.managers import cloud_storage
+from galaxy.web.base.controller import BaseAPIController
 
 log = logging.getLogger(__name__)
 
@@ -96,4 +96,4 @@ class CloudStorageController(BaseAPIController):
         :param kwargs:
         :return:
         """
-        pass
\ No newline at end of file
+        pass

From 5e4b63bf7635eacf53a808e0c487a476ad1bee1b Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Fri, 30 Mar 2018 13:50:09 -0700
Subject: [PATCH 15/41] Add history_id to list of required fields of cloud
 storage API payload.

---
 lib/galaxy/webapps/galaxy/api/cloud_storage.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
index 8e4637c807f..e180c0f575b 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud_storage.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
@@ -43,6 +43,7 @@ class CloudStorageController(BaseAPIController):
 
         :type  payload: dict
         :param payload: A dictionary structure containing the following keys:
+            *   history_id: the (encoded) id of history to which the object should be downloaded to.
 
 
         :param kwargs:
@@ -57,6 +58,10 @@ class CloudStorageController(BaseAPIController):
                                'but received data of type `%s`.' % str(type(payload))}
 
         missing_arguments = []
+        history_id = payload.get("history_id", None)
+        if history_id is None:
+            missing_arguments.append("history_id")
+
         provider = payload.get("provider", None)
         if provider is None:
             missing_arguments.append("provider")
@@ -79,6 +84,7 @@ class CloudStorageController(BaseAPIController):
                     'message': "The following required arguments are missing in the payload: %s" % missing_arguments}
 
         status, message = self.cloud_storage_manager.download(trans=trans,
+                                                              history_id=history_id,
                                                               provider=provider,
                                                               container=container,
                                                               obj=obj,

From ae527da165774d9ca2d7a1ac5b1649fef1ce6d49 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Fri, 30 Mar 2018 14:29:49 -0700
Subject: [PATCH 16/41] Decode history ID and validate it in cloud storage API.

---
 lib/galaxy/webapps/galaxy/api/cloud_storage.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
index e180c0f575b..122229f0d0e 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud_storage.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
@@ -4,6 +4,7 @@ API operations on Cloud-based storages, such as Amazon Simple Storage Service (S
 
 import logging
 
+from galaxy import exceptions
 from galaxy import web
 from galaxy.web.base.controller import BaseAPIController
 from galaxy.managers import cloud_storage
@@ -58,8 +59,8 @@ class CloudStorageController(BaseAPIController):
                                'but received data of type `%s`.' % str(type(payload))}
 
         missing_arguments = []
-        history_id = payload.get("history_id", None)
-        if history_id is None:
+        encoded_history_id = payload.get("history_id", None)
+        if encoded_history_id is None:
             missing_arguments.append("history_id")
 
         provider = payload.get("provider", None)
@@ -83,6 +84,12 @@ class CloudStorageController(BaseAPIController):
             return {'status': 'error',
                     'message': "The following required arguments are missing in the payload: %s" % missing_arguments}
 
+        try:
+            history_id = self.decode_id(encoded_history_id)
+        except exceptions.MalformedId as e:
+            trans.response.status = 400
+            return {'status': 'error', 'message': 'Invalid history ID. {}'.format(e)}
+
         status, message = self.cloud_storage_manager.download(trans=trans,
                                                               history_id=history_id,
                                                               provider=provider,

From ab8ca7055bf6d2599c2f3888d8619da9d6bb57dd Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Fri, 30 Mar 2018 15:27:12 -0700
Subject: [PATCH 17/41] Specify a history to the upload tool in cloud storage
 manager.

---
 lib/galaxy/managers/cloud_storage.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/managers/cloud_storage.py b/lib/galaxy/managers/cloud_storage.py
index af167e93449..05f45ab16c7 100644
--- a/lib/galaxy/managers/cloud_storage.py
+++ b/lib/galaxy/managers/cloud_storage.py
@@ -29,7 +29,7 @@ class CloudStorageManager(sharable.SharableModelManager):
     def __init__(self, app, *args, **kwargs):
         super(CloudStorageManager, self).__init__(app, *args, **kwargs)
 
-    def download(self, trans, provider, container, obj, credentials):
+    def download(self, trans, history_id, provider, container, obj, credentials):
         if CloudProviderFactory is None:
             raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE)
 
@@ -73,7 +73,8 @@ class CloudStorageManager(sharable.SharableModelManager):
             params = Params(inputs, sanitize=False)
             incoming = params.__dict__
             upload_tool = trans.app.toolbox.get_tool('upload1')
-            output = upload_tool.handle_input(trans, incoming, history=None)
+            history = trans.sa_session.query(trans.app.model.History).get(history_id)
+            output = upload_tool.handle_input(trans, incoming, history=history)
 
             hids = {}
             job_errors = output.get('job_errors', [])

From 651d4c7f2ae5861d896ba4ab44518fcd6b33a582 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Mon, 2 Apr 2018 11:22:36 -0700
Subject: [PATCH 18/41] Transaction response is returned in standard http
 response codes.

---
 lib/galaxy/managers/cloud_storage.py           |  5 +++--
 lib/galaxy/webapps/galaxy/api/cloud_storage.py | 14 +++++++-------
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/managers/cloud_storage.py b/lib/galaxy/managers/cloud_storage.py
index 7a49996d79f..d0e9d08ca7a 100644
--- a/lib/galaxy/managers/cloud_storage.py
+++ b/lib/galaxy/managers/cloud_storage.py
@@ -40,11 +40,11 @@ class CloudStorageManager(sharable.SharableModelManager):
         try:
             container_obj = connection.object_store.get(container)
             if container_obj is None:
-                return 400, "The container `{}` not found.".format(container)
+                return "400", "The container `{}` not found.".format(container)
         except Exception:
             msg = "Could not get the container `{}`".format(container)
             log.exception(msg)
-            return 400, msg
+            return "400", msg
 
         key = container_obj.get(obj)
         staging_file_name = os.path.abspath(os.path.join(
@@ -83,6 +83,7 @@ class CloudStorageManager(sharable.SharableModelManager):
                 raise ValueError('Cannot upload a dataset.')
             else:
                 hids.update({staging_file: output['out_data'][0][1].hid})
+        return "200", 'The dataset is downloaded successfully.'
 
     def upload(self, dataset, provider, container, obj):
         # TODO: implement the upload logic.
diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
index b5fc3601bc8..f7d1197fd9a 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud_storage.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud_storage.py
@@ -53,8 +53,8 @@ class CloudStorageController(BaseAPIController):
         :return: True/False if the given object is successfully downloaded from the cloud-based storage.
         """
         if not isinstance(payload, dict):
-            trans.response.status = 400
-            return {'status': 'error',
+            trans.response.status = "400"
+            return {'status': "400",
                     'message': 'Invalid payload data type. The payload is expected to be a dictionary, '
                                'but received data of type `%s`.' % str(type(payload))}
 
@@ -80,15 +80,15 @@ class CloudStorageController(BaseAPIController):
             missing_arguments.append("credentials")
 
         if len(missing_arguments) > 0:
-            trans.response.status = 400
-            return {'status': 'error',
+            trans.response.status = "400"
+            return {'status': "400",
                     'message': "The following required arguments are missing in the payload: %s" % missing_arguments}
 
         try:
             history_id = self.decode_id(encoded_history_id)
         except exceptions.MalformedId as e:
-            trans.response.status = 400
-            return {'status': 'error', 'message': 'Invalid history ID. {}'.format(e)}
+            trans.response.status = "400"
+            return {'status': "400", 'message': 'Invalid history ID. {}'.format(e)}
 
         status, message = self.cloud_storage_manager.download(trans=trans,
                                                               history_id=history_id,
@@ -96,7 +96,7 @@ class CloudStorageController(BaseAPIController):
                                                               container=container,
                                                               obj=obj,
                                                               credentials=credentials)
-        trans.response.status = 200 if status == 'ok' else 500
+        trans.response.status = status
         return {'status': status, 'message': message}
 
     @web.expose_api

From 6e0fea91b3e05f0203b25f46a87b375ad93897a8 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Mon, 2 Apr 2018 12:04:13 -0700
Subject: [PATCH 19/41] Name dataset with the same name as of the object being
 downloaded.

---
 lib/galaxy/managers/cloud_storage.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/galaxy/managers/cloud_storage.py b/lib/galaxy/managers/cloud_storage.py
index d0e9d08ca7a..4c5d6234a9a 100644
--- a/lib/galaxy/managers/cloud_storage.py
+++ b/lib/galaxy/managers/cloud_storage.py
@@ -55,8 +55,7 @@ class CloudStorageManager(sharable.SharableModelManager):
 
         with open(staging_file_name, "r") as f:
             content = f.read()
-            headers = {'content-disposition': 'form-data; name="{}"; filename="{}"'.format(
-                'files_0|file_data', staging_file), }
+            headers = {'content-disposition': 'form-data; name="{}"; filename="{}"'.format('files_0|file_data', obj), }
 
             input_file = FieldStorage(headers=headers)
             input_file.file = input_file.make_file()

From 10644a86a95910d9106cd00fddd9632fd1a0d1ee Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Mon, 2 Apr 2018 12:13:14 -0700
Subject: [PATCH 20/41] Refactor "CloudStorage" & "cloud_storage" to "Cloud" &
 "cloud" resp.

---
 .../managers/{cloud_storage.py => cloud.py}   |  4 +--
 .../galaxy/api/{cloud_storage.py => cloud.py} | 26 +++++++++----------
 lib/galaxy/webapps/galaxy/buildapp.py         |  8 +++---
 3 files changed, 19 insertions(+), 19 deletions(-)
 rename lib/galaxy/managers/{cloud_storage.py => cloud.py} (96%)
 rename lib/galaxy/webapps/galaxy/api/{cloud_storage.py => cloud.py} (80%)

diff --git a/lib/galaxy/managers/cloud_storage.py b/lib/galaxy/managers/cloud.py
similarity index 96%
rename from lib/galaxy/managers/cloud_storage.py
rename to lib/galaxy/managers/cloud.py
index 4c5d6234a9a..57fed4b30d5 100644
--- a/lib/galaxy/managers/cloud_storage.py
+++ b/lib/galaxy/managers/cloud.py
@@ -25,10 +25,10 @@ NO_CLOUDBRIDGE_ERROR_MESSAGE = (
 )
 
 
-class CloudStorageManager(sharable.SharableModelManager):
+class CloudManager(sharable.SharableModelManager):
 
     def __init__(self, app, *args, **kwargs):
-        super(CloudStorageManager, self).__init__(app, *args, **kwargs)
+        super(CloudManager, self).__init__(app, *args, **kwargs)
 
     def download(self, trans, history_id, provider, container, obj, credentials):
         if CloudProviderFactory is None:
diff --git a/lib/galaxy/webapps/galaxy/api/cloud_storage.py b/lib/galaxy/webapps/galaxy/api/cloud.py
similarity index 80%
rename from lib/galaxy/webapps/galaxy/api/cloud_storage.py
rename to lib/galaxy/webapps/galaxy/api/cloud.py
index f7d1197fd9a..2e5a756af76 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud_storage.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud.py
@@ -6,25 +6,25 @@ import logging
 
 from galaxy import exceptions
 from galaxy import web
-from galaxy.managers import cloud_storage
+from galaxy.managers import cloud
 from galaxy.web.base.controller import BaseAPIController
 
 log = logging.getLogger(__name__)
 
 
-class CloudStorageController(BaseAPIController):
+class CloudController(BaseAPIController):
     """
     RESTfull controller for interaction with Amazon S3.
     """
 
     def __init__(self, app):
-        super(CloudStorageController, self).__init__(app)
-        self.cloud_storage_manager = cloud_storage.CloudStorageManager(app)
+        super(CloudController, self).__init__(app)
+        self.cloud_manager = cloud.CloudManager(app)
 
     @web.expose_api
     def index(self, trans, **kwargs):
         """
-        * GET /api/cloud_storage
+        * GET /api/cloud
             Lists cloud-based containers (e.g., S3 bucket, Azure blob) user has defined.
         :param trans:
         :param kwargs:
@@ -37,7 +37,7 @@ class CloudStorageController(BaseAPIController):
     @web.expose_api
     def download(self, trans, payload, **kwargs):
         """
-        * POST /api/cloud_storage/download
+        * POST /api/cloud/download
             Downloads a given object from a given cloud-based container.
         :type  trans: galaxy.web.framework.webapp.GalaxyWebTransaction
         :param trans: Galaxy web transaction
@@ -90,19 +90,19 @@ class CloudStorageController(BaseAPIController):
             trans.response.status = "400"
             return {'status': "400", 'message': 'Invalid history ID. {}'.format(e)}
 
-        status, message = self.cloud_storage_manager.download(trans=trans,
-                                                              history_id=history_id,
-                                                              provider=provider,
-                                                              container=container,
-                                                              obj=obj,
-                                                              credentials=credentials)
+        status, message = self.cloud_manager.download(trans=trans,
+                                                      history_id=history_id,
+                                                      provider=provider,
+                                                      container=container,
+                                                      obj=obj,
+                                                      credentials=credentials)
         trans.response.status = status
         return {'status': status, 'message': message}
 
     @web.expose_api
     def upload(self, trans, payload, **kwargs):
         """
-        * POST /api/cloud_storage/upload
+        * POST /api/cloud/upload
             Uploads a given dataset to a given cloud-based container.
         :param trans:
         :param payload:
diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py
index 1be9eaf8914..e6aaa085fac 100644
--- a/lib/galaxy/webapps/galaxy/buildapp.py
+++ b/lib/galaxy/webapps/galaxy/buildapp.py
@@ -231,10 +231,10 @@ def populate_api_routes(webapp, app):
     webapp.mapper.connect('/api/histories/published', action='published', controller="histories", conditions=dict(method=["GET"]))
     webapp.mapper.connect('/api/histories/shared_with_me', action='shared_with_me', controller="histories")
 
-    webapp.mapper.resource('cloud_storage', 'cloud_storage', path_prefix='/api')
-    webapp.mapper.connect('cloud_storage_download',
-                          '/api/cloud_storage/download',
-                          controller='cloud_storage',
+    webapp.mapper.resource('cloud', 'cloud', path_prefix='/api')
+    webapp.mapper.connect('cloud_download',
+                          '/api/cloud/download',
+                          controller='cloud',
                           action='download',
                           conditions=dict(method=["POST"]))
 

From fb9656a1a0e7b00c1d9087988ed2efbcf1255595 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Mon, 2 Apr 2018 13:34:00 -0700
Subject: [PATCH 21/41] Delete from-cloud-downloaded staged file after upload
 job is created.

---
 lib/galaxy/managers/cloud.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index 57fed4b30d5..b05c87fd1fb 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -82,6 +82,7 @@ class CloudManager(sharable.SharableModelManager):
                 raise ValueError('Cannot upload a dataset.')
             else:
                 hids.update({staging_file: output['out_data'][0][1].hid})
+        os.remove(staging_file_name)
         return "200", 'The dataset is downloaded successfully.'
 
     def upload(self, dataset, provider, container, obj):

From 547443d396f2c04c05396f3f37c8bc13bd7ed8fd Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Mon, 2 Apr 2018 14:31:37 -0700
Subject: [PATCH 22/41] Verify credentials when initializing a cloud provider.

---
 lib/galaxy/managers/cloud.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index b05c87fd1fb..d1ea775e873 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -13,6 +13,7 @@ from galaxy.util import Params
 
 try:
     from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList
+    from cloudbridge.cloud.interfaces.exceptions import *
 except ImportError:
     CloudProviderFactory = None
     ProviderList = None
@@ -30,13 +31,28 @@ class CloudManager(sharable.SharableModelManager):
     def __init__(self, app, *args, **kwargs):
         super(CloudManager, self).__init__(app, *args, **kwargs)
 
+    def _configure_provider(self, provider, credentials):
+        if provider == 'aws':
+            aws_config = {'aws_access_key': credentials.get('access_key'),
+                          'aws_secret_key': credentials.get('secret_key')}
+            connection = CloudProviderFactory().create_provider(ProviderList.AWS, aws_config)
+        else:
+            return "400", "Unrecognized provider '{}'.".format(provider), None
+
+        try:
+            if connection.authenticate():
+                return "200", "", connection
+        except ProviderConnectionException as e:
+            return "400", "Could not authenticate to the '{}' provider. {}".format(provider, e), None
+
+
     def download(self, trans, history_id, provider, container, obj, credentials):
         if CloudProviderFactory is None:
             raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE)
 
-        aws_config = {'aws_access_key': credentials.get('access_key'),
-                      'aws_secret_key': credentials.get('secret_key')}
-        connection = CloudProviderFactory().create_provider(ProviderList.AWS, aws_config)
+        status, msg, connection = self._configure_provider(provider, credentials)
+        if status != "200":
+            return status, msg
         try:
             container_obj = connection.object_store.get(container)
             if container_obj is None:

From d9e5806070662e07dc66af4816b02ed77e1d56bf Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Mon, 2 Apr 2018 14:45:35 -0700
Subject: [PATCH 23/41] Add support for Azure to cloud storage manager.

---
 lib/galaxy/managers/cloud.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index d1ea775e873..668178d4d6f 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -32,12 +32,20 @@ class CloudManager(sharable.SharableModelManager):
         super(CloudManager, self).__init__(app, *args, **kwargs)
 
     def _configure_provider(self, provider, credentials):
+        supported_providers = "{aws, azure}"
         if provider == 'aws':
-            aws_config = {'aws_access_key': credentials.get('access_key'),
-                          'aws_secret_key': credentials.get('secret_key')}
-            connection = CloudProviderFactory().create_provider(ProviderList.AWS, aws_config)
+            config = {'aws_access_key': credentials.get('access_key'),
+                      'aws_secret_key': credentials.get('secret_key')}
+            connection = CloudProviderFactory().create_provider(ProviderList.AWS, config)
+        elif provider == "azure":
+            config = {'azure_subscription_id': credentials.get('subscription_id'),
+                      'azure_client_id': credentials.get('client_id'),
+                      'azure_secret': credentials.get('secret'),
+                      'azure_tenant': credentials.get('tenant')}
+            connection = CloudProviderFactory().create_provider(ProviderList.AZURE, config)
         else:
-            return "400", "Unrecognized provider '{}'.".format(provider), None
+            return "400", "Unrecognized provider '{}'; the following are the supported providers: {}.".format(
+                provider, supported_providers), None
 
         try:
             if connection.authenticate():

From eb0df62df78f1b5625961137c20ffde0e11437c3 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Mon, 2 Apr 2018 15:53:22 -0700
Subject: [PATCH 24/41] Check for provider-specific missing (None) required
 credentials.

---
 lib/galaxy/managers/cloud.py | 39 ++++++++++++++++++++++++++++++------
 1 file changed, 33 insertions(+), 6 deletions(-)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index 668178d4d6f..5a44ab3eaed 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -33,15 +33,42 @@ class CloudManager(sharable.SharableModelManager):
 
     def _configure_provider(self, provider, credentials):
         supported_providers = "{aws, azure}"
+        missing_credentials = []
         if provider == 'aws':
-            config = {'aws_access_key': credentials.get('access_key'),
-                      'aws_secret_key': credentials.get('secret_key')}
+            access = credentials.get('access_key', None)
+            if access is None:
+                missing_credentials.append('access_key')
+            secret = credentials.get('secret_key', None)
+            if secret is None:
+                missing_credentials.append('secret_key')
+            if len(missing_credentials) > 0:
+                return "400", "The following required key(s) are missing from the provided credentials object: " \
+                              "{}".format(missing_credentials), None
+
+            config = {'aws_access_key': access,
+                      'aws_secret_key': secret}
             connection = CloudProviderFactory().create_provider(ProviderList.AWS, config)
         elif provider == "azure":
-            config = {'azure_subscription_id': credentials.get('subscription_id'),
-                      'azure_client_id': credentials.get('client_id'),
-                      'azure_secret': credentials.get('secret'),
-                      'azure_tenant': credentials.get('tenant')}
+            subscription = credentials.get('subscription_id', None)
+            if subscription is None:
+                missing_credentials.append('subscription_id')
+            client = credentials.get('client_id', None)
+            if client is None:
+                missing_credentials.append('client_id')
+            secret = credentials.get('secret', None)
+            if secret is None:
+                missing_credentials.append('secret')
+            tenant = credentials.get('tenant', None)
+            if tenant is None:
+                missing_credentials.append('tenant')
+            if len(missing_credentials) > 0:
+                return "400", "The following required key(s) are missing from the provided credentials object: " \
+                              "{}".format(missing_credentials), None
+
+            config = {'azure_subscription_id': subscription,
+                      'azure_client_id': client,
+                      'azure_secret': secret,
+                      'azure_tenant': tenant}
             connection = CloudProviderFactory().create_provider(ProviderList.AZURE, config)
         else:
             return "400", "Unrecognized provider '{}'; the following are the supported providers: {}.".format(

From 23e1209851b16e01b8c9e1acbc36e8070310ab38 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Mon, 2 Apr 2018 15:59:38 -0700
Subject: [PATCH 25/41] Store supported provides name in a constant.

---
 lib/galaxy/managers/cloud.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index 5a44ab3eaed..47e8addbd71 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -25,6 +25,8 @@ NO_CLOUDBRIDGE_ERROR_MESSAGE = (
     "Please install CloudBridge or modify ObjectStore configuration."
 )
 
+SUPPORTED_PROVIDERS = "{aws, azure}"
+
 
 class CloudManager(sharable.SharableModelManager):
 
@@ -32,7 +34,6 @@ class CloudManager(sharable.SharableModelManager):
         super(CloudManager, self).__init__(app, *args, **kwargs)
 
     def _configure_provider(self, provider, credentials):
-        supported_providers = "{aws, azure}"
         missing_credentials = []
         if provider == 'aws':
             access = credentials.get('access_key', None)
@@ -72,7 +73,7 @@ class CloudManager(sharable.SharableModelManager):
             connection = CloudProviderFactory().create_provider(ProviderList.AZURE, config)
         else:
             return "400", "Unrecognized provider '{}'; the following are the supported providers: {}.".format(
-                provider, supported_providers), None
+                provider, SUPPORTED_PROVIDERS), None
 
         try:
             if connection.authenticate():

From 177fe583c5c8274850ab69138fb5e664f0bcef2f Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Mon, 2 Apr 2018 16:07:58 -0700
Subject: [PATCH 26/41] Throw NotImplementedError for cloud api & manager
 upload functionality.

---
 lib/galaxy/managers/cloud.py           | 3 +--
 lib/galaxy/webapps/galaxy/api/cloud.py | 3 ++-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index 47e8addbd71..980ebe2a5a5 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -138,5 +138,4 @@ class CloudManager(sharable.SharableModelManager):
         return "200", 'The dataset is downloaded successfully.'
 
     def upload(self, dataset, provider, container, obj):
-        # TODO: implement the upload logic.
-        pass
+        raise NotImplementedError
diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py
index 2e5a756af76..b0e2bd29eee 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud.py
@@ -109,4 +109,5 @@ class CloudController(BaseAPIController):
         :param kwargs:
         :return:
         """
-        pass
+        trans.response.status = 501
+        return 'Not Implemented'

From 9daa216b830c620ed6d5f8231ed0852a337882b2 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Tue, 3 Apr 2018 11:22:37 -0700
Subject: [PATCH 27/41] Remove some blank lines and fixed an import.

---
 lib/galaxy/managers/cloud.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index 980ebe2a5a5..304830396e1 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -13,7 +13,7 @@ from galaxy.util import Params
 
 try:
     from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList
-    from cloudbridge.cloud.interfaces.exceptions import *
+    from cloudbridge.cloud.interfaces.exceptions import ProviderConnectionException
 except ImportError:
     CloudProviderFactory = None
     ProviderList = None
@@ -81,7 +81,6 @@ class CloudManager(sharable.SharableModelManager):
         except ProviderConnectionException as e:
             return "400", "Could not authenticate to the '{}' provider. {}".format(provider, e), None
 
-
     def download(self, trans, history_id, provider, container, obj, credentials):
         if CloudProviderFactory is None:
             raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE)

From 70b669992b10682d45c519ef885667ddf52f381c Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Tue, 3 Apr 2018 11:35:45 -0700
Subject: [PATCH 28/41] Update cloud Download API documentation.

---
 lib/galaxy/webapps/galaxy/api/cloud.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py
index b0e2bd29eee..b08ab4d2d59 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud.py
@@ -44,13 +44,19 @@ class CloudController(BaseAPIController):
 
         :type  payload: dict
         :param payload: A dictionary structure containing the following keys:
-            *   history_id: the (encoded) id of history to which the object should be downloaded to.
-
+            *   history_id:    the (encoded) id of history to which the object should be downloaded to.
+            *   provider:      the name of cloud-based resource provided (e.g., 'aws' or 'azure').
+            *   container:     is the name of container from which data should be downloaded (e.g., a bucket name on AWS S3).
+            *   object:        is the name of an object to be downloaded.
+            *   credentials:   is a dictionary containing all the credentials required to authenticated to the
+            specified provider (e.g., {"secret_key": YOUR_AWS_SECRET_TOKEN, "access_key": YOUR_AWS_ACCESS_TOKEN}).
 
         :param kwargs:
 
-        :rtype:  boolean
-        :return: True/False if the given object is successfully downloaded from the cloud-based storage.
+        :rtype:  dictionary
+        :return: a dictionary with the following keys:
+            *   status:     HTTP status code (e.g., "200" if the provided object is successfully downloaded to Galaxy).
+            *   message:    a description complementary to the status code.
         """
         if not isinstance(payload, dict):
             trans.response.status = "400"

From 9960e3bde1eb13506b4f01349fc95d12880ad705 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Fri, 20 Apr 2018 14:17:56 -0700
Subject: [PATCH 29/41] Leverage _future_expose_api for the expose_api
 decorator in cloud api.

---
 lib/galaxy/webapps/galaxy/api/cloud.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py
index b08ab4d2d59..e970820b9a2 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud.py
@@ -5,8 +5,8 @@ API operations on Cloud-based storages, such as Amazon Simple Storage Service (S
 import logging
 
 from galaxy import exceptions
-from galaxy import web
 from galaxy.managers import cloud
+from galaxy.web import  _future_expose_api as expose_api
 from galaxy.web.base.controller import BaseAPIController
 
 log = logging.getLogger(__name__)
@@ -21,7 +21,7 @@ class CloudController(BaseAPIController):
         super(CloudController, self).__init__(app)
         self.cloud_manager = cloud.CloudManager(app)
 
-    @web.expose_api
+    @expose_api
     def index(self, trans, **kwargs):
         """
         * GET /api/cloud
@@ -34,7 +34,7 @@ class CloudController(BaseAPIController):
         trans.response.status = 501
         return 'Not Implemented'
 
-    @web.expose_api
+    @expose_api
     def download(self, trans, payload, **kwargs):
         """
         * POST /api/cloud/download
@@ -105,7 +105,7 @@ class CloudController(BaseAPIController):
         trans.response.status = status
         return {'status': status, 'message': message}
 
-    @web.expose_api
+    @expose_api
     def upload(self, trans, payload, **kwargs):
         """
         * POST /api/cloud/upload

From c83194d9bddae4989296f6424f76d59abf3eab54 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Mon, 23 Apr 2018 10:49:52 -0700
Subject: [PATCH 30/41] Replace explicit dict errors in cloud API with
 Galaxy.exceptions.

---
 lib/galaxy/webapps/galaxy/api/cloud.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py
index e970820b9a2..61c0eabf3ef 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud.py
@@ -5,6 +5,7 @@ API operations on Cloud-based storages, such as Amazon Simple Storage Service (S
 import logging
 
 from galaxy import exceptions
+from galaxy.exceptions import ActionInputError
 from galaxy.managers import cloud
 from galaxy.web import  _future_expose_api as expose_api
 from galaxy.web.base.controller import BaseAPIController
@@ -59,10 +60,8 @@ class CloudController(BaseAPIController):
             *   message:    a description complementary to the status code.
         """
         if not isinstance(payload, dict):
-            trans.response.status = "400"
-            return {'status': "400",
-                    'message': 'Invalid payload data type. The payload is expected to be a dictionary, '
-                               'but received data of type `%s`.' % str(type(payload))}
+            raise ActionInputError('Invalid payload data type. The payload is expected to be a dictionary, '
+                                   'but received data of type `{}`.'.format(str(type(payload))))
 
         missing_arguments = []
         encoded_history_id = payload.get("history_id", None)
@@ -86,15 +85,12 @@ class CloudController(BaseAPIController):
             missing_arguments.append("credentials")
 
         if len(missing_arguments) > 0:
-            trans.response.status = "400"
-            return {'status': "400",
-                    'message': "The following required arguments are missing in the payload: %s" % missing_arguments}
+            raise ActionInputError("The following required arguments are missing in the payload: {}".format(missing_arguments))
 
         try:
             history_id = self.decode_id(encoded_history_id)
         except exceptions.MalformedId as e:
-            trans.response.status = "400"
-            return {'status': "400", 'message': 'Invalid history ID. {}'.format(e)}
+            raise ActionInputError('Invalid history ID. {}'.format(e))
 
         status, message = self.cloud_manager.download(trans=trans,
                                                       history_id=history_id,

From 61a1e7eb4c80fdb95c338695a38efa4e0fba0f5e Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Mon, 23 Apr 2018 12:15:51 -0700
Subject: [PATCH 31/41] Return summary view of datasets created for data
 downloaded from cloud.

---
 lib/galaxy/managers/cloud.py           |  5 ++++-
 lib/galaxy/webapps/galaxy/api/cloud.py | 26 ++++++++++++++++----------
 2 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index 304830396e1..c5b5cf99c07 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -104,6 +104,7 @@ class CloudManager(sharable.SharableModelManager):
         staging_file = open(staging_file_name, "w+")
         key.save_content(staging_file)
 
+        datasets = []
         with open(staging_file_name, "r") as f:
             content = f.read()
             headers = {'content-disposition': 'form-data; name="{}"; filename="{}"'.format('files_0|file_data', obj), }
@@ -133,8 +134,10 @@ class CloudManager(sharable.SharableModelManager):
                 raise ValueError('Cannot upload a dataset.')
             else:
                 hids.update({staging_file: output['out_data'][0][1].hid})
+                for d in output['out_data']:
+                    datasets.append(d[1].dataset)
         os.remove(staging_file_name)
-        return "200", 'The dataset is downloaded successfully.'
+        return datasets
 
     def upload(self, dataset, provider, container, obj):
         raise NotImplementedError
diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py
index 61c0eabf3ef..5647118482b 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud.py
@@ -6,8 +6,11 @@ import logging
 
 from galaxy import exceptions
 from galaxy.exceptions import ActionInputError
-from galaxy.managers import cloud
-from galaxy.web import  _future_expose_api as expose_api
+from galaxy.managers import (
+    cloud,
+    datasets
+)
+from galaxy.web import _future_expose_api as expose_api
 from galaxy.web.base.controller import BaseAPIController
 
 log = logging.getLogger(__name__)
@@ -21,6 +24,7 @@ class CloudController(BaseAPIController):
     def __init__(self, app):
         super(CloudController, self).__init__(app)
         self.cloud_manager = cloud.CloudManager(app)
+        self.datasets_serializer = datasets.DatasetSerializer(app)
 
     @expose_api
     def index(self, trans, **kwargs):
@@ -92,14 +96,16 @@ class CloudController(BaseAPIController):
         except exceptions.MalformedId as e:
             raise ActionInputError('Invalid history ID. {}'.format(e))
 
-        status, message = self.cloud_manager.download(trans=trans,
-                                                      history_id=history_id,
-                                                      provider=provider,
-                                                      container=container,
-                                                      obj=obj,
-                                                      credentials=credentials)
-        trans.response.status = status
-        return {'status': status, 'message': message}
+        datasets = self.cloud_manager.download(trans=trans,
+                                               history_id=history_id,
+                                               provider=provider,
+                                               container=container,
+                                               obj=obj,
+                                               credentials=credentials)
+        rtv = []
+        for dataset in datasets:
+            rtv.append(self.datasets_serializer.serialize_to_view(dataset, view='summary'))
+        return rtv
 
     @expose_api
     def upload(self, trans, payload, **kwargs):

From 907ef7b3ea67530bc41b601f975eda3d2dad0b02 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Mon, 23 Apr 2018 15:04:11 -0700
Subject: [PATCH 32/41] Replace returning http err code in cloud manager with
 Galaxy Exceptions.

---
 lib/galaxy/managers/cloud.py | 37 ++++++++++++++++++++----------------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index c5b5cf99c07..648b6b695f5 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -8,6 +8,13 @@ import random
 import string
 from cgi import FieldStorage
 
+from galaxy.exceptions import (
+    AuthenticationFailed,
+    ItemAccessibilityException,
+    ObjectNotFound,
+    RequestParameterInvalidException,
+    RequestParameterMissingException
+)
 from galaxy.managers import sharable
 from galaxy.util import Params
 
@@ -43,8 +50,8 @@ class CloudManager(sharable.SharableModelManager):
             if secret is None:
                 missing_credentials.append('secret_key')
             if len(missing_credentials) > 0:
-                return "400", "The following required key(s) are missing from the provided credentials object: " \
-                              "{}".format(missing_credentials), None
+                raise RequestParameterMissingException("The following required key(s) are missing from the provided "
+                                                       "credentials object: {}".format(missing_credentials))
 
             config = {'aws_access_key': access,
                       'aws_secret_key': secret}
@@ -63,8 +70,8 @@ class CloudManager(sharable.SharableModelManager):
             if tenant is None:
                 missing_credentials.append('tenant')
             if len(missing_credentials) > 0:
-                return "400", "The following required key(s) are missing from the provided credentials object: " \
-                              "{}".format(missing_credentials), None
+                raise RequestParameterMissingException("The following required key(s) are missing from the provided "
+                                                       "credentials object: {}".format(missing_credentials))
 
             config = {'azure_subscription_id': subscription,
                       'azure_client_id': client,
@@ -72,32 +79,30 @@ class CloudManager(sharable.SharableModelManager):
                       'azure_tenant': tenant}
             connection = CloudProviderFactory().create_provider(ProviderList.AZURE, config)
         else:
-            return "400", "Unrecognized provider '{}'; the following are the supported providers: {}.".format(
-                provider, SUPPORTED_PROVIDERS), None
+            raise RequestParameterInvalidException("Unrecognized provider '{}'; the following are the supported "
+                                                   "providers: {}.".format(provider, SUPPORTED_PROVIDERS))
 
         try:
             if connection.authenticate():
-                return "200", "", connection
+                return connection
         except ProviderConnectionException as e:
-            return "400", "Could not authenticate to the '{}' provider. {}".format(provider, e), None
+            raise AuthenticationFailed("Could not authenticate to the '{}' provider. {}".format(provider, e))
 
     def download(self, trans, history_id, provider, container, obj, credentials):
         if CloudProviderFactory is None:
             raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE)
 
-        status, msg, connection = self._configure_provider(provider, credentials)
-        if status != "200":
-            return status, msg
+        connection = self._configure_provider(provider, credentials)
         try:
             container_obj = connection.object_store.get(container)
             if container_obj is None:
-                return "400", "The container `{}` not found.".format(container)
-        except Exception:
-            msg = "Could not get the container `{}`".format(container)
-            log.exception(msg)
-            return "400", msg
+                raise RequestParameterInvalidException("The container `{}` not found.".format(container))
+        except Exception as e:
+            raise ItemAccessibilityException("Could not get the container `{}`: {}".format(container, str(e)))
 
         key = container_obj.get(obj)
+        if key is None:
+            raise ObjectNotFound("Could not get the object `{}`.".format(obj))
         staging_file_name = os.path.abspath(os.path.join(
             trans.app.config.new_file_path,
             "cd_" + ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(11))))

From 55646658165aefd6cc48dcb216b8de4d3ac29963 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Tue, 24 Apr 2018 09:02:47 -0700
Subject: [PATCH 33/41] close staging file used in cloud api.

---
 lib/galaxy/managers/cloud.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index 648b6b695f5..fd344e00e72 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -141,6 +141,7 @@ class CloudManager(sharable.SharableModelManager):
                 hids.update({staging_file: output['out_data'][0][1].hid})
                 for d in output['out_data']:
                     datasets.append(d[1].dataset)
+        staging_file.close()
         os.remove(staging_file_name)
         return datasets
 

From 24a1a6bc906cdcbfdab74808c5ef7e116005cb2b Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Tue, 24 Apr 2018 10:14:51 -0700
Subject: [PATCH 34/41] Describe functions in cloud manager.

---
 lib/galaxy/managers/cloud.py | 42 ++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index fd344e00e72..0e3c82c456a 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -41,6 +41,21 @@ class CloudManager(sharable.SharableModelManager):
         super(CloudManager, self).__init__(app, *args, **kwargs)
 
     def _configure_provider(self, provider, credentials):
+        """
+        Given a provider name and required credentials, it configures and returns a cloudbridge
+        connection to the provider.
+
+        :type  provider: string
+        :param provider: the name of cloud-based resource provided. A list of supported providers is given in
+        `SUPPORTED_PROVIDERS` variable.
+
+        :type  credentials: dict
+        :param credentials: a dictionary containing all the credentials required to authenticated to the
+        specified provider.
+
+        :rtype: provider specific, e.g., `cloudbridge.cloud.providers.aws.provider.AWSCloudProvider` for AWS.
+        :return: a cloudbridge connection to the specified provider.
+        """
         missing_credentials = []
         if provider == 'aws':
             access = credentials.get('access_key', None)
@@ -89,6 +104,33 @@ class CloudManager(sharable.SharableModelManager):
             raise AuthenticationFailed("Could not authenticate to the '{}' provider. {}".format(provider, e))
 
     def download(self, trans, history_id, provider, container, obj, credentials):
+        """
+        Implements the logic of downloading a file from a cloud-based storage (e.g., Amazon S3)
+        and persisting it as a Galaxy dataset.
+
+        :type  trans: galaxy.web.framework.webapp.GalaxyWebTransaction
+        :param trans: Galaxy web transaction
+
+        :type  history_id: string
+        :param history_id: the (encoded) id of history to which the object should be downloaded to.
+
+        :type  provider: string
+        :param provider: the name of cloud-based resource provided. A list of supported providers is given in
+        `SUPPORTED_PROVIDERS` variable.
+
+        :type  container: string
+        :param container: is the name of container from which data should be downloaded (e.g., a bucket name on AWS S3).
+
+        :type  obj: string
+        :param obj: is the name of an object to be downloaded.
+
+        :type  credentials: dict
+        :param credentials: a dictionary containing all the credentials required to authenticated to the
+        specified provider (e.g., {"secret_key": YOUR_AWS_SECRET_TOKEN, "access_key": YOUR_AWS_ACCESS_TOKEN}).
+
+        :rtype:  list of galaxy.model.Dataset
+        :return: a list of datasets created for the downloaded files.
+        """
         if CloudProviderFactory is None:
             raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE)
 

From a3de1c5a66e034637cb0cdfb9c99617823ab2eec Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Tue, 24 Apr 2018 10:33:01 -0700
Subject: [PATCH 35/41] Support OpenStack in the cloud download manager.

---
 lib/galaxy/managers/cloud.py           | 31 +++++++++++++++++++++++++-
 lib/galaxy/webapps/galaxy/api/cloud.py |  2 +-
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index 0e3c82c456a..3020f2b16e6 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -32,7 +32,7 @@ NO_CLOUDBRIDGE_ERROR_MESSAGE = (
     "Please install CloudBridge or modify ObjectStore configuration."
 )
 
-SUPPORTED_PROVIDERS = "{aws, azure}"
+SUPPORTED_PROVIDERS = "{aws, azure, openstack}"
 
 
 class CloudManager(sharable.SharableModelManager):
@@ -93,6 +93,35 @@ class CloudManager(sharable.SharableModelManager):
                       'azure_secret': secret,
                       'azure_tenant': tenant}
             connection = CloudProviderFactory().create_provider(ProviderList.AZURE, config)
+        elif provider == "openstack":
+            username = credentials.get('username', None)
+            if username is None:
+                missing_credentials.append('username')
+            password = credentials.get('password', None)
+            if password is None:
+                missing_credentials.append('password')
+            auth_url = credentials.get('authentication URL', None)
+            if auth_url is None:
+                missing_credentials.append('authentication URL')
+            prj_name = credentials.get('project name', None)
+            if prj_name is None:
+                missing_credentials.append('project name')
+            prj_domain_name = credentials.get('project domain name', None)
+            if prj_domain_name is None:
+                missing_credentials.append('project domain name')
+            domain_name = credentials.get('domain name', None)
+            if domain_name is None:
+                missing_credentials.append('domain name')
+            if len(missing_credentials) > 0:
+                raise RequestParameterMissingException("The following required key(s) are missing from the provided "
+                                                       "credentials object: {}".format(missing_credentials))
+            config = {'username': username,
+                      'password': password,
+                      'authentication URL': auth_url,
+                      'project name': prj_name,
+                      'project domain name': prj_domain_name,
+                      'domain name': domain_name}
+            connection = CloudProviderFactory().create_provider(ProviderList.OPENSTACK, config)
         else:
             raise RequestParameterInvalidException("Unrecognized provider '{}'; the following are the supported "
                                                    "providers: {}.".format(provider, SUPPORTED_PROVIDERS))
diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py
index 5647118482b..b1d66198b69 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud.py
@@ -50,7 +50,7 @@ class CloudController(BaseAPIController):
         :type  payload: dict
         :param payload: A dictionary structure containing the following keys:
             *   history_id:    the (encoded) id of history to which the object should be downloaded to.
-            *   provider:      the name of cloud-based resource provided (e.g., 'aws' or 'azure').
+            *   provider:      the name of cloud-based resource provided (e.g., `aws`, `azure`, or `openstack`).
             *   container:     is the name of container from which data should be downloaded (e.g., a bucket name on AWS S3).
             *   object:        is the name of an object to be downloaded.
             *   credentials:   is a dictionary containing all the credentials required to authenticated to the

From c656f36ffd7df4d1586bf822cc03a387764ff020 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Tue, 24 Apr 2018 10:41:33 -0700
Subject: [PATCH 36/41] Change OpenStack required credentials field names.

---
 lib/galaxy/managers/cloud.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index 3020f2b16e6..790f4b58acc 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -100,27 +100,27 @@ class CloudManager(sharable.SharableModelManager):
             password = credentials.get('password', None)
             if password is None:
                 missing_credentials.append('password')
-            auth_url = credentials.get('authentication URL', None)
+            auth_url = credentials.get('auth_url', None)
             if auth_url is None:
-                missing_credentials.append('authentication URL')
-            prj_name = credentials.get('project name', None)
+                missing_credentials.append('auth_url')
+            prj_name = credentials.get('project_name', None)
             if prj_name is None:
-                missing_credentials.append('project name')
-            prj_domain_name = credentials.get('project domain name', None)
+                missing_credentials.append('project_name')
+            prj_domain_name = credentials.get('project_domain_name', None)
             if prj_domain_name is None:
-                missing_credentials.append('project domain name')
-            domain_name = credentials.get('domain name', None)
-            if domain_name is None:
-                missing_credentials.append('domain name')
+                missing_credentials.append('project_domain_name')
+            user_domain_name = credentials.get('user_domain_name', None)
+            if user_domain_name is None:
+                missing_credentials.append('user_domain_name')
             if len(missing_credentials) > 0:
                 raise RequestParameterMissingException("The following required key(s) are missing from the provided "
                                                        "credentials object: {}".format(missing_credentials))
-            config = {'username': username,
-                      'password': password,
-                      'authentication URL': auth_url,
-                      'project name': prj_name,
-                      'project domain name': prj_domain_name,
-                      'domain name': domain_name}
+            config = {'os_username': username,
+                      'os_password': password,
+                      'os_auth_url': auth_url,
+                      'os_project_name': prj_name,
+                      'os_project_domain_name': prj_domain_name,
+                      'os_user_domain_name': user_domain_name}
             connection = CloudProviderFactory().create_provider(ProviderList.OPENSTACK, config)
         else:
             raise RequestParameterInvalidException("Unrecognized provider '{}'; the following are the supported "

From 8302e574446a8719da430f1174762bf98f8200fe Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Tue, 24 Apr 2018 10:54:05 -0700
Subject: [PATCH 37/41] Refactor `container` to `bucket` in cloud API and
 manager.

---
 lib/galaxy/managers/cloud.py           | 20 ++++++++++----------
 lib/galaxy/webapps/galaxy/api/cloud.py | 24 ++++++++++++------------
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index 790f4b58acc..9e7d3c895f9 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -132,7 +132,7 @@ class CloudManager(sharable.SharableModelManager):
         except ProviderConnectionException as e:
             raise AuthenticationFailed("Could not authenticate to the '{}' provider. {}".format(provider, e))
 
-    def download(self, trans, history_id, provider, container, obj, credentials):
+    def download(self, trans, history_id, provider, bucket, obj, credentials):
         """
         Implements the logic of downloading a file from a cloud-based storage (e.g., Amazon S3)
         and persisting it as a Galaxy dataset.
@@ -147,11 +147,11 @@ class CloudManager(sharable.SharableModelManager):
         :param provider: the name of cloud-based resource provided. A list of supported providers is given in
         `SUPPORTED_PROVIDERS` variable.
 
-        :type  container: string
-        :param container: is the name of container from which data should be downloaded (e.g., a bucket name on AWS S3).
+        :type  bucket: string
+        :param bucket: the name of a bucket from which data should be downloaded (e.g., a bucket name on AWS S3).
 
         :type  obj: string
-        :param obj: is the name of an object to be downloaded.
+        :param obj: the name of an object to be downloaded.
 
         :type  credentials: dict
         :param credentials: a dictionary containing all the credentials required to authenticated to the
@@ -165,13 +165,13 @@ class CloudManager(sharable.SharableModelManager):
 
         connection = self._configure_provider(provider, credentials)
         try:
-            container_obj = connection.object_store.get(container)
-            if container_obj is None:
-                raise RequestParameterInvalidException("The container `{}` not found.".format(container))
+            bucket_obj = connection.object_store.get(bucket)
+            if bucket_obj is None:
+                raise RequestParameterInvalidException("The bucket `{}` not found.".format(bucket))
         except Exception as e:
-            raise ItemAccessibilityException("Could not get the container `{}`: {}".format(container, str(e)))
+            raise ItemAccessibilityException("Could not get the bucket `{}`: {}".format(bucket, str(e)))
 
-        key = container_obj.get(obj)
+        key = bucket_obj.get(obj)
         if key is None:
             raise ObjectNotFound("Could not get the object `{}`.".format(obj))
         staging_file_name = os.path.abspath(os.path.join(
@@ -216,5 +216,5 @@ class CloudManager(sharable.SharableModelManager):
         os.remove(staging_file_name)
         return datasets
 
-    def upload(self, dataset, provider, container, obj):
+    def upload(self, dataset, provider, bucket, obj):
         raise NotImplementedError
diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py
index b1d66198b69..902650490fe 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud.py
@@ -30,10 +30,10 @@ class CloudController(BaseAPIController):
     def index(self, trans, **kwargs):
         """
         * GET /api/cloud
-            Lists cloud-based containers (e.g., S3 bucket, Azure blob) user has defined.
+            Lists cloud-based buckets (e.g., S3 bucket, Azure blob) user has defined.
         :param trans:
         :param kwargs:
-        :return: A list of cloud-based containers user has defined.
+        :return: A list of cloud-based buckets user has defined.
         """
         # TODO: This can be implemented leveraging PluggedMedia objects (part of the user-based object store project)
         trans.response.status = 501
@@ -43,17 +43,17 @@ class CloudController(BaseAPIController):
     def download(self, trans, payload, **kwargs):
         """
         * POST /api/cloud/download
-            Downloads a given object from a given cloud-based container.
+            Downloads a given object from a given cloud-based bucket.
         :type  trans: galaxy.web.framework.webapp.GalaxyWebTransaction
         :param trans: Galaxy web transaction
 
         :type  payload: dict
         :param payload: A dictionary structure containing the following keys:
             *   history_id:    the (encoded) id of history to which the object should be downloaded to.
-            *   provider:      the name of cloud-based resource provided (e.g., `aws`, `azure`, or `openstack`).
-            *   container:     is the name of container from which data should be downloaded (e.g., a bucket name on AWS S3).
-            *   object:        is the name of an object to be downloaded.
-            *   credentials:   is a dictionary containing all the credentials required to authenticated to the
+            *   provider:      the name of a cloud-based resource provided (e.g., `aws`, `azure`, or `openstack`).
+            *   bucket:        the name of a bucket from which data should be downloaded (e.g., a bucket name on AWS S3).
+            *   object:        the name of an object to be downloaded.
+            *   credentials:   a dictionary containing all the credentials required to authenticated to the
             specified provider (e.g., {"secret_key": YOUR_AWS_SECRET_TOKEN, "access_key": YOUR_AWS_ACCESS_TOKEN}).
 
         :param kwargs:
@@ -76,9 +76,9 @@ class CloudController(BaseAPIController):
         if provider is None:
             missing_arguments.append("provider")
 
-        container = payload.get("container", None)
-        if container is None:
-            missing_arguments.append("container")
+        bucket = payload.get("bucket", None)
+        if bucket is None:
+            missing_arguments.append("bucket")
 
         obj = payload.get("object", None)
         if obj is None:
@@ -99,7 +99,7 @@ class CloudController(BaseAPIController):
         datasets = self.cloud_manager.download(trans=trans,
                                                history_id=history_id,
                                                provider=provider,
-                                               container=container,
+                                               bucket=bucket,
                                                obj=obj,
                                                credentials=credentials)
         rtv = []
@@ -111,7 +111,7 @@ class CloudController(BaseAPIController):
     def upload(self, trans, payload, **kwargs):
         """
         * POST /api/cloud/upload
-            Uploads a given dataset to a given cloud-based container.
+            Uploads a given dataset to a given cloud-based bucket.
         :param trans:
         :param payload:
         :param kwargs:

From 6e11334fa7122858012c644038eb0bf58de7e6f2 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Wed, 4 Jul 2018 17:51:20 -0700
Subject: [PATCH 38/41] Refactor cloud download api and use only one handle to
 the staging file.

---
 lib/galaxy/managers/cloud.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index 9e7d3c895f9..9b08598fe4d 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -177,12 +177,12 @@ class CloudManager(sharable.SharableModelManager):
         staging_file_name = os.path.abspath(os.path.join(
             trans.app.config.new_file_path,
             "cd_" + ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(11))))
-        staging_file = open(staging_file_name, "w+")
-        key.save_content(staging_file)
 
-        datasets = []
-        with open(staging_file_name, "r") as f:
-            content = f.read()
+        with open(staging_file_name, "w+b") as staging_file:
+            key.save_content(staging_file)
+            staging_file.seek(0)
+            datasets = []
+            content = staging_file.read()
             headers = {'content-disposition': 'form-data; name="{}"; filename="{}"'.format('files_0|file_data', obj), }
 
             input_file = FieldStorage(headers=headers)
@@ -212,7 +212,6 @@ class CloudManager(sharable.SharableModelManager):
                 hids.update({staging_file: output['out_data'][0][1].hid})
                 for d in output['out_data']:
                     datasets.append(d[1].dataset)
-        staging_file.close()
         os.remove(staging_file_name)
         return datasets
 

From 3e551cd9994e76820151bbd002ed782e36c23a69 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Thu, 5 Jul 2018 17:57:24 -0700
Subject: [PATCH 39/41] Rename the cloud storage APIs from Download and Upload
 to Copy-from and Copy-To, respectively.

---
 lib/galaxy/managers/cloud.py           | 4 ++--
 lib/galaxy/webapps/galaxy/api/cloud.py | 4 ++--
 lib/galaxy/webapps/galaxy/buildapp.py  | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index 9b08598fe4d..7fde40c7a0d 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -132,7 +132,7 @@ class CloudManager(sharable.SharableModelManager):
         except ProviderConnectionException as e:
             raise AuthenticationFailed("Could not authenticate to the '{}' provider. {}".format(provider, e))
 
-    def download(self, trans, history_id, provider, bucket, obj, credentials):
+    def copy_from(self, trans, history_id, provider, bucket, obj, credentials):
         """
         Implements the logic of downloading a file from a cloud-based storage (e.g., Amazon S3)
         and persisting it as a Galaxy dataset.
@@ -215,5 +215,5 @@ class CloudManager(sharable.SharableModelManager):
         os.remove(staging_file_name)
         return datasets
 
-    def upload(self, dataset, provider, bucket, obj):
+    def copy_to(self, dataset, provider, bucket, obj):
         raise NotImplementedError
diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py
index 902650490fe..f94e16c0dcd 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud.py
@@ -40,7 +40,7 @@ class CloudController(BaseAPIController):
         return 'Not Implemented'
 
     @expose_api
-    def download(self, trans, payload, **kwargs):
+    def copy_from(self, trans, payload, **kwargs):
         """
         * POST /api/cloud/download
             Downloads a given object from a given cloud-based bucket.
@@ -108,7 +108,7 @@ class CloudController(BaseAPIController):
         return rtv
 
     @expose_api
-    def upload(self, trans, payload, **kwargs):
+    def copy_to(self, trans, payload, **kwargs):
         """
         * POST /api/cloud/upload
             Uploads a given dataset to a given cloud-based bucket.
diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py
index e6aaa085fac..e188804b50a 100644
--- a/lib/galaxy/webapps/galaxy/buildapp.py
+++ b/lib/galaxy/webapps/galaxy/buildapp.py
@@ -232,10 +232,10 @@ def populate_api_routes(webapp, app):
     webapp.mapper.connect('/api/histories/shared_with_me', action='shared_with_me', controller="histories")
 
     webapp.mapper.resource('cloud', 'cloud', path_prefix='/api')
-    webapp.mapper.connect('cloud_download',
-                          '/api/cloud/download',
+    webapp.mapper.connect('cloud_storage_copy_from',
+                          '/api/cloud/storage/copy-from',
                           controller='cloud',
-                          action='download',
+                          action='copy_from',
                           conditions=dict(method=["POST"]))
 
     _add_item_tags_controller(webapp,

From cfe6a9a67757150d7fce0d42f992cd1065cac9ee Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Fri, 6 Jul 2018 10:54:19 -0700
Subject: [PATCH 40/41] Update documentation of the cloud copy from/to APIs
 reflecting previous refactoring.

---
 lib/galaxy/managers/cloud.py           |  2 +-
 lib/galaxy/webapps/galaxy/api/cloud.py | 16 +++++++---------
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py
index 7fde40c7a0d..f9dc3d217a7 100644
--- a/lib/galaxy/managers/cloud.py
+++ b/lib/galaxy/managers/cloud.py
@@ -134,7 +134,7 @@ class CloudManager(sharable.SharableModelManager):
 
     def copy_from(self, trans, history_id, provider, bucket, obj, credentials):
         """
-        Implements the logic of downloading a file from a cloud-based storage (e.g., Amazon S3)
+        Implements the logic of copying a file from a cloud-based storage (e.g., Amazon S3)
         and persisting it as a Galaxy dataset.
 
         :type  trans: galaxy.web.framework.webapp.GalaxyWebTransaction
diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py
index f94e16c0dcd..e344032685e 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud.py
@@ -42,26 +42,24 @@ class CloudController(BaseAPIController):
     @expose_api
     def copy_from(self, trans, payload, **kwargs):
         """
-        * POST /api/cloud/download
-            Downloads a given object from a given cloud-based bucket.
+        * POST /api/cloud/storage/copy-from
+            Copies a given object from a given cloud-based bucket to a Galaxy history.
         :type  trans: galaxy.web.framework.webapp.GalaxyWebTransaction
         :param trans: Galaxy web transaction
 
         :type  payload: dict
         :param payload: A dictionary structure containing the following keys:
-            *   history_id:    the (encoded) id of history to which the object should be downloaded to.
+            *   history_id:    the (encoded) id of history to which the object should be copied to.
             *   provider:      the name of a cloud-based resource provided (e.g., `aws`, `azure`, or `openstack`).
-            *   bucket:        the name of a bucket from which data should be downloaded (e.g., a bucket name on AWS S3).
-            *   object:        the name of an object to be downloaded.
+            *   bucket:        the name of a bucket from which data should be copied from (e.g., a bucket name on AWS S3).
+            *   object:        the name of an object to be copied.
             *   credentials:   a dictionary containing all the credentials required to authenticated to the
             specified provider (e.g., {"secret_key": YOUR_AWS_SECRET_TOKEN, "access_key": YOUR_AWS_ACCESS_TOKEN}).
 
         :param kwargs:
 
         :rtype:  dictionary
-        :return: a dictionary with the following keys:
-            *   status:     HTTP status code (e.g., "200" if the provided object is successfully downloaded to Galaxy).
-            *   message:    a description complementary to the status code.
+        :return: a dictionary containing a `summary` view of the datasets copied from the given cloud-based storage.
         """
         if not isinstance(payload, dict):
             raise ActionInputError('Invalid payload data type. The payload is expected to be a dictionary, '
@@ -111,7 +109,7 @@ class CloudController(BaseAPIController):
     def copy_to(self, trans, payload, **kwargs):
         """
         * POST /api/cloud/upload
-            Uploads a given dataset to a given cloud-based bucket.
+            Copies a given dataset to a given cloud-based bucket.
         :param trans:
         :param payload:
         :param kwargs:

From b61bb88daa3566c8b929d81e74f7cc9710875026 Mon Sep 17 00:00:00 2001
From: vjalili <jalili.vahid@gmail.com>
Date: Fri, 6 Jul 2018 11:13:44 -0700
Subject: [PATCH 41/41] Refactor api access to the cloud storage index.

---
 lib/galaxy/webapps/galaxy/api/cloud.py | 2 +-
 lib/galaxy/webapps/galaxy/buildapp.py  | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py
index e344032685e..9cdeefa01fb 100644
--- a/lib/galaxy/webapps/galaxy/api/cloud.py
+++ b/lib/galaxy/webapps/galaxy/api/cloud.py
@@ -29,7 +29,7 @@ class CloudController(BaseAPIController):
     @expose_api
     def index(self, trans, **kwargs):
         """
-        * GET /api/cloud
+        * GET /api/cloud/storage
             Lists cloud-based buckets (e.g., S3 bucket, Azure blob) user has defined.
         :param trans:
         :param kwargs:
diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py
index e188804b50a..559e28da6ec 100644
--- a/lib/galaxy/webapps/galaxy/buildapp.py
+++ b/lib/galaxy/webapps/galaxy/buildapp.py
@@ -231,7 +231,11 @@ def populate_api_routes(webapp, app):
     webapp.mapper.connect('/api/histories/published', action='published', controller="histories", conditions=dict(method=["GET"]))
     webapp.mapper.connect('/api/histories/shared_with_me', action='shared_with_me', controller="histories")
 
-    webapp.mapper.resource('cloud', 'cloud', path_prefix='/api')
+    webapp.mapper.connect('cloud_storage',
+                          '/api/cloud/storage/',
+                          controller='cloud',
+                          action='index',
+                          conditions=dict(method=["GET"]))
     webapp.mapper.connect('cloud_storage_copy_from',
                           '/api/cloud/storage/copy-from',
                           controller='cloud',