Merge pull request #6844 from jmchilton/compressed_types_and_extra_files

Infrastructure/API Support for Composite Uploads/Downloads

config/datatypes_conf.xml.sample

@@ -197,7 +197,11 @@
     <datatype extension="nhdr" type="galaxy.datatypes.images:Nrrd" subclass="true"/>
     <datatype extension="rna_eps" type="galaxy.datatypes.sequence:RNADotPlotMatrix" mimetype="image/eps" display_in_upload="true"/>
     <datatype extension="zip" type="galaxy.datatypes.binary:CompressedZipArchive" display_in_upload="true"/>
-    <datatype extension="tar" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true"/>
+    <datatype extension="tar" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true">
+      <converter file="tar_to_directory.xml" target_datatype="directory"/>
+    </datatype>
+    <datatype extension="directory" type="galaxy.datatypes.data:Directory">
+    </datatype>
     <!-- Proteomics Datatypes -->
     <datatype extension="pepxml" type="galaxy.datatypes.proteomics:PepXml" mimetype="application/xml" display_in_upload="true"/>
     <datatype extension="raw_pepxml" type="galaxy.datatypes.proteomics:PepXml" mimetype="application/xml" subclass="true"/>

lib/galaxy/datatypes/converters/tar_to_directory.xml

@@ -0,0 +1,16 @@
+<tool id="CONVERTER_tar_to_directory" name="Convert tar to directory" version="1.0.0" profile="17.05">
+    <!-- Don't use tar directly so we can verify safety of results - tar -xzf '$input1'; -->
+    <command>
+        mkdir '$output1.files_path';
+        cd '$output1.files_path';
+        python -c "from galaxy.util.compression_utils import CompressedFile; CompressedFile('$input1').extract('.');"
+    </command>
+    <inputs>
+        <param format="tar" name="input1" type="data"/>
+    </inputs>
+    <outputs>
+        <data format="directory" name="output1"/>
+    </outputs>
+    <help>
+    </help>
+</tool>

lib/galaxy/datatypes/data.py

@@ -927,6 +927,10 @@ class Text(Data):
         return dataproviders.line.RegexLineDataProvider(dataset_source, **settings)
 
 
+class Directory(Data):
+    """Class representing a directory of files."""
+
+
 class GenericAsn1(Text):
     """Class for generic ASN.1 text format"""
     edam_data = "data_0849"

lib/galaxy/tools/__init__.py

@@ -138,6 +138,7 @@ GALAXY_LIB_TOOLS_UNVERSIONED = [
     "CONVERTER_maf_to_fasta_0",
     "CONVERTER_maf_to_interval_0",
     "CONVERTER_wiggle_to_interval_0",
+    "CONVERTER_tar_to_directory",
     # Tools improperly migrated to the tool shed (devteam)
     "qualityFilter",
     "winSplitter",

lib/galaxy/tools/parameters/grouping.py

@@ -16,6 +16,7 @@ from galaxy.exceptions import (
     ConfigDoesNotAllowException,
 )
 from galaxy.util import (
+    asbool,
     inflector,
     relpath,
     sanitize_for_filename
@@ -217,7 +218,7 @@ class UploadDataset(Group):
             filenames = list()
             for composite_file in context.get('files', []):
                 if not composite_file.get('ftp_files', ''):
-                    filenames.append(composite_file.get('file_data', {}).get('filename', ''))
+                    filenames.append((composite_file.get('file_data') or {}).get('filename', ''))
                 else:
                     filenames.append(composite_file.get('ftp_files', [])[0])
             dataset_name = os.path.commonprefix(filenames).rstrip('.') or None
@@ -543,13 +544,14 @@ class UploadDataset(Group):
         d_type = self.get_datatype(trans, context)
         dbkey = self.get_dbkey(context)
         tag_using_filenames = context.get('tag_using_filenames', False)
+        force_composite = asbool(context.get('force_composite', 'False'))
         writable_files = d_type.writable_files
         writable_files_offset = 0
         groups_incoming = [None for _ in range(file_count)]
         for group_incoming in context.get(self.name, []):
             i = int(group_incoming['__index__'])
             groups_incoming[i] = group_incoming
-        if d_type.composite_type is not None:
+        if d_type.composite_type is not None or force_composite:
             # handle uploading of composite datatypes
             # Only one Dataset can be created
             dataset = Bunch()
@@ -594,18 +596,23 @@ class UploadDataset(Group):
                 dataset.warnings.extend(warnings)
             if dataset.primary_file is None:  # remove this before finish, this should create an empty dataset
                 raise Exception('No primary dataset file was available for composite upload')
-            keys = [value.name for value in writable_files.values()]
-            for i, group_incoming in enumerate(groups_incoming[writable_files_offset :]):
+            if not force_composite:
+                keys = [value.name for value in writable_files.values()]
+            else:
+                keys = [str(index) for index in range(file_count)]
+            for i, group_incoming in enumerate(groups_incoming[writable_files_offset:]):
                 key = keys[i + writable_files_offset]
-                if group_incoming is None and not writable_files[list(writable_files.keys())[keys.index(key)]].optional:
+                if not force_composite and group_incoming is None and not writable_files[list(writable_files.keys())[keys.index(key)]].optional:
                     dataset.warnings.append("A required composite file (%s) was not specified." % (key))
                     dataset.composite_files[key] = None
                 else:
                     file_bunch, warnings = get_one_filename(group_incoming)
                     dataset.warnings.extend(warnings)
                     if file_bunch.path:
+                        if force_composite:
+                            key = group_incoming.get("NAME") or i
                         dataset.composite_files[key] = file_bunch.__dict__
-                    else:
+                    elif not force_composite:
                         dataset.composite_files[key] = None
                         if not writable_files[list(writable_files.keys())[keys.index(key)]].optional:
                             dataset.warnings.append("A required composite file (%s) was not specified." % (key))

lib/galaxy/util/compression_utils.py

@@ -65,6 +65,10 @@ def get_fileobj_raw(filename, mode="r", compressed_formats=None):
 
 class CompressedFile(object):
 
+    @staticmethod
+    def can_decompress(file_path):
+        return tarfile.is_tarfile(file_path) or zipfile.is_zipfile(file_path)
+
     def __init__(self, file_path, mode='r'):
         if tarfile.is_tarfile(file_path):
             self.file_type = 'tar'

lib/galaxy/webapps/galaxy/api/datasets.py

@@ -2,6 +2,7 @@
 API operations on the contents of a history dataset.
 """
 import logging
+import os
 
 from six import string_types
 
@@ -13,6 +14,9 @@ from galaxy import (
     web
 )
 from galaxy.datatypes import dataproviders
+from galaxy.util.path import (
+    safe_walk
+)
 from galaxy.visualization.data_providers.genome import (
     BamDataProvider,
     FeatureLocationIndexDataProvider,
@@ -302,6 +306,25 @@ class DatasetsController(BaseAPIController, UsesVisualizationMixin):
 
         return data
 
+    @web.expose_api_anonymous
+    def extra_files(self, trans, history_content_id, history_id, **kwd):
+        """
+        GET /api/histories/{encoded_history_id}/contents/{encoded_content_id}/extra_files
+        Generate list of extra files.
+        """
+        decoded_content_id = self.decode_id(history_content_id)
+
+        hda = self.hda_manager.get_accessible(decoded_content_id, trans.user)
+        extra_files_path = hda.extra_files_path
+        rval = []
+        for root, directories, files in safe_walk(extra_files_path):
+            for directory in directories:
+                rval.append({"class": "Directory", "path": os.path.relpath(os.path.join(root, directory), extra_files_path)})
+            for file in files:
+                rval.append({"class": "File", "path": os.path.relpath(os.path.join(root, file), extra_files_path)})
+
+        return rval
+
     @web.expose_api_raw_anonymous
     def display(self, trans, history_content_id, history_id,
                 preview=False, filename=None, to_ext=None, raw=False, **kwd):

lib/galaxy/webapps/galaxy/buildapp.py

@@ -223,6 +223,11 @@ def populate_api_routes(webapp, app):
                           controller="history_contents",
                           action="update_permissions",
                           conditions=dict(method=["PUT"]))
+    webapp.mapper.connect("history_contents_extra_files",
+                          "/api/histories/{history_id}/contents/{history_content_id}/extra_files",
+                          controller="datasets",
+                          action="extra_files",
+                          conditions=dict(method=["GET"]))
     webapp.mapper.connect("history_contents_metadata_file",
                           "/api/histories/{history_id}/contents/{history_content_id}/metadata_file",
                           controller="datasets",

test/api/test_tools.py

@@ -221,6 +221,59 @@ class ToolsTestCase(api.ApiTestCase):
         test_data = test_data_response.json()
         assert len(test_data) == 3
 
+    @uses_test_history(require_new=False)
+    def test_upload_composite_as_tar(self, history_id):
+        tar_path = self.test_data_resolver.get_filename("testdir.tar")
+        with open(tar_path, "rb") as tar_f:
+            payload = self.dataset_populator.upload_payload(history_id, "Test123",
+                extra_inputs={
+                    "files_1|file_data": tar_f,
+                    "files_1|NAME": "composite",
+                    "file_count": "2",
+                    "force_composite": "True",
+                }
+            )
+            run_response = self.dataset_populator.tools_post(payload)
+            self.dataset_populator.wait_for_tool_run(history_id, run_response)
+            dataset = run_response.json()["outputs"][0]
+            content = self.dataset_populator.get_history_dataset_content(history_id, dataset=dataset)
+            assert content.strip() == "Test123"
+            extra_files = self.dataset_populator.get_history_dataset_extra_files(history_id, dataset_id=dataset["id"])
+            assert len(extra_files) == 5, extra_files
+            expected_contents = {
+                "testdir": "Directory",
+                "testdir/c": "Directory",
+                "testdir/a": "File",
+                "testdir/b": "File",
+                "testdir/c/d": "File",
+            }
+            found_files = set()
+            for extra_file in extra_files:
+                path = extra_file["path"]
+                assert path in expected_contents
+                assert extra_file["class"] == expected_contents[path]
+                found_files.add(path)
+
+            assert len(found_files) == 5, found_files
+
+    @uses_test_history(require_new=False)
+    def test_upload_composite_from_bad_tar(self, history_id):
+        tar_path = self.test_data_resolver.get_filename("unsafe.tar")
+        with open(tar_path, "rb") as tar_f:
+            payload = self.dataset_populator.upload_payload(history_id, "Test123",
+                extra_inputs={
+                    "files_1|file_data": tar_f,
+                    "files_1|NAME": "composite",
+                    "file_count": "2",
+                    "force_composite": "True",
+                }
+            )
+            run_response = self.dataset_populator.tools_post(payload)
+            self.dataset_populator.wait_for_tool_run(history_id, run_response, assert_ok=False)
+            dataset = run_response.json()["outputs"][0]
+            details = self.dataset_populator.get_history_dataset_details(history_id, dataset=dataset, assert_ok=False)
+            assert details["state"] == "error"
+
     def test_unzip_collection(self):
         with self.dataset_populator.test_history() as history_id:
             hdca_id = self.__build_pair(history_id, ["123", "456"])

test/api/test_tools_upload.py

@@ -463,6 +463,27 @@ class ToolsUploadTestCase(api.ApiTestCase):
             content = self.dataset_populator.get_history_dataset_content(history_id, dataset=datasets[2])
             assert content == ONE_TO_SIX_WITH_TABS
 
+    def test_upload_force_composite(self):
+        with self.dataset_populator.test_history() as history_id:
+            payload = self.dataset_populator.upload_payload(history_id, "Test123",
+                extra_inputs={
+                    "files_1|url_paste": "CompositeContent",
+                    "files_1|NAME": "composite",
+                    "file_count": "2",
+                    "force_composite": "True",
+                }
+            )
+            run_response = self.dataset_populator.tools_post(payload)
+            self.dataset_populator.wait_for_tool_run(history_id, run_response)
+            dataset = run_response.json()["outputs"][0]
+            content = self.dataset_populator.get_history_dataset_content(history_id, dataset=dataset)
+            assert content.strip() == "Test123"
+            extra_files = self.dataset_populator.get_history_dataset_extra_files(history_id, dataset_id=dataset["id"])
+            assert len(extra_files) == 1, extra_files  # [{u'path': u'1', u'class': u'File'}]
+            extra_file = extra_files[0]
+            assert extra_file["path"] == "composite"
+            assert extra_file["class"] == "File"
+
     def test_upload_from_invalid_url(self):
         history_id, new_dataset = self._upload('https://usegalaxy.org/bla123', assert_ok=False)
         dataset_details = self.dataset_populator.get_history_dataset_details(history_id, dataset_id=new_dataset["id"], assert_ok=False)

test/base/populators.py

@@ -372,6 +372,12 @@ class BaseDatasetPopulator(object):
         assert details_response.status_code == 200
         return details_response.json()
 
+    def get_history_dataset_extra_files(self, history_id, **kwds):
+        dataset_id = self.__history_content_id(history_id, **kwds)
+        details_response = self._get_contents_request(history_id, "/%s/extra_files" % dataset_id)
+        assert details_response.status_code == 200, details_response.content
+        return details_response.json()
+
     def get_history_collection_details(self, history_id, **kwds):
         hdca_id = self.__history_content_id(history_id, **kwds)
         details_response = self._get_contents_request(history_id, "/dataset_collections/%s" % hdca_id)

test/functional/tools/sample_datatypes_conf.xml

@@ -42,5 +42,12 @@
     <datatype extension="data_manager_json" type="galaxy.datatypes.text:Json" mimetype="application/json" subclass="true" display_in_upload="false"/>
     <datatype extension="data" type="galaxy.datatypes.data:Data" mimetype="application/octet-stream" max_optional_metadata_filesize="1048576" />
     <datatype extension="binary" type="galaxy.datatypes.binary:Binary" mimetype="application/octet-stream" max_optional_metadata_filesize="1048576" />
+    <datatype extension="zip" type="galaxy.datatypes.binary:CompressedZipArchive" display_in_upload="true">
+    </datatype>
+    <datatype extension="tar" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true">
+      <converter file="tar_to_directory.xml" target_datatype="directory"/>
+    </datatype>
+    <datatype extension="directory" type="galaxy.datatypes.data:Directory">
+    </datatype>
   </registration>
 </datatypes>

test/integration/test_upload_configuration_options.py

@@ -839,3 +839,25 @@ class FetchByPathTestCase(BaseUploadContentConfigurationTestCase):
         library = matching[0]
         dataset = self.library_populator.get_library_contents_with_path(library["id"], "/file1")
         assert dataset["file_size"] == 6, dataset
+
+
+class TestDirectoryAndCompressedTypes(BaseUploadContentConfigurationTestCase):
+
+    require_admin_user = True
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        config["allow_path_paste"] = True
+
+    def test_tar_to_directory(self):
+        dataset = self.dataset_populator.new_dataset(
+            self.history_id, 'file://%s/testdir.tar' % TEST_DATA_DIRECTORY, file_type="tar", auto_decompress=False, wait=True
+        )
+        dataset = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=dataset)
+        assert dataset["file_ext"] == "tar", dataset
+        response = self.dataset_populator.run_tool(
+            tool_id="CONVERTER_tar_to_directory",
+            inputs={"input1": {"src": "hda", "id": dataset["id"]}},
+            history_id=self.history_id,
+        )
+        self.dataset_populator.wait_for_job(response["jobs"][0]["id"])

tools/data_source/upload.py

@@ -18,8 +18,10 @@ from galaxy.datatypes.registry import Registry
 from galaxy.datatypes.upload_util import handle_upload, UploadProblemException
 from galaxy.util import (
     bunch,
+    safe_makedirs,
     unicodify
 )
+from galaxy.util.compression_utils import CompressedFile
 
 assert sys.version_info[:2] >= (2, 7)
 
@@ -172,6 +174,7 @@ def add_file(dataset, registry, output_path):
 
 
 def add_composite_file(dataset, registry, output_path, files_path):
+    datatype = None
 
     # Find data type
     if dataset.file_type is not None:
@@ -180,39 +183,74 @@ def add_composite_file(dataset, registry, output_path, files_path):
         except Exception:
             print("Unable to instantiate the datatype object for the file type '%s'" % dataset.file_type)
 
+    def to_path(path_or_url):
+        is_url = path_or_url.find('://') != -1  # todo fixme
+        if is_url:
+            try:
+                temp_name = sniff.stream_to_file(urlopen(path_or_url), prefix='url_paste')
+            except Exception as e:
+                raise UploadProblemException('Unable to fetch %s\n%s' % (path_or_url, str(e)))
+
+            return temp_name, is_url
+
+        return path_or_url, is_url
+
+    def make_files_path():
+        safe_makedirs(files_path)
+
+    def stage_file(name, composite_file_path, is_binary=False):
+        dp = composite_file_path['path']
+        path, is_url = to_path(dp)
+        if is_url:
+            dataset.path = path
+            dp = path
+
+        auto_decompress = composite_file_path.get('auto_decompress', True)
+        if auto_decompress and not datatype.composite_type and CompressedFile.can_decompress(dp):
+            # It isn't an explictly composite datatype, so these are just extra files to attach
+            # as composite data. It'd be better if Galaxy was communicating this to the tool
+            # a little more explicitly so we didn't need to dispatch on the datatype and so we
+            # could attach arbitrary extra composite data to an existing composite datatype if
+            # if need be? Perhaps that would be a mistake though.
+            CompressedFile(dp).extract(files_path)
+        else:
+            if not is_binary:
+                tmpdir = output_adjacent_tmpdir(output_path)
+                tmp_prefix = 'data_id_%s_convert_' % dataset.dataset_id
+                if composite_file_path.get('space_to_tab'):
+                    sniff.convert_newlines_sep2tabs(dp, tmp_dir=tmpdir, tmp_prefix=tmp_prefix)
+                else:
+                    sniff.convert_newlines(dp, tmp_dir=tmpdir, tmp_prefix=tmp_prefix)
+
+            file_output_path = os.path.join(files_path, name)
+            shutil.move(dp, file_output_path)
+
+            # groom the dataset file content if required by the corresponding datatype definition
+            if datatype.dataset_content_needs_grooming(file_output_path):
+                datatype.groom_dataset_content(file_output_path)
+
+    # Do we have pre-defined composite files from the datatype definition.
     if dataset.composite_files:
-        os.mkdir(files_path)
+        make_files_path()
         for name, value in dataset.composite_files.items():
             value = bunch.Bunch(**value)
+            if value.name not in dataset.composite_file_paths:
+                raise UploadProblemException("Failed to find file_path %s in %s" % (value.name, dataset.composite_file_paths))
             if dataset.composite_file_paths[value.name] is None and not value.optional:
                 raise UploadProblemException('A required composite data file was not provided (%s)' % name)
             elif dataset.composite_file_paths[value.name] is not None:
-                dp = dataset.composite_file_paths[value.name]['path']
-                isurl = dp.find('://') != -1  # todo fixme
-                if isurl:
-                    try:
-                        temp_name = sniff.stream_to_file(urlopen(dp), prefix='url_paste')
-                    except Exception as e:
-                        raise UploadProblemException('Unable to fetch %s\n%s' % (dp, str(e)))
-                    dataset.path = temp_name
-                    dp = temp_name
-                if not value.is_binary:
-                    tmpdir = output_adjacent_tmpdir(output_path)
-                    tmp_prefix = 'data_id_%s_convert_' % dataset.dataset_id
-                    if dataset.composite_file_paths[value.name].get('space_to_tab', value.space_to_tab):
-                        sniff.convert_newlines_sep2tabs(dp, tmp_dir=tmpdir, tmp_prefix=tmp_prefix)
-                    else:
-                        sniff.convert_newlines(dp, tmp_dir=tmpdir, tmp_prefix=tmp_prefix)
+                composite_file_path = dataset.composite_file_paths[value.name]
+                stage_file(name, composite_file_path, value.is_binary)
 
-                # move the file to its final destination
-                file_output_path = os.path.join(files_path, name)
-                shutil.move(dp, file_output_path)
-                # groom the dataset file content if required by the corresponding datatype definition
-                if datatype.dataset_content_needs_grooming(file_output_path):
-                    datatype.groom_dataset_content(file_output_path)
+    # Do we have ad-hoc user supplied composite files.
+    elif dataset.composite_file_paths:
+        make_files_path()
+        for key, composite_file in dataset.composite_file_paths.items():
+            stage_file(key, composite_file)  # TODO: replace these defaults
 
     # Move the dataset to its "real" path
-    shutil.move(dataset.primary_file, output_path)
+    primary_file_path, _ = to_path(dataset.primary_file)
+    shutil.move(primary_file_path, output_path)
 
     # Write the job info
     return dict(type='dataset',

tools/data_source/upload.xml

@@ -54,6 +54,7 @@
       </param>
       <param name="NAME" type="hidden" help="Name for dataset in upload"></param>
     </upload_dataset>
+    <param name="force_composite" type="hidden" value="false" />
     <param name="dbkey" type="genomebuild" label="Genome" />
     <conditional name="files_metadata" value_from="self:app.datatypes_registry.get_upload_metadata_params" value_ref="file_type" value_ref_in_group="False" />
     <!-- <param name="other_dbkey" type="text" label="Or user-defined Genome" /> -->