From 60d0d9d5ec36fb1e6a79a4f51ab919325687b10d Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Wed, 31 Jan 2018 13:50:34 +0100
Subject: [PATCH 01/26] Add grooming for composite files.

Add call to groom_dataset_content() for composite files.
---
 tools/data_source/upload.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py
index 6b4f636d7da..e2e2a1ab5c3 100644
--- a/tools/data_source/upload.py
+++ b/tools/data_source/upload.py
@@ -166,7 +166,15 @@ def add_file(dataset, registry, output_path):
     return info
 
 
-def add_composite_file(dataset, output_path, files_path):
+def add_composite_file(dataset, registry, output_path, files_path):
+
+    # Find data type
+    if dataset.file_type is not None:
+        try:
+            datatype = registry.get_datatype_by_extension(dataset.file_type)
+        except Exception as e:
+            print("Unable to instantiate the datatype object for the file type '%s'" % dataset.file_type)
+
     if dataset.composite_files:
         os.mkdir(files_path)
         for name, value in dataset.composite_files.items():
@@ -190,9 +198,17 @@ def add_composite_file(dataset, output_path, files_path):
                         sniff.convert_newlines_sep2tabs(dp, tmp_dir=tmpdir, tmp_prefix=tmp_prefix)
                     else:
                         sniff.convert_newlines(dp, tmp_dir=tmpdir, tmp_prefix=tmp_prefix)
-                shutil.move(dp, os.path.join(files_path, name))
+
+                # move the file to its final destination
+                file_output_path = os.path.join(files_path, name)
+                shutil.move(dp, file_output_path)
+                # groom the dataset file content if required by the corresponding datatype definition
+                if datatype.dataset_content_needs_grooming(file_output_path):
+                    datatype.groom_dataset_content(file_output_path)
+
     # Move the dataset to its "real" path
     shutil.move(dataset.primary_file, output_path)
+
     # Write the job info
     return dict(type='dataset',
                 dataset_id=dataset.dataset_id,
@@ -258,7 +274,7 @@ def __main__():
         try:
             if dataset.type == 'composite':
                 files_path = output_paths[int(dataset.dataset_id)][1]
-                metadata.append(add_composite_file(dataset, output_path, files_path))
+                metadata.append(add_composite_file(dataset, registry, output_path, files_path))
             else:
                 metadata.append(add_file(dataset, registry, output_path))
         except UploadProblemException as e:

From 665e11facbf0b9b86fbcde04593fcd49ebe96777 Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Wed, 31 Jan 2018 13:51:38 +0100
Subject: [PATCH 02/26] Add ISA composite data type.

Add new data types isa-tab and isa-json.
Add isa-rwval library (isatools Python 2 version) in requirements.
---
 config/datatypes_conf.xml.sample              |   6 +
 lib/galaxy/datatypes/isa.py                   | 516 ++++++++++++++++++
 .../pipfiles/default/pinned-requirements.txt  |   1 +
 3 files changed, 523 insertions(+)
 create mode 100644 lib/galaxy/datatypes/isa.py

diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample
index d4422b7adeb..a66d8b69540 100644
--- a/config/datatypes_conf.xml.sample
+++ b/config/datatypes_conf.xml.sample
@@ -147,6 +147,11 @@
       <display file="rviewer/bed.xml" inherit="true"/>
       <display file="igv/interval_as_bed.xml" inherit="true"/>
     </datatype>
+
+    <!-- ISA data types -->
+    <datatype extension="isa-tab" type="galaxy.datatypes.isa:IsaTab" mimetype="application/isa-tools" display_in_upload="true" description="ISA-Tab data type." description_url="https://isa-tools.org"/>
+    <datatype extension="isa-json" type="galaxy.datatypes.isa:IsaJson" mimetype="application/isa-tools" display_in_upload="true" description="ISA-JSON data type." description_url="https://isa-tools.org"/>
+
     <datatype extension="picard_interval_list" type="galaxy.datatypes.tabular:Tabular" subclass="true" display_in_upload="true">
       <converter file="picard_interval_list_to_bed6_converter.xml" target_datatype="bed6"/>
     </datatype>
@@ -809,6 +814,7 @@
     <sniffer type="galaxy.datatypes.images:Xpm"/>
     <sniffer type="galaxy.datatypes.images:Eps"/>
     <sniffer type="galaxy.datatypes.images:Rast"/>
+
     <!--
     Keep this commented until the sniff method in the assembly.py
     module is fixed to not read the entire file.
diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
new file mode 100644
index 00000000000..74b3ce400f0
--- /dev/null
+++ b/lib/galaxy/datatypes/isa.py
@@ -0,0 +1,516 @@
+"""
+ISA datatype
+
+See http://isa-tools.org
+
+"""
+
+from __future__ import print_function
+
+import re
+import os
+import os.path
+import sys
+import glob
+import json
+import shutil
+import zipfile
+import logging
+import tarfile
+import tempfile
+import csv
+from isatools import isatab
+from isatools import isajson
+from json import dumps
+from io import BytesIO
+from cgi import escape
+from galaxy import util
+from galaxy.datatypes import data
+from galaxy.datatypes import metadata
+from galaxy.util.sanitize_html import sanitize_html
+from galaxy import model
+
+# CONSTANTS {{{1
+################################################################
+
+# Main files regex
+JSON_FILE_REGEX = re.compile(r"^.*\.json$", flags = re.IGNORECASE)
+INVESTIGATION_FILE_REGEX = re.compile(r"^i_\w+\.txt$", flags = re.IGNORECASE)
+
+# The name of the ISA archive (compressed file) as saved inside Galaxy
+ISA_ARCHIVE_NAME = "archive"
+
+# Archives types
+_FILE_TYPE_PREFIX = {
+    "\x1f\x8b\x08": "gz",
+    "\x42\x5a\x68": "bz2",
+    "\x50\x4b\x03\x04": "zip"
+}
+_MAX_LEN_FILE_TYPE_PREFIX = max(len(x) for x in _FILE_TYPE_PREFIX)
+_FILE_TYPE_REGEX = re.compile("(%s)" % "|".join(map(re.escape, _FILE_TYPE_PREFIX.keys())))
+
+# Set max number of lines of the history peek
+_MAX_LINES_HISTORY_PEEK = 11
+
+# Configure logger {{{1
+################################################################
+
+logger = logging.getLogger(__name__)
+ch = logging.StreamHandler(sys.stdout)
+formatter = logging.Formatter("%(name)s %(levelname)s %(asctime)s %(message)s")
+ch.setFormatter(formatter)
+logger.handlers = []
+logger.propagate = False
+logger.addHandler(ch)
+logger.setLevel(logging.ERROR)
+
+# Function for opening correctly a CSV file for csv.reader() for both Python 2 and 3 {{{1
+################################################################
+
+def utf8_text_file_open(path):
+    if sys.version_info[0] < 3: 
+        fp = open(path, 'rb')
+    else:
+        fp = open(path, 'r', newline='', encoding='utf8')
+    return fp
+
+# ISA class {{{1
+################################################################
+
+class _Isa(data.Data):
+    """ Base class for implementing ISA datatypes """
+    composite_type = 'auto_primary_file'
+    allow_datatype_change = False
+    is_binary = True
+    _main_file_regex = None
+
+    # Make investigation instance {{{2
+    ################################################################
+
+    def _make_investigation_instance(self, filename):
+        raise NotImplementedError()
+
+    # Constructor {{{2
+    ################################################################
+
+    def __init__(self, main_file_regex, **kwd):
+        super(_Isa, self).__init__(**kwd)
+        self._main_file_regex = main_file_regex
+
+        # Add the archive file as the only composite file
+        self.add_composite_file(ISA_ARCHIVE_NAME, is_binary=True, optional=True)
+
+    # Get ISA folder path {{{2
+    ################################################################
+
+    def _get_isa_folder_path(self, dataset):
+
+        isa_folder = None
+
+        if dataset:
+            if isinstance(dataset, model.Dataset):
+                isa_folder = dataset.extra_files_path
+            if isinstance(dataset, model.HistoryDatasetAssociation):
+                # XXX With this loop the dataset name is reset inside the history to the ISA archive ID. Why?
+                for attr, value in dataset.__dict__.iteritems():
+                    if str(attr) == '_metadata_collection':
+                        datatype = value.parent.datatype
+                isa_folder = dataset.dataset.extra_files_path
+
+        if isa_folder is None:
+            raise Exception('Unvalid dataset object, or no extra files path found for this dataset.')
+
+        return isa_folder
+
+    # Get main file {{{2
+    ################################################################
+
+    def _get_main_file(self, dataset):
+        """Get the main file of the ISA archive. Either the investigation file i_*.txt for ISA-Tab, or the JSON file for ISA-JSON."""
+
+        main_file = None
+        isa_folder = self._get_isa_folder_path(dataset)
+
+        if os.path.exists(isa_folder):
+
+            # Get ISA archive older
+            isa_files = os.listdir(isa_folder)
+
+            # Try to find main file 
+            main_file = self._find_main_file_in_archive(isa_files)
+
+            if main_file is None:
+                raise Exception('Invalid ISA archive. No main file found.')
+
+            # Make full path
+            main_file = os.path.join(isa_folder, main_file)
+
+        return main_file
+
+    # Get investigation {{{2
+    ################################################################
+
+    def _get_investigation(self, dataset):
+        """Create a contained instance specific to the exact ISA type (Tab or Json).
+           We will use it to parse and access information from the archive."""
+
+        investigation = None
+        main_file = self._get_main_file(dataset)
+        if main_file is not None:
+            investigation = self._make_investigation_instance(main_file)
+
+        return investigation
+
+    # Find main file in archive {{{2
+    ################################################################
+
+    def _find_main_file_in_archive(self, files_list):
+        """Find the main file inside the ISA archive."""
+
+        found_file = None
+
+        for f in files_list:
+            match = self._main_file_regex.match(f)
+            if match:
+                if found_file is None:
+                    found_file = match.group()
+                else:
+                    raise Exception('More than one file match the pattern "', str(file_regex), '" to identify the investigation file')
+
+        return found_file
+
+    # Extract archive {{{2
+    ################################################################
+
+    def _extract_archive(self, stream, output_path=None):
+        """Extract files from archive and put them is predefined folder."""
+
+        # extract the archive to a temp folder
+        if output_path is None:
+            output_path = tempfile.mkdtemp()
+        # try to detect the type of the compressed archive
+        a_type = self._detect_file_type(stream)
+        # decompress the archive
+        if a_type == "zip":
+            self._extract_zip_archive(stream, output_path)
+        elif a_type == "gz":
+            self._extract_tar_archive(stream, output_path)
+        else:
+            raise Exception("Not supported archive format!!!")
+
+        return output_path
+
+    # Extract ZIP archive {{{2
+    ################################################################
+
+    def _extract_zip_archive(self, stream, target_path):
+        """Extract files from a ZIP archive."""
+
+        temp_folder = tempfile.mkdtemp()
+        data = BytesIO(stream.read())
+        zip_ref = zipfile.ZipFile(data)
+        zip_ref.extractall(path=temp_folder)
+        self._move_to_target_path(temp_folder, target_path)
+
+    # Extract TAR archive {{{2
+    ################################################################
+
+    def _extract_tar_archive(self, stream, target_path):
+        """Extract files from a TAR archive."""
+
+        # extract the TAR archive
+        temp_folder = tempfile.mkdtemp()
+        with tarfile.open(fileobj=stream) as tar:
+            tar.extractall(path=temp_folder)
+        self._move_to_target_path(temp_folder, target_path)
+
+    # Move to target path {{{2
+    ################################################################
+
+    def _move_to_target_path(self, temp_folder, target_path, delete_temp_folder=True):
+        """Move extracted files to the destination folder imposed by Galaxy."""
+
+        # find the root folder containing the dataset
+        tmp_subfolders = [f for f in os.listdir(temp_folder) if
+                          not f.startswith(".") and f not in (ISA_ARCHIVE_NAME, "__MACOSX")]
+        # move files contained within the root dataset folder to their target path
+        root_folder = os.path.join(temp_folder, tmp_subfolders[0])
+        if len(tmp_subfolders) == 1 and os.path.isdir(root_folder):
+            # move the root dataset folder to its final destination and clean the temp data
+            for f in os.listdir(root_folder):
+                shutil.move(os.path.join(root_folder, f), target_path)
+        elif len(tmp_subfolders) > 1:
+            for f in tmp_subfolders:
+                shutil.move(os.path.join(temp_folder, f), target_path)
+        # clean temp data if required
+        if delete_temp_folder:
+            shutil.rmtree(temp_folder)
+
+    # List archive files {{{2
+    ################################################################
+
+    def _list_archive_files(self, stream):
+        """List files contained inside the ISA archive."""
+
+        # try to detect the type of the compressed archive
+        a_type = self._detect_file_type(stream)
+        # decompress the archive
+        if a_type == "zip":
+            data = BytesIO(stream.read())
+            zip_ref = zipfile.ZipFile(data)
+            files_list = zip_ref.namelist()
+        elif a_type == "gz":
+            with tarfile.open(fileobj=stream) as tar:
+                files_list = [i.name for i in tar]
+        else:
+            raise Exception("Not supported archive format!!!")
+        # filter the base path if it exists
+        if len(files_list) > 0:
+            base_path = files_list[0].split("/")[0]
+            if base_path:
+                # the TAR archive encodes the base_path without a final '/'
+                if base_path in files_list:
+                    files_list.remove(base_path)
+                # the ZIP archive encodes the base_path with a final '/'
+                base_path = os.path.join(base_path, '')
+                if base_path in files_list:
+                    files_list.remove(base_path)
+                # remove the base_path from all remaining files
+                files_list = [f.replace(base_path, '') for f in files_list]
+        return files_list
+
+    # Detect file type {{{2
+    ################################################################
+
+    def _detect_file_type(self, stream):
+        """
+        Try to detect the type of the ISA archive: is it ZIP, or GUNZIP?
+
+        :return: "zip" or "gz" if the file type is detected; None otherwise.
+        """
+        file_type = None
+        file_start = stream.read(_MAX_LEN_FILE_TYPE_PREFIX)
+        stream.seek(0)  # reset the stream
+        matched_prefix = _FILE_TYPE_REGEX.match(file_start)
+        if matched_prefix:
+            file_type = _FILE_TYPE_PREFIX[matched_prefix.string[matched_prefix.start():matched_prefix.end()]]
+
+        return file_type
+
+    # Set peek {{{2
+    ################################################################
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        """Set the peek and blurb text. Get first lines of the main file and set it as the peek."""
+
+        main_file = self._get_main_file(dataset)
+
+        if main_file is None:
+            raise RuntimeError("Unable to find the main file within the 'files_path' folder")
+
+        # Read first lines of main file
+        with open(main_file, "r") as f:
+            data = []
+            for line in f:
+                if len(data) < _MAX_LINES_HISTORY_PEEK:
+                    data.append(line)
+                else:
+                    break
+            if not dataset.dataset.purged and data:
+                dataset.peek = json.dumps({"data": data})
+                dataset.blurb = 'data'
+            else:
+                dataset.peek = 'file does not exist'
+                dataset.blurb = 'file purged from disk'
+
+    # Display peek {{{2
+    ################################################################
+
+    def display_peek(self, dataset):
+        """Create the HTML table used for displaying peek, from the peek text found by set_peek() method."""
+
+        out = ['<table cellspacing="0" cellpadding="3">']
+        try:
+            if not dataset.peek:
+                dataset.set_peek()
+            json_data = json.loads(dataset.peek)
+            for line in json_data["data"]:
+                line = line.strip()
+                if not line:
+                    continue
+                out.append('<tr><td>%s</td></tr>' % escape(util.unicodify(line, 'utf-8')))
+            out.append('</table>')
+            out = "".join(out)
+        except Exception as exc:
+            out = "Can't create peek %s" % str(exc)
+        return out
+
+    # Generate primary file {{{2
+    ################################################################
+
+    def generate_primary_file(self, dataset=None):
+        """Generate the primary file. It is an HTML file containing description of the composite dataset
+           as well as a list of the composite files that it contains."""
+
+        if dataset:
+            rval = ['<html><head><title>ISA Dataset </title></head><p/>']
+            if hasattr(dataset, "extra_files_path"):
+                rval.append('<div>ISA Dataset composed of the following files:<p/><ul>')
+                for cmp_file in os.listdir(dataset.extra_files_path):
+                    opt_text = ''
+                    rval.append('<li><a href="%s" type="text/plain">%s</a>%s</li>' % (cmp_file, cmp_file, opt_text))
+                rval.append('</ul></div></html>')
+            else:
+                rval.append('<div>ISA Dataset is empty!<p/><ul>')
+            return "\n".join(rval)
+        return "<div>No dataset available</div>"
+
+    # Dataset content needs grooming {{{2
+    ################################################################
+
+    def dataset_content_needs_grooming(self, file_name):
+        """This function is called on an output dataset file after the content is initially generated."""
+        return os.path.basename(file_name) == ISA_ARCHIVE_NAME
+
+    # Groom dataset content {{{2
+    ################################################################
+
+    def groom_dataset_content(self, file_name):
+        """This method is called by Galaxy to extract files contained in a composite data type."""
+        # XXX Is the right place to extract files? Should this step not be a cleaning step instead?
+        # Could extracting be done earlier and composite files declared as files contained inside the archive
+        # instead of the archive itself?
+
+        # extract basename and folder of the current file whose content has to be groomed
+        basename = os.path.basename(file_name)
+        output_path = os.path.dirname(file_name)
+        # extract archive if the file corresponds to the ISA archive
+        if basename == ISA_ARCHIVE_NAME:
+            # perform extraction
+            with open(file_name, 'rb') as stream:
+                self._extract_archive(stream, output_path=output_path)
+            # remove the original archive file
+            os.remove(file_name)
+
+    # Set meta {{{2
+    ################################################################
+
+    def set_meta( self, dataset, overwrite=True, **kwd ):
+        """Set meta data information."""
+        super(_Isa, self).set_meta(dataset, **kwd)
+        self._set_dataset_name(dataset)
+        return True
+
+    # Set dataset name {{{2
+    ################################################################
+
+    def _set_dataset_name(self, dataset):
+        investigation = self._get_investigation(dataset)
+        if investigation is not None:
+            dataset.name = investigation.identifier
+        else:
+            dataset.name = 'ISA DATASET'
+
+    # Display data {{{2
+    ################################################################
+
+
+    def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, offset=None, ck_size=None, **kwd):
+        """Downloads the ISA dataset if `preview` is `False`;
+           if `preview` is `True`, it returns a preview of the ISA dataset as a HTML page.
+           The preview is triggered when user clicks on the eye icon of the composite dataset."""
+
+        self._set_dataset_name(dataset)
+        # if it is not required a preview use the default behaviour of `display_data`
+        if not preview:
+            return super(_Isa, self).display_data(trans, dataset, preview, filename, to_ext, **kwd)
+
+        # prepare the preview of the ISA dataset
+        investigation = self._get_investigation(dataset)
+        if investigation is None:
+            html = """<html><header><title>Error while reading ISA archive.</title></header>
+                   <body>
+                        <h1>An error occured while reading content of ISA archive.</h1>
+                        <p>If you have tried to load your archive with the uploader by selecting isa-tab as composite data type, then try to load it again with isa-json instead. Conversely, if you have tried to load your archive with the uploader by selecting isa-json as composite data type, then try isa-tab instead.</p>
+                        <p>You may also try to look into your zip file in order to find out if this is a proper ISA archive. If you see a file i_Investigation.txt inside, then it is an ISA-Tab archive. If you see a file with extension .json inside, then it is an ISA-JSON archive. If you see nothing like that, then either your ISA archive is corrupted, or it is not an ISA archive.</p>
+                   </body></html>"""
+        else:
+            html = '<html><body>'
+            html += '<h1>{0} {1}</h1>'.format(investigation.title, investigation.identifier)
+
+            # Loop on all studies
+            for study in investigation.studies:
+                html += '<h2>Study %s</h2>' % study.identifier
+                html += '<h3>%s</h3>' % study.title
+                html += '<p>%s</p>' % study.description
+                html += '<p>Submitted the %s</p>' % study.submission_date
+                html += '<p>Released on %s</p>' % study.public_release_date
+
+                # Loop on all assays of this study
+                for assay in study.assays:
+                    html += '<h3>Assay %s</h3>' % assay.filename
+                    html += '<p>Measurement type: %s</p>' % assay.measurement_type.term # OntologyAnnotation
+                    html += '<p>Technology type: %s</p>' % assay.technology_type.term # OntologyAnnotation
+                    html += '<p>Technology platform: %s</p>' % assay.technology_platform
+                    if assay.data_files is not None:
+                        html += '<p>Data files:</p>'
+                        html += '<ul>'
+                        for data_file in assay.data_files:
+                            html += '<li>' + str(data_file.id) + ' - ' + str(data_file.filename) + ' - ' + str(data_file.label) + '</li>'
+                        html += '</ul>'
+
+            html += '</body></html>'
+
+        # Set mime type
+        mime = 'text/html'
+        self._clean_and_set_mime_type(trans, mime)
+
+        return sanitize_html(html).encode('utf-8')
+
+# ISA-Tab class {{{1
+################################################################
+
+class IsaTab(_Isa):
+    file_ext = "isa-tab"
+
+    # Constructor {{{2
+    ################################################################
+
+    def __init__(self, **kwd):
+        super(IsaTab, self).__init__(main_file_regex = INVESTIGATION_FILE_REGEX, **kwd)
+
+    # Make investigation instance {{{2
+    ################################################################
+
+    def _make_investigation_instance(self, filename):
+
+        # Parse ISA-Tab investigation file
+        parser = isatab.InvestigationParser()
+        fp = utf8_text_file_open(filename)
+        parser.parse(fp)
+        isa = parser.isa
+
+        return isa
+
+# ISA-JSON class {{{1
+################################################################
+
+class IsaJson(_Isa):
+    file_ext = "isa-json"
+
+    # Constructor {{{2
+    ################################################################
+
+    def __init__(self, **kwd):
+        super(IsaJson, self).__init__(main_file_regex = JSON_FILE_REGEX, **kwd)
+
+    # Make investigation instance {{{2
+    ################################################################
+
+    def _make_investigation_instance(self, filename):
+
+        # Parse JSON file
+        fp = utf8_text_file_open(filename)
+        isa = isajson.load(fp)
+
+        return isa
diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
index 5ed36dd808c..6af205df7b6 100644
--- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
@@ -41,6 +41,7 @@ h5py==2.6.0
 html5lib==1.0.1
 idna==2.6
 ipaddress==1.0.19; python_version < '3.3'
+isa-rwval==0.10.3
 iso8601==0.1.12
 jmespath==0.9.3
 jsonpatch==1.23

From 1fb63a1369e8fa75b1f7825693ef286682659a64 Mon Sep 17 00:00:00 2001
From: Pablo Moreno <pcm32@users.noreply.github.com>
Date: Thu, 22 Feb 2018 19:35:31 +0000
Subject: [PATCH 03/26] [ISA datatype] adds logger setting for isatools.isatab
 to avoid failing uploads when no pandas (#31)

* [ISA datatype] adds logger setting for isatools.isatab to avoid failing uploads when no pandas.

* Update isa.py
---
 lib/galaxy/datatypes/isa.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index 74b3ce400f0..64f87a8d8ab 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -19,6 +19,8 @@ import logging
 import tarfile
 import tempfile
 import csv
+# Imports isatab after turning off warnings inside logger settings to avoid pandas warning making uploads fail.
+logging.getLogger("isatools.isatab").setLevel(logging.ERROR)
 from isatools import isatab
 from isatools import isajson
 from json import dumps

From fe139ad8dfe6c83188c33498d8d2854b33670c04 Mon Sep 17 00:00:00 2001
From: Pablo Moreno <pcm32@users.noreply.github.com>
Date: Thu, 22 Mar 2018 17:23:42 +0000
Subject: [PATCH 04/26] Installs isa-rwval from develop

Otherwise failing currently when building in the container.
---
 .../dependencies/pinned-requirements.txt      | 95 ++++++++++++++++++-
 1 file changed, 94 insertions(+), 1 deletion(-)
 mode change 120000 => 100644 lib/galaxy/dependencies/pinned-requirements.txt

diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
deleted file mode 120000
index 26ec2b5e875..00000000000
--- a/lib/galaxy/dependencies/pinned-requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-pipfiles/default/pinned-requirements.txt
\ No newline at end of file
diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
new file mode 100644
index 00000000000..f812d0246da
--- /dev/null
+++ b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -0,0 +1,94 @@
+# bx-python and dependencies
+numpy==1.9.2
+python-lzo==1.11
+bx-python==0.8.1
+
+# packages with C extensions
+MarkupSafe==1.0
+PyYAML==3.12
+SQLAlchemy==1.2.2
+sqlalchemy-utils==0.32.19
+mercurial==3.7.3; python_version < '3.0'
+pycrypto==2.6.1
+uWSGI==2.0.15
+# Flexible BAM index naming is new to core pysam
+pysam==0.14
+
+# pure Python packages
+bdbag==1.1.1
+bleach==2.1.3
+bz2file==0.98; python_version < '3.3'
+ipaddress==1.0.18; python_version < '3.3'
+boltons==17.1.0
+Paste==2.0.3
+PasteDeploy==1.5.2
+docutils==0.14
+repoze.lru==0.6
+Routes==2.4.1
+WebOb==1.4.1
+Mako==1.0.2
+pytz==2017.3
+Babel==2.5.1
+Beaker==1.7.0
+dictobj==0.3.1
+nose==1.3.7
+Parsley==1.3
+six==1.11.0
+Whoosh==2.7.4
+galaxy_sequence_utils==1.0.2
+h5py==2.7.1
+packaging==16.8
+
+# pykwalify and dependencies
+pykwalify==1.6.0
+python-dateutil==2.5.3
+docopt==0.6.2
+
+# Cheetah and dependencies
+Cheetah3==3.0.0
+Markdown==2.6.3
+
+# Requests and dependencies
+requests==2.18.4
+certifi==2017.7.27.1
+urllib3==1.22
+chardet==3.0.4
+idna==2.6
+
+# BioBlend and dependencies
+bioblend==0.7.0
+boto==2.38.0
+requests-toolbelt==0.8.0
+
+
+# kombu and dependencies
+kombu==3.0.30
+amqp==1.4.8
+anyjson==0.3.3
+
+# Pulsar requirements
+psutil==4.1.0
+pulsar-galaxy-lib==0.8.3
+
+# sqlalchemy-migrate and dependencies
+sqlalchemy-migrate==0.11.0
+decorator==4.0.2
+Tempita==0.5.3dev
+sqlparse==0.1.16
+pbr==3.1.1
+
+# svgwrite and dependencies
+svgwrite==1.1.6
+pyparsing==2.1.1
+
+# Fabric and dependencies
+Fabric3==1.13.1.post1
+paramiko==2.2.1
+ecdsa==0.13
+
+# GenomeSpace dependencies
+python-genomespaceclient==0.1.8
+
+# ISA
+git+https://github.com/ISA-tools/isa-rwval.git@develop#egg=isa-rwval
+

From 06c57810d9050ffd4c81ce8e189741f255d6eafc Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Tue, 27 Mar 2018 12:30:20 +0200
Subject: [PATCH 05/26] Correct linting issues detected by Travis-CI run.

---
 lib/galaxy/datatypes/isa.py | 39 +++++++++++++++++--------------------
 1 file changed, 18 insertions(+), 21 deletions(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index 64f87a8d8ab..69b53d8323d 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -7,37 +7,35 @@ See http://isa-tools.org
 
 from __future__ import print_function
 
-import re
 import os
+import re
 import os.path
 import sys
-import glob
 import json
 import shutil
-import zipfile
 import logging
+import zipfile
 import tarfile
 import tempfile
-import csv
+from cgi import escape
+from io import BytesIO
+from json import dumps
+
 # Imports isatab after turning off warnings inside logger settings to avoid pandas warning making uploads fail.
 logging.getLogger("isatools.isatab").setLevel(logging.ERROR)
-from isatools import isatab
 from isatools import isajson
-from json import dumps
-from io import BytesIO
-from cgi import escape
+from isatools import isatab
 from galaxy import util
 from galaxy.datatypes import data
-from galaxy.datatypes import metadata
-from galaxy.util.sanitize_html import sanitize_html
 from galaxy import model
+from galaxy.util.sanitize_html import sanitize_html
 
 # CONSTANTS {{{1
 ################################################################
 
 # Main files regex
-JSON_FILE_REGEX = re.compile(r"^.*\.json$", flags = re.IGNORECASE)
-INVESTIGATION_FILE_REGEX = re.compile(r"^i_\w+\.txt$", flags = re.IGNORECASE)
+JSON_FILE_REGEX = re.compile(r"^.*\.json$", flags=re.IGNORECASE)
+INVESTIGATION_FILE_REGEX = re.compile(r"^i_\w+\.txt$", flags=re.IGNORECASE)
 
 # The name of the ISA archive (compressed file) as saved inside Galaxy
 ISA_ARCHIVE_NAME = "archive"
@@ -70,7 +68,7 @@ logger.setLevel(logging.ERROR)
 ################################################################
 
 def utf8_text_file_open(path):
-    if sys.version_info[0] < 3: 
+    if sys.version_info[0] < 3:
         fp = open(path, 'rb')
     else:
         fp = open(path, 'r', newline='', encoding='utf8')
@@ -138,7 +136,7 @@ class _Isa(data.Data):
             # Get ISA archive older
             isa_files = os.listdir(isa_folder)
 
-            # Try to find main file 
+            # Try to find main file
             main_file = self._find_main_file_in_archive(isa_files)
 
             if main_file is None:
@@ -177,7 +175,7 @@ class _Isa(data.Data):
                 if found_file is None:
                     found_file = match.group()
                 else:
-                    raise Exception('More than one file match the pattern "', str(file_regex), '" to identify the investigation file')
+                    raise Exception('More than one file match the pattern "', str(self._main_file_regex), '" to identify the investigation file')
 
         return found_file
 
@@ -397,7 +395,7 @@ class _Isa(data.Data):
     # Set meta {{{2
     ################################################################
 
-    def set_meta( self, dataset, overwrite=True, **kwd ):
+    def set_meta(self, dataset, overwrite=True, **kwd):
         """Set meta data information."""
         super(_Isa, self).set_meta(dataset, **kwd)
         self._set_dataset_name(dataset)
@@ -416,7 +414,6 @@ class _Isa(data.Data):
     # Display data {{{2
     ################################################################
 
-
     def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, offset=None, ck_size=None, **kwd):
         """Downloads the ISA dataset if `preview` is `False`;
            if `preview` is `True`, it returns a preview of the ISA dataset as a HTML page.
@@ -451,8 +448,8 @@ class _Isa(data.Data):
                 # Loop on all assays of this study
                 for assay in study.assays:
                     html += '<h3>Assay %s</h3>' % assay.filename
-                    html += '<p>Measurement type: %s</p>' % assay.measurement_type.term # OntologyAnnotation
-                    html += '<p>Technology type: %s</p>' % assay.technology_type.term # OntologyAnnotation
+                    html += '<p>Measurement type: %s</p>' % assay.measurement_type.term  # OntologyAnnotation
+                    html += '<p>Technology type: %s</p>' % assay.technology_type.term    # OntologyAnnotation
                     html += '<p>Technology platform: %s</p>' % assay.technology_platform
                     if assay.data_files is not None:
                         html += '<p>Data files:</p>'
@@ -479,7 +476,7 @@ class IsaTab(_Isa):
     ################################################################
 
     def __init__(self, **kwd):
-        super(IsaTab, self).__init__(main_file_regex = INVESTIGATION_FILE_REGEX, **kwd)
+        super(IsaTab, self).__init__(main_file_regex=INVESTIGATION_FILE_REGEX, **kwd)
 
     # Make investigation instance {{{2
     ################################################################
@@ -504,7 +501,7 @@ class IsaJson(_Isa):
     ################################################################
 
     def __init__(self, **kwd):
-        super(IsaJson, self).__init__(main_file_regex = JSON_FILE_REGEX, **kwd)
+        super(IsaJson, self).__init__(main_file_regex=JSON_FILE_REGEX, **kwd)
 
     # Make investigation instance {{{2
     ################################################################

From 5babb80fc24febbbbed1af5b0bf56c84f16ab8a4 Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Tue, 27 Mar 2018 12:51:40 +0200
Subject: [PATCH 06/26] Correct again lint errors for Travis-CI.

---
 lib/galaxy/datatypes/isa.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index 69b53d8323d..67ed6687be4 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -7,16 +7,16 @@ See http://isa-tools.org
 
 from __future__ import print_function
 
-import os
-import re
-import os.path
-import sys
 import json
-import shutil
 import logging
-import zipfile
+import os
+import os.path
+import re
+import sys
+import shutil
 import tarfile
 import tempfile
+import zipfile
 from cgi import escape
 from io import BytesIO
 from json import dumps
@@ -25,9 +25,10 @@ from json import dumps
 logging.getLogger("isatools.isatab").setLevel(logging.ERROR)
 from isatools import isajson
 from isatools import isatab
+
+from galaxy import model
 from galaxy import util
 from galaxy.datatypes import data
-from galaxy import model
 from galaxy.util.sanitize_html import sanitize_html
 
 # CONSTANTS {{{1
@@ -64,6 +65,7 @@ logger.propagate = False
 logger.addHandler(ch)
 logger.setLevel(logging.ERROR)
 
+
 # Function for opening correctly a CSV file for csv.reader() for both Python 2 and 3 {{{1
 ################################################################
 
@@ -74,6 +76,7 @@ def utf8_text_file_open(path):
         fp = open(path, 'r', newline='', encoding='utf8')
     return fp
 
+
 # ISA class {{{1
 ################################################################
 
@@ -114,7 +117,7 @@ class _Isa(data.Data):
                 # XXX With this loop the dataset name is reset inside the history to the ISA archive ID. Why?
                 for attr, value in dataset.__dict__.iteritems():
                     if str(attr) == '_metadata_collection':
-                        datatype = value.parent.datatype
+                        datatype = value.parent.datatype  # noqa: F841
                 isa_folder = dataset.dataset.extra_files_path
 
         if isa_folder is None:
@@ -466,6 +469,7 @@ class _Isa(data.Data):
 
         return sanitize_html(html).encode('utf-8')
 
+
 # ISA-Tab class {{{1
 ################################################################
 
@@ -491,6 +495,7 @@ class IsaTab(_Isa):
 
         return isa
 
+
 # ISA-JSON class {{{1
 ################################################################
 

From 84c002283209a5717682c31d1013034e9cfc828b Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Tue, 27 Mar 2018 12:57:15 +0200
Subject: [PATCH 07/26] Correct some more lint errors for Travis-CI.

---
 lib/galaxy/datatypes/isa.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index 67ed6687be4..2b425f5a4b6 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -12,14 +12,14 @@ import logging
 import os
 import os.path
 import re
-import sys
 import shutil
+import sys
 import tarfile
 import tempfile
 import zipfile
 from cgi import escape
 from io import BytesIO
-from json import dumps
+from json import dumps  # noqa: F401
 
 # Imports isatab after turning off warnings inside logger settings to avoid pandas warning making uploads fail.
 logging.getLogger("isatools.isatab").setLevel(logging.ERROR)

From b9c63df60e12b6220f088645c17bf6d8bd68ab1b Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Mon, 16 Apr 2018 15:38:55 +0200
Subject: [PATCH 08/26] Change ISA Tools web site link.

Replace ISA Tools .org web site that has just been hacked, by the GitHub
page of the organization.
---
 lib/galaxy/datatypes/isa.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index 2b425f5a4b6..8d13102b294 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -1,7 +1,7 @@
 """
 ISA datatype
 
-See http://isa-tools.org
+See https://github.com/ISA-tools
 
 """
 

From cf7034399083c77d38e29c2832339776df103983 Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Tue, 24 Apr 2018 09:11:09 +0200
Subject: [PATCH 09/26] Escape filename in primary file.

---
 lib/galaxy/datatypes/isa.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index 8d13102b294..fe9791ce1de 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -360,8 +360,7 @@ class _Isa(data.Data):
             if hasattr(dataset, "extra_files_path"):
                 rval.append('<div>ISA Dataset composed of the following files:<p/><ul>')
                 for cmp_file in os.listdir(dataset.extra_files_path):
-                    opt_text = ''
-                    rval.append('<li><a href="%s" type="text/plain">%s</a>%s</li>' % (cmp_file, cmp_file, opt_text))
+                    rval.append('<li><a href="%s" type="text/plain">%s</a></li>' % (cmp_file, escape(cmp_file)))
                 rval.append('</ul></div></html>')
             else:
                 rval.append('<div>ISA Dataset is empty!<p/><ul>')

From a50537cd5a2cb1039bcc7b0fa215d523c6e97499 Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Tue, 24 Apr 2018 10:00:28 +0200
Subject: [PATCH 10/26] Escape assay information in ISA display_data().

---
 lib/galaxy/datatypes/isa.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index fe9791ce1de..d5f57007e8c 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -457,7 +457,7 @@ class _Isa(data.Data):
                         html += '<p>Data files:</p>'
                         html += '<ul>'
                         for data_file in assay.data_files:
-                            html += '<li>' + str(data_file.id) + ' - ' + str(data_file.filename) + ' - ' + str(data_file.label) + '</li>'
+                            html += '<li>' + escape(util.unicodify(str(data_file.id), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.filename), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.label), 'utf-8')) + '</li>'
                         html += '</ul>'
 
             html += '</body></html>'

From 86de0d3bc431404d6fca26e7506ec11415411788 Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Tue, 24 Apr 2018 10:55:23 +0200
Subject: [PATCH 11/26] Remove unused method _list_archive_files().

---
 lib/galaxy/datatypes/isa.py | 33 ---------------------------------
 1 file changed, 33 deletions(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index d5f57007e8c..fdab87daed4 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -249,39 +249,6 @@ class _Isa(data.Data):
         if delete_temp_folder:
             shutil.rmtree(temp_folder)
 
-    # List archive files {{{2
-    ################################################################
-
-    def _list_archive_files(self, stream):
-        """List files contained inside the ISA archive."""
-
-        # try to detect the type of the compressed archive
-        a_type = self._detect_file_type(stream)
-        # decompress the archive
-        if a_type == "zip":
-            data = BytesIO(stream.read())
-            zip_ref = zipfile.ZipFile(data)
-            files_list = zip_ref.namelist()
-        elif a_type == "gz":
-            with tarfile.open(fileobj=stream) as tar:
-                files_list = [i.name for i in tar]
-        else:
-            raise Exception("Not supported archive format!!!")
-        # filter the base path if it exists
-        if len(files_list) > 0:
-            base_path = files_list[0].split("/")[0]
-            if base_path:
-                # the TAR archive encodes the base_path without a final '/'
-                if base_path in files_list:
-                    files_list.remove(base_path)
-                # the ZIP archive encodes the base_path with a final '/'
-                base_path = os.path.join(base_path, '')
-                if base_path in files_list:
-                    files_list.remove(base_path)
-                # remove the base_path from all remaining files
-                files_list = [f.replace(base_path, '') for f in files_list]
-        return files_list
-
     # Detect file type {{{2
     ################################################################
 

From 62ff476a7c3eeaca51147b5ae0cac753b71ba9ee Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Tue, 24 Apr 2018 15:28:59 +0200
Subject: [PATCH 12/26] Use CompressedFile class.

Replace unzip custom code by Galaxy call to CompressedFile class.
---
 lib/galaxy/datatypes/isa.py | 113 +++++-------------------------------
 1 file changed, 15 insertions(+), 98 deletions(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index fdab87daed4..c669486c408 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -29,6 +29,7 @@ from isatools import isatab
 from galaxy import model
 from galaxy import util
 from galaxy.datatypes import data
+from galaxy.util.compression_utils import CompressedFile
 from galaxy.util.sanitize_html import sanitize_html
 
 # CONSTANTS {{{1
@@ -41,15 +42,6 @@ INVESTIGATION_FILE_REGEX = re.compile(r"^i_\w+\.txt$", flags=re.IGNORECASE)
 # The name of the ISA archive (compressed file) as saved inside Galaxy
 ISA_ARCHIVE_NAME = "archive"
 
-# Archives types
-_FILE_TYPE_PREFIX = {
-    "\x1f\x8b\x08": "gz",
-    "\x42\x5a\x68": "bz2",
-    "\x50\x4b\x03\x04": "zip"
-}
-_MAX_LEN_FILE_TYPE_PREFIX = max(len(x) for x in _FILE_TYPE_PREFIX)
-_FILE_TYPE_REGEX = re.compile("(%s)" % "|".join(map(re.escape, _FILE_TYPE_PREFIX.keys())))
-
 # Set max number of lines of the history peek
 _MAX_LINES_HISTORY_PEEK = 11
 
@@ -182,91 +174,6 @@ class _Isa(data.Data):
 
         return found_file
 
-    # Extract archive {{{2
-    ################################################################
-
-    def _extract_archive(self, stream, output_path=None):
-        """Extract files from archive and put them is predefined folder."""
-
-        # extract the archive to a temp folder
-        if output_path is None:
-            output_path = tempfile.mkdtemp()
-        # try to detect the type of the compressed archive
-        a_type = self._detect_file_type(stream)
-        # decompress the archive
-        if a_type == "zip":
-            self._extract_zip_archive(stream, output_path)
-        elif a_type == "gz":
-            self._extract_tar_archive(stream, output_path)
-        else:
-            raise Exception("Not supported archive format!!!")
-
-        return output_path
-
-    # Extract ZIP archive {{{2
-    ################################################################
-
-    def _extract_zip_archive(self, stream, target_path):
-        """Extract files from a ZIP archive."""
-
-        temp_folder = tempfile.mkdtemp()
-        data = BytesIO(stream.read())
-        zip_ref = zipfile.ZipFile(data)
-        zip_ref.extractall(path=temp_folder)
-        self._move_to_target_path(temp_folder, target_path)
-
-    # Extract TAR archive {{{2
-    ################################################################
-
-    def _extract_tar_archive(self, stream, target_path):
-        """Extract files from a TAR archive."""
-
-        # extract the TAR archive
-        temp_folder = tempfile.mkdtemp()
-        with tarfile.open(fileobj=stream) as tar:
-            tar.extractall(path=temp_folder)
-        self._move_to_target_path(temp_folder, target_path)
-
-    # Move to target path {{{2
-    ################################################################
-
-    def _move_to_target_path(self, temp_folder, target_path, delete_temp_folder=True):
-        """Move extracted files to the destination folder imposed by Galaxy."""
-
-        # find the root folder containing the dataset
-        tmp_subfolders = [f for f in os.listdir(temp_folder) if
-                          not f.startswith(".") and f not in (ISA_ARCHIVE_NAME, "__MACOSX")]
-        # move files contained within the root dataset folder to their target path
-        root_folder = os.path.join(temp_folder, tmp_subfolders[0])
-        if len(tmp_subfolders) == 1 and os.path.isdir(root_folder):
-            # move the root dataset folder to its final destination and clean the temp data
-            for f in os.listdir(root_folder):
-                shutil.move(os.path.join(root_folder, f), target_path)
-        elif len(tmp_subfolders) > 1:
-            for f in tmp_subfolders:
-                shutil.move(os.path.join(temp_folder, f), target_path)
-        # clean temp data if required
-        if delete_temp_folder:
-            shutil.rmtree(temp_folder)
-
-    # Detect file type {{{2
-    ################################################################
-
-    def _detect_file_type(self, stream):
-        """
-        Try to detect the type of the ISA archive: is it ZIP, or GUNZIP?
-
-        :return: "zip" or "gz" if the file type is detected; None otherwise.
-        """
-        file_type = None
-        file_start = stream.read(_MAX_LEN_FILE_TYPE_PREFIX)
-        stream.seek(0)  # reset the stream
-        matched_prefix = _FILE_TYPE_REGEX.match(file_start)
-        if matched_prefix:
-            file_type = _FILE_TYPE_PREFIX[matched_prefix.string[matched_prefix.start():matched_prefix.end()]]
-
-        return file_type
-
     # Set peek {{{2
     ################################################################
 
@@ -356,10 +263,20 @@ class _Isa(data.Data):
         # extract archive if the file corresponds to the ISA archive
         if basename == ISA_ARCHIVE_NAME:
             # perform extraction
-            with open(file_name, 'rb') as stream:
-                self._extract_archive(stream, output_path=output_path)
-            # remove the original archive file
-            os.remove(file_name)
+            # For some ZIP files CompressedFile::extract() extract the file inside <output_folder>/<file_name> instead of outputing it inside <output_folder>. So we first create a temporary folder, extract inside it, and move content to final destination.
+            temp_folder = tempfile.mkdtemp()
+            CompressedFile(file_name).extract(temp_folder)
+            shutil.rmtree(output_path)
+            extracted_files = os.listdir(temp_folder)
+            logger.debug(' '.join(extracted_files))
+            if len(extracted_files) == 0:
+                os.makedirs(output_path)
+                shutil.rmtree(temp_folder)
+            elif len(extracted_files) == 1 and os.path.isdir(os.path.join(temp_folder, extracted_files[0])):
+                shutil.move(os.path.join(temp_folder, extracted_files[0]), output_path)
+                shutil.rmtree(temp_folder)
+            else:
+                shutil.move(temp_folder, output_path)
 
     # Set meta {{{2
     ################################################################

From abd22c12f22b15c010bcfe6b7dca63d1c2135d18 Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Tue, 24 Apr 2018 15:30:44 +0200
Subject: [PATCH 13/26] Remove logger configuration.

---
 lib/galaxy/datatypes/isa.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index c669486c408..7d3f5a0c92c 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -49,14 +49,6 @@ _MAX_LINES_HISTORY_PEEK = 11
 ################################################################
 
 logger = logging.getLogger(__name__)
-ch = logging.StreamHandler(sys.stdout)
-formatter = logging.Formatter("%(name)s %(levelname)s %(asctime)s %(message)s")
-ch.setFormatter(formatter)
-logger.handlers = []
-logger.propagate = False
-logger.addHandler(ch)
-logger.setLevel(logging.ERROR)
-
 
 # Function for opening correctly a CSV file for csv.reader() for both Python 2 and 3 {{{1
 ################################################################

From 55bae21f77c03ebb1d114e9dbe593490b3e07993 Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Tue, 24 Apr 2018 23:28:53 +0200
Subject: [PATCH 14/26] Comment out raw files listing in ISA type.

---
 lib/galaxy/datatypes/isa.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index 7d3f5a0c92c..c2ccf961727 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -329,12 +329,13 @@ class _Isa(data.Data):
                     html += '<p>Measurement type: %s</p>' % assay.measurement_type.term  # OntologyAnnotation
                     html += '<p>Technology type: %s</p>' % assay.technology_type.term    # OntologyAnnotation
                     html += '<p>Technology platform: %s</p>' % assay.technology_platform
-                    if assay.data_files is not None:
-                        html += '<p>Data files:</p>'
-                        html += '<ul>'
-                        for data_file in assay.data_files:
-                            html += '<li>' + escape(util.unicodify(str(data_file.id), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.filename), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.label), 'utf-8')) + '</li>'
-                        html += '</ul>'
+                    # The list of Raw data files contained in the assay file cannot be obtained anymore, due to limitation in isa-rwval library. David Johnson (@djcomlab) is working on a solution, and will update isa-rwval soon.
+#                    if assay.data_files is not None:
+#                        html += '<p>Data files:</p>'
+#                        html += '<ul>'
+#                        for data_file in assay.data_files:
+#                            html += '<li>' + escape(util.unicodify(str(data_file.id), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.filename), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.label), 'utf-8')) + '</li>'
+#                        html += '</ul>'
 
             html += '</body></html>'
 

From e9f00065ee190ebdeca5c02d21c068d815674999 Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Wed, 25 Apr 2018 22:48:31 +0200
Subject: [PATCH 15/26] Remove dynamical setting of study name for ISA.

---
 lib/galaxy/datatypes/isa.py | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index c2ccf961727..78f92236365 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -270,25 +270,6 @@ class _Isa(data.Data):
             else:
                 shutil.move(temp_folder, output_path)
 
-    # Set meta {{{2
-    ################################################################
-
-    def set_meta(self, dataset, overwrite=True, **kwd):
-        """Set meta data information."""
-        super(_Isa, self).set_meta(dataset, **kwd)
-        self._set_dataset_name(dataset)
-        return True
-
-    # Set dataset name {{{2
-    ################################################################
-
-    def _set_dataset_name(self, dataset):
-        investigation = self._get_investigation(dataset)
-        if investigation is not None:
-            dataset.name = investigation.identifier
-        else:
-            dataset.name = 'ISA DATASET'
-
     # Display data {{{2
     ################################################################
 
@@ -297,7 +278,6 @@ class _Isa(data.Data):
            if `preview` is `True`, it returns a preview of the ISA dataset as a HTML page.
            The preview is triggered when user clicks on the eye icon of the composite dataset."""
 
-        self._set_dataset_name(dataset)
         # if it is not required a preview use the default behaviour of `display_data`
         if not preview:
             return super(_Isa, self).display_data(trans, dataset, preview, filename, to_ext, **kwd)

From 2be9e1f637180d04d4e085b1c91292df3e09c72e Mon Sep 17 00:00:00 2001
From: djcomlab <djcomlab@gmail.com>
Date: Thu, 26 Apr 2018 10:18:27 +0100
Subject: [PATCH 16/26] Add new lazy table parsing for isa-tab datatype and
 reintroduce data files

---
 lib/galaxy/datatypes/isa.py | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index 78f92236365..d60e42a88f8 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -303,19 +303,20 @@ class _Isa(data.Data):
                 html += '<p>Submitted the %s</p>' % study.submission_date
                 html += '<p>Released on %s</p>' % study.public_release_date
 
+                html += '<p>Factors %s</p>' % ', '.join([x.name for x in study.factors])
+
                 # Loop on all assays of this study
                 for assay in study.assays:
                     html += '<h3>Assay %s</h3>' % assay.filename
                     html += '<p>Measurement type: %s</p>' % assay.measurement_type.term  # OntologyAnnotation
                     html += '<p>Technology type: %s</p>' % assay.technology_type.term    # OntologyAnnotation
                     html += '<p>Technology platform: %s</p>' % assay.technology_platform
-                    # The list of Raw data files contained in the assay file cannot be obtained anymore, due to limitation in isa-rwval library. David Johnson (@djcomlab) is working on a solution, and will update isa-rwval soon.
-#                    if assay.data_files is not None:
-#                        html += '<p>Data files:</p>'
-#                        html += '<ul>'
-#                        for data_file in assay.data_files:
-#                            html += '<li>' + escape(util.unicodify(str(data_file.id), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.filename), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.label), 'utf-8')) + '</li>'
-#                        html += '</ul>'
+                    if assay.data_files is not None:
+                        html += '<p>Data files:</p>'
+                        html += '<ul>'
+                        for data_file in assay.data_files:
+                            html += '<li>' + escape(util.unicodify(str(data_file.id), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.filename), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.label), 'utf-8')) + '</li>'
+                        html += '</ul>'
 
             html += '</body></html>'
 
@@ -347,6 +348,12 @@ class IsaTab(_Isa):
         parser = isatab.InvestigationParser()
         fp = utf8_text_file_open(filename)
         parser.parse(fp)
+        for study in parser.isa.studies:
+            s_parser = isatab.LazyStudySampleTableParser(parser.isa)
+            s_parser.parse(study.filename)
+            for assay in study.assays:
+                a_parser = isatab.LazyAssayTableParser(parser.isa)
+                a_parser.parse(assay.filename)
         isa = parser.isa
 
         return isa

From 87cdd2156d46036807134f8d91fde4fcc2ac10de Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Thu, 26 Apr 2018 12:03:01 +0200
Subject: [PATCH 17/26] Start writing upload test for isa-tab.

Do not know how to read zip file content and submit it to uploader for a
composite type.
---
 test-data/MTBLS6.zip          | Bin 0 -> 4873 bytes
 test/api/test_tools_upload.py |  13 +++++++++++++
 2 files changed, 13 insertions(+)
 create mode 100644 test-data/MTBLS6.zip

diff --git a/test-data/MTBLS6.zip b/test-data/MTBLS6.zip
new file mode 100644
index 0000000000000000000000000000000000000000..61543c14d472f2e12a0ec01c7197f8b0eb863ff3
GIT binary patch
literal 4873
zcmbVQXHXN`x(zKv>0OE<O*$&QOOXx&N)ZH-(2-66DMnf-0qGLyU3wFQBfa+`Js?u0
zgH$092wXkq%-r+t-1mOGZ~s_(@A>v#vuF02`PQeWML<Xk0D(XN(<iCgfPaMKuQ?R#
z<lt@d@B!G_)6xm-Y~u;Fbais@w7Ki%;cDyP<ltflc80=WcVTWeR-PWNzY;urg*<&c
zf27X4e0@M0SSjJK4{@*6AVDDVe|UFim-CSsh#9&T+(?16mEN|in@DIuTR?Q{5)VY|
z^^*h|4?;hHxK0lb`_I$&BaINg10W`=C#J5Z=%@BnBw;Qi_aoH=pk_44y$6GRe>mod
zN`oDcRyrz2IQ70bb!?y*ykjmx=XJLrP>WY`+LFz^VQ9=S_tTK1hG+aXN<D@>zCYOs
z^_~MKc~7(5+dln}H@<iLChCyT{UOKh^Vu}tjKcintH?q`JfokHzm!>J40=&K(mnWT
zdsV*lBO_zRHW^*@6>G&9mCH(U^Y(l`2Ju0W4o$Xn2<j1)$^9uFCHM!o7H16Eo5%W^
zmXNklO2@ZCJTe|(JxU%!;%Y99pJgKgt2Bc>w&_jF)>TSyq8z!01`kR!^qI{$RSMA#
z3w2m@NdNoI)7-@j&+AOj=VTO_QD?cGjKS*y-F&_2_XzddGPI;a{3G<G@<?6ji{33H
zrid_9alkBx)??TeruEmjzb0}1khX3nLa@H!c`yoA(AS&m-%6QNdrxr76#R<eNgLhU
zY>dks$>=`#l7nwwQjLFAIVTG*Ghgx?u&gb+7qcqVi{+awsm{T-(4s#s)-zBpTo!lB
z81=$v-WYijjK`$XX%0LqiR``+{AvNf-`@G<>CVq*g^9KawJKvr#a+U#B=wtZR(7;H
z4HY}sLOaR}kxr2m#U$%SKll#<2Z}J6I&9#N!5IzBgXWjL+sl+v8SRdvQqh>{+J)PJ
z7y+M}Hh-};f3mg<`X-Yl9Mcjm0Jv{5l}5Bsn&Vd)v@KWaH_>Kn+>BN@$uw+sSSl@D
zaLo2wELjjZctbXIL1wW3Vk!;T(y)yKmSD=abqK)bCvOO*FlF1r_=}}ESc^aBe{MGe
zOMc({JK3Mifc5z=68WMWY<AE#!CxfsQH7JBzewcmv$3PUNuXp?;lGki|2o$RETQ<D
z1PY%5lYiUmlKbIDBSD&#$Qt$DO+t#O@S_q=n=XJX_cNSV1tmT2{Z1!9!D`9H^_a@s
zWTEodqRXG<>!H&X(*c3K81foVPm357_{aw(M*skP<N*Nm{#S|8{F0~xSi{BJ2IlEt
z2laGtb@{EnHIp5e@1N<e<Vbd9D1{>LJ&&z)7w*3-{x129K!c?#(}_XlVK&!STMi^2
zuhik?w#-izY~lT?*m3TySLzz^m4_BL1x6KB>-u5jUvTW?`#qg$%Z02n_GjlgXgdnr
z&Xw|Y1^W6l?c*pLbFqPc@cf}(Q>j@@IGt?cPZkUEgQnZd?r7$>KYG_7-*CqV<ct0R
z4Y$9>$_JH?H=ywBhZ8wliqP=PgRhVI=(zawYHW12Oyn-uLW>(YD2Vp12qD~UL2X7O
z1W*w#7E)YEErGu9+8xs7ZeXkub8+sgq-kEBTIut=S^Mw4<JrWx{Q2Wk>C>bKnJ9Nl
zUb7DKiUucvXbqz8@n5iGXd51nC1?+WZ;XMt)me2DD!}I12S$L>vQ{jy6Qv+RWA*I>
zZ<tv$bEoRZ18TDD$CY~ZH4ufJ!#*0OnKqsA$5n@Bw=%aU%M_42cn03q>`lI_$+sRq
zClS0Q!#=@MlaqO<o*Flbe{i>ODHhPzy21YG+d`yu#p`Fb#WOo<5F!-hrDkEgNqQgz
zj~(UKg0pqQ%&<liXEl6pKOlcJs0krpb^7j;lM{WoHoW<%np_xlg9tsa0vwGz7xz}%
z;zS4v-qr1*kGSxpR*#j7P;le0$noPNZC?ianL_&Q76+|I{P<D_OiBc|!Hb65S!Ay(
zE0kf=C6!L84Wi#_`1fB#+m?D=amA77v%l`tF#j0gO%={~_ISQ;R`#I_1wuNyrpLra
z=a1d1<F(;zyVlq`kWUl`&{!t)3#}AO6W*B4GDGUTdSVk^iiykpA)aN8m*n`C;uejW
z-08gg<8DfCU=Lo>Hy06;JE-_JFVBCTV|L{uFX#@hP2^`x+mMQLEM7#^CsipTW|rHH
za&8+uS{0sYhCebMlb1gm+#9ag=TO3=&u2WbxqBX?J`i}v;O^;Y)m(U*BZ0%PUf$tG
zDeFOe{*{M+kK2osR&!)1$UuA928IbH9)G3AX3hm`-+TWDM_tP6=+t{(%%XyqG>sVf
z=^bn1S|G(#6QU4<0;i6XfGwn9kBJg8QJpEi%FZSE2}cjjI<-P}oG)4M7fz;0-)OJd
z{Jy641Yf?l1cx0;=Sty6<3ed}`wj($H>dSLh%n5@ykh+2Kxn`~T<iG3=K*bjtT?Vr
zUaHy!UBe|2M&CZTs;Wqja-8~Gj;stC={?SY1dY2g=$7L4Zbl4TgX!)VNCp7lrf+w!
zj+2eZ{a~GMJ_5@0)D+Djr0N9suXX`MOFyWi2shWd`|_xPqa2A(>ctHUoP3XOVe(_M
zWZ%UP1SEU*h(&D-bis-`6gJJ!{VaT`!^k}Cgxd{vS$_M_Co$EH2u?A+KFDN#(As!I
zs3>ag?ix)Wiib`iak__a1@ycvU2~bFwgV?VK%W$ci9WrU_Qk8Ib#o5NBNrRg>6y;9
z{jQa)9c+4(u*%j`fz4|#l)-Mwk?!?&hU@cEuaYvt8AAY4A`L-|Ki3uQ+<3(%WHawC
zhOaIctS$p;p0ay5-3i=PdDmPjOYi=4)?;*p@aa#adr3$tyK#zrMUA+7u9L+1hoAuN
z4Leum#h4aR=IC{eLPBQW@wCZ<G8GStv|<$;7wPu4R1t<%?Iahk6uS1RdEKEDUiTt6
z$;oJls?tie0ylMJp8SVF-QYd2AUaKE$HtFl#aQ>f%r>@SSdPf@(Lmi%TZMj;7P<d$
zGvSI%ENgvNVjHI(HX?yV<+~D*gF1hf&s#o(f_5L2^|8`<&r6Wpb@|c6-TjzeSl2m=
zZ@d5xEOWN)<TyeJAcSy6#IFj}k0ttqEq<TF2p>(sY@$^do-sx}1YM#S=V=n>uD!>f
zM@44LOn;sqE4omIzL0QB5@xz@_v!wJ1jHxhd2P{)>iCqSF5_IEDB9GQP>yyJ8|Gq(
zIB$o|Ximh~ylvU()|MjiyD2rRYrc^)uggKgS@j`*sy8{5+;-8*o2&wza!pBLGp~~N
zNR7hTr8n#{<Og^XFmy5lMzP>$Tb*vQyJW5w$3;fSBRzyMH9kFlQJ0GLHPMvZ_1KGI
z$Md2VquGk0YCK-yZiIP1vG|UN@Qz{g6qE6T8ft7Sku5pdzFgnMiXVY<ZTlM|@#9(G
z#7ShB9Er|HZcAFGtW=9n$QvEd8pE=yE(`Zln16RV^BlJFkKke8#o?l&g%r&8#_+7F
zeTTN2YFI<It(od(oB25%$sK-YusC1<JqW6+QVH}g_Gn&@^e8$++<)AXZJSZcB#=JU
z8E7D4d|xY}k3Qa~_=E0ZtSPFO&oQ&QzM|SF9Cuc^@Now_maYshRw@^RT@mUSvDr};
z*~MiEJlwXg4<v?r+03q(#q^-fPj5S3Wk$RDb9j5kZVl0}nz)Y+NEI{Ro8caUBSAAi
za7EodsXW{$Tc&!>;*aqF5&67!F5uY-w#l*<%daC)XBjDIw^}0qQW_eQF&-MFTS!yw
zDZe**qg-{F@JAGV`h7i~I}rmU)i>0hzL?rl8UX;JLzD--tV7kgO&I7OA|u#cLteF~
z>*>tm$@(;*COm;JlNof?2f6H(qlPE=j)W)N)h$BFg9{Rn2r#o`YtW;)nx)o^>gxh`
zQvZY2!r&Q>&GMF}<^G<5C624hYc+Ij##gM{sz><fDhR^EJry>(d)T1N^t0&8prqrZ
zoBb`NOOPiV*`TujN~7|e9!r%B6Kdv8<A}Kvf8c}294WocVQBl|Xt&chZ%{l;j1t()
zi#gg{t}8lCcBDg;t87xjS+vSu++i2Srj<UuWvd*~8AXWR7FE|W+>s*kQmqtO3M!|V
z71DsPjYc+@w&Pl%pS&yHQMr$-xRWuAqkN^ILiosJ+0>4Lw;r5BAs<OyJa#Du1ahDu
z*+N>Pyp#_T`B}=gYB6SR#$4H1Z!4=y-5`*79+6tg(&}J)?Xt`a`W4l(XKhx6KQ6l?
zZ)W0>W(4V6GsA-lKkeuKVJIVK)1&M&1s$upO7dd5KJR^`%N+gP)MBmioIk|*jv=Yj
zxlfPfp4=g0aPPC0Hi1&xpdw$svfdLPMy5jPHY%rJCemkJMQ`LkfKz3^4J!(q(L^mf
zO0LeBP=%}r<hFe3!G247@%U+DmY+e_1m_bP;ClI4htg-Ga1kxRHm8?12jAvSn6N=d
zDc*w(fx45zc;#->RvHF&Vy1;GwpLwcRhHO#$f#M~z_-V+_EhK6Z+u?j-nZCs(EJr$
zhB9-TG}suCqlwA<D#@4rf_(IMP%xpk#g1C+c5mM_MTemKMBTSh%Mtxv6J{!T6(u@-
z)E3gT*f;O%+Pm<!k*O9nMZ>}z#`W%*;ix_)Xa6dpf*wRcPgOD#SS6UjyKio9a-dOi
zr_7i62F&>smc<Gr_Jnuq%%{a+LUoQY+xvW&n-<4kSp%9(HFlVPR5we6aJ8NEW)4<P
zTCXp%wSer;LaCmY;pdN9pO0PTE-7cEA!E6y#r>QcF@~G52Y1&KW`d}99jYFU2VkmL
z!xLXP)>E)gK7*FAD-P@D%;M^Jpr7m43N0^pmo51-B-Tk~fGrDK*0d6P9u>P8d&W=x
zs0&b-0lp3cmRE=AwanJ9kZ9aDRJEjWIq}|jqmrDIb{l+4dg|JWi47Q{p>yUCU8wNj
z<I%J++TI%dXfZ>4j`b@ycX=Yk&y}N-joEYH^{J|TV%m!E=F{9Gad2gF=_JaG_*QCe
zm&LGK>wvsk-`ms#cIGGX>A^R&De+F-M+sijAj^GJEY88BW*aT9pVi`Y;5HjQOy#>u
zj`mSaje3Rp368Xap)lE;40YjF9n6ps(?$AgiOU9&fQ#XPKkM*y_82@h5&)o^{r{@N
zseaYruzy<R-&Od0%AgC=Ls|uN^khKQrt||>u6ewhyd-yT=9kkN_K8R?^;pGTf|mFF
zi9tcrI$AT&RYB+O@Jcu$mT=-02?t4V*k{gYis(IjEi-%e;_**<7!Il4^pYduA)N?;
zG4adO5{&S;(9K+0Iu<gW6q<8YdIM;2td<u?D19houIix>f8CrjMI;E>Vrkol^VY27
z^Dskp3edqhBRqa`Ci9|G_9UnFb$z1wJ(VSG{k%HrT#)z7(8n~U)S{Fkoo@4w*fy-$
zkftDU8Ma=D0Xs`yrC>aGiXyyyN+aSWyDs60RMpd;oD1L8s`FpJ(GIKW3A9fqxu<zH
zl2@m1YN_geY<lxqM)>f@{(uA}Wqt<TcQpACqFfyU5?pH=FkTc39pWL+7}ZS&r+ocU
zMsFFM#>Is_=tt=E)P_;1aDo_&j}Wcr*7EYElxwr6X(Lhj;NXKWLy9Vqx?*GQtha9l
zIU#eR@1MKStmP6ZP5>-4JP!T#m)M0R@za@lzDmqH*vW3yb~-!{pOmlzMi0x7EfJ`z
z_m!ao_f^wUvKjYtb8Q<O?ev_{H^5e>x!3maOV{oAfJqKaaqpFWtoesK=Z?vJw#htR
zQ4O!ckWOLl(SKuz-)<$;G7Z|%C%C=9c(0DJN9=DT+eWsAX*h-Te+%BrfKaDFSVHzP
zz|?8r^uSDP>HEoo_cMZ<3tL>ZTip-;#v7X{t$zJZe(o!-l|$fWYw>1D;JnPAhzGx>
z8W*SZ4gvl=Cq-!;DR3^Uz$>b$!_bZUpD;@T?*~(bjc!(w>uo%L9o(0yM{DNt1|;vu
zf!0N|nt5TbflcIT(}hp2@Xs2U5fe-7ZwnMrx8M12uh_F5?i|b^{`%p=E_8}{_yEAc
zo!>tk51$tB?{?5XUFh$2(Ep78Z*%BBS^lZ9-;Va*k@w31{}+GzpBMb|3BRTI@8GBS
XFT&K*A_D$SA^0`ce<2z5@72EmrHLnr

literal 0
HcmV?d00001

diff --git a/test/api/test_tools_upload.py b/test/api/test_tools_upload.py
index c4c27576b03..bebe1ecd3a7 100644
--- a/test/api/test_tools_upload.py
+++ b/test/api/test_tools_upload.py
@@ -219,6 +219,19 @@ class ToolsUploadTestCase(api.ApiTestCase):
             roadmaps_content = self._get_roadmaps_content(history_id, dataset)
             assert roadmaps_content.strip() == "roadmaps\ncontent", roadmaps_content
 
+    @skip_without_datatype("isa-tab")
+    def test_composite_datatype_isatab(self):
+        zip_path = TestDataResolver().get_filename("MTBLS6.zip")
+        with self.dataset_populator.test_history() as history_id:
+            payload = self.dataset_populator.upload_payload(
+                history_id,
+                "????", # content
+                file_type="zip"
+            )
+            run_response = self.dataset_populator.tools_post(payload)
+            self.dataset_populator.wait_for_tool_run(history_id, run_response)
+            datasets = run_response.json()["outputs"]
+            
     def test_upload_dbkey(self):
         with self.dataset_populator.test_history() as history_id:
             payload = self.dataset_populator.upload_payload(history_id, "Test123", dbkey="hg19")

From 08ce54fe6e0f3cc178ad41a6fa3c00177cfbfef1 Mon Sep 17 00:00:00 2001
From: djcomlab <djcomlab@gmail.com>
Date: Thu, 26 Apr 2018 11:21:51 +0100
Subject: [PATCH 18/26] Introduce new table parsers to get data files back;
 temporarily puts dependency back to Git repo to test against (updated PyPI
 package for isa-rwval will be pushed soon).

---
 lib/galaxy/datatypes/isa.py                     | 13 +++++++------
 lib/galaxy/dependencies/pinned-requirements.txt |  3 +--
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index d60e42a88f8..b9e95cec035 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -24,7 +24,7 @@ from json import dumps  # noqa: F401
 # Imports isatab after turning off warnings inside logger settings to avoid pandas warning making uploads fail.
 logging.getLogger("isatools.isatab").setLevel(logging.ERROR)
 from isatools import isajson
-from isatools import isatab
+from isatools import isatab_meta
 
 from galaxy import model
 from galaxy import util
@@ -345,15 +345,16 @@ class IsaTab(_Isa):
     def _make_investigation_instance(self, filename):
 
         # Parse ISA-Tab investigation file
-        parser = isatab.InvestigationParser()
+        parser = isatab_meta.InvestigationParser()
+        isa_dir = os.path.dirname(filename)
         fp = utf8_text_file_open(filename)
         parser.parse(fp)
         for study in parser.isa.studies:
-            s_parser = isatab.LazyStudySampleTableParser(parser.isa)
-            s_parser.parse(study.filename)
+            s_parser = isatab_meta.LazyStudySampleTableParser(parser.isa)
+            s_parser.parse(os.path.join(isa_dir, study.filename))
             for assay in study.assays:
-                a_parser = isatab.LazyAssayTableParser(parser.isa)
-                a_parser.parse(assay.filename)
+                a_parser = isatab_meta.LazyAssayTableParser(parser.isa)
+                a_parser.parse(os.path.join(isa_dir, assay.filename))
         isa = parser.isa
 
         return isa
diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
index f812d0246da..f55089254e1 100644
--- a/lib/galaxy/dependencies/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -72,7 +72,7 @@ pulsar-galaxy-lib==0.8.3
 
 # sqlalchemy-migrate and dependencies
 sqlalchemy-migrate==0.11.0
-decorator==4.0.2
+decorator==4.1.0
 Tempita==0.5.3dev
 sqlparse==0.1.16
 pbr==3.1.1
@@ -91,4 +91,3 @@ python-genomespaceclient==0.1.8
 
 # ISA
 git+https://github.com/ISA-tools/isa-rwval.git@develop#egg=isa-rwval
-

From eac29d55f9567b68ca94b3c2e468c380a64261ba Mon Sep 17 00:00:00 2001
From: djcomlab <djcomlab@gmail.com>
Date: Thu, 26 Apr 2018 12:31:39 +0100
Subject: [PATCH 19/26] Fix requirements again for isa-rwval==0.10.1

---
 lib/galaxy/dependencies/pinned-requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
index f55089254e1..c310dffa6af 100644
--- a/lib/galaxy/dependencies/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -90,4 +90,4 @@ ecdsa==0.13
 python-genomespaceclient==0.1.8
 
 # ISA
-git+https://github.com/ISA-tools/isa-rwval.git@develop#egg=isa-rwval
+isa-rwval==0.10.1

From b4fa3d28064e1c527a3b1a3232d10873490b6825 Mon Sep 17 00:00:00 2001
From: djcomlab <djcomlab@gmail.com>
Date: Thu, 26 Apr 2018 15:00:10 +0100
Subject: [PATCH 20/26] Cleanup presentation of data files (unused rendering
 using datafile id)

---
 lib/galaxy/datatypes/isa.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index b9e95cec035..498da9d2db9 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -303,7 +303,7 @@ class _Isa(data.Data):
                 html += '<p>Submitted the %s</p>' % study.submission_date
                 html += '<p>Released on %s</p>' % study.public_release_date
 
-                html += '<p>Factors %s</p>' % ', '.join([x.name for x in study.factors])
+                html += '<p>Experimental factors used: %s</p>' % ', '.join([x.name for x in study.factors])
 
                 # Loop on all assays of this study
                 for assay in study.assays:
@@ -315,7 +315,7 @@ class _Isa(data.Data):
                         html += '<p>Data files:</p>'
                         html += '<ul>'
                         for data_file in assay.data_files:
-                            html += '<li>' + escape(util.unicodify(str(data_file.id), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.filename), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.label), 'utf-8')) + '</li>'
+                            html += '<li>' + escape(util.unicodify(str(data_file.filename), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.label), 'utf-8')) + '</li>'
                         html += '</ul>'
 
             html += '</body></html>'

From 832f51b5168cc9751eecb77a62ff7a06807f92ed Mon Sep 17 00:00:00 2001
From: Pierrick Roger <pk.roger@icloud.com>
Date: Thu, 7 Jun 2018 12:09:11 +0100
Subject: [PATCH 21/26] Pinned requirements as a symbolic link as done in dev
 branch.

---
 .../dependencies/pinned-requirements.txt      | 94 +------------------
 1 file changed, 1 insertion(+), 93 deletions(-)
 mode change 100644 => 120000 lib/galaxy/dependencies/pinned-requirements.txt

diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
deleted file mode 100644
index c310dffa6af..00000000000
--- a/lib/galaxy/dependencies/pinned-requirements.txt
+++ /dev/null
@@ -1,93 +0,0 @@
-# bx-python and dependencies
-numpy==1.9.2
-python-lzo==1.11
-bx-python==0.8.1
-
-# packages with C extensions
-MarkupSafe==1.0
-PyYAML==3.12
-SQLAlchemy==1.2.2
-sqlalchemy-utils==0.32.19
-mercurial==3.7.3; python_version < '3.0'
-pycrypto==2.6.1
-uWSGI==2.0.15
-# Flexible BAM index naming is new to core pysam
-pysam==0.14
-
-# pure Python packages
-bdbag==1.1.1
-bleach==2.1.3
-bz2file==0.98; python_version < '3.3'
-ipaddress==1.0.18; python_version < '3.3'
-boltons==17.1.0
-Paste==2.0.3
-PasteDeploy==1.5.2
-docutils==0.14
-repoze.lru==0.6
-Routes==2.4.1
-WebOb==1.4.1
-Mako==1.0.2
-pytz==2017.3
-Babel==2.5.1
-Beaker==1.7.0
-dictobj==0.3.1
-nose==1.3.7
-Parsley==1.3
-six==1.11.0
-Whoosh==2.7.4
-galaxy_sequence_utils==1.0.2
-h5py==2.7.1
-packaging==16.8
-
-# pykwalify and dependencies
-pykwalify==1.6.0
-python-dateutil==2.5.3
-docopt==0.6.2
-
-# Cheetah and dependencies
-Cheetah3==3.0.0
-Markdown==2.6.3
-
-# Requests and dependencies
-requests==2.18.4
-certifi==2017.7.27.1
-urllib3==1.22
-chardet==3.0.4
-idna==2.6
-
-# BioBlend and dependencies
-bioblend==0.7.0
-boto==2.38.0
-requests-toolbelt==0.8.0
-
-
-# kombu and dependencies
-kombu==3.0.30
-amqp==1.4.8
-anyjson==0.3.3
-
-# Pulsar requirements
-psutil==4.1.0
-pulsar-galaxy-lib==0.8.3
-
-# sqlalchemy-migrate and dependencies
-sqlalchemy-migrate==0.11.0
-decorator==4.1.0
-Tempita==0.5.3dev
-sqlparse==0.1.16
-pbr==3.1.1
-
-# svgwrite and dependencies
-svgwrite==1.1.6
-pyparsing==2.1.1
-
-# Fabric and dependencies
-Fabric3==1.13.1.post1
-paramiko==2.2.1
-ecdsa==0.13
-
-# GenomeSpace dependencies
-python-genomespaceclient==0.1.8
-
-# ISA
-isa-rwval==0.10.1
diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
new file mode 120000
index 00000000000..26ec2b5e875
--- /dev/null
+++ b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -0,0 +1 @@
+pipfiles/default/pinned-requirements.txt
\ No newline at end of file

From 39d4447a97ab8fe8359d3fd2a1df2815014570ea Mon Sep 17 00:00:00 2001
From: djcomlab <djcomlab@gmail.com>
Date: Thu, 7 Jun 2018 13:02:43 +0100
Subject: [PATCH 22/26] Add check so that empty filenames not added to file
 list.

---
 lib/galaxy/datatypes/isa.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index 498da9d2db9..f39c24e4b74 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -315,7 +315,8 @@ class _Isa(data.Data):
                         html += '<p>Data files:</p>'
                         html += '<ul>'
                         for data_file in assay.data_files:
-                            html += '<li>' + escape(util.unicodify(str(data_file.filename), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.label), 'utf-8')) + '</li>'
+                            if data_file.filename != '':
+                                html += '<li>' + escape(util.unicodify(str(data_file.filename), 'utf-8')) + ' - ' + escape(util.unicodify(str(data_file.label), 'utf-8')) + '</li>'
                         html += '</ul>'
 
             html += '</body></html>'

From a98cc55db25492217ff5cb6d91716122e39347fb Mon Sep 17 00:00:00 2001
From: djcomlab <djcomlab@gmail.com>
Date: Fri, 29 Jun 2018 07:15:46 -0700
Subject: [PATCH 23/26] Fix upload test for isa-tab datatype

---
 test/api/test_tools_upload.py | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/test/api/test_tools_upload.py b/test/api/test_tools_upload.py
index bebe1ecd3a7..be0a6bbb7ce 100644
--- a/test/api/test_tools_upload.py
+++ b/test/api/test_tools_upload.py
@@ -221,16 +221,11 @@ class ToolsUploadTestCase(api.ApiTestCase):
 
     @skip_without_datatype("isa-tab")
     def test_composite_datatype_isatab(self):
-        zip_path = TestDataResolver().get_filename("MTBLS6.zip")
-        with self.dataset_populator.test_history() as history_id:
-            payload = self.dataset_populator.upload_payload(
-                history_id,
-                "????", # content
-                file_type="zip"
-            )
-            run_response = self.dataset_populator.tools_post(payload)
-            self.dataset_populator.wait_for_tool_run(history_id, run_response)
-            datasets = run_response.json()["outputs"]
+        isatab_zip_path = TestDataResolver().get_filename("MTBLS6.zip")
+        details = self._upload_and_get_details(open(isatab_zip_path, "rb"), file_type="isa-tab")
+        assert details["state"] == "ok"
+        assert details["file_ext"] == "isa-tab", details
+        assert details["file_size"] == 85, details
             
     def test_upload_dbkey(self):
         with self.dataset_populator.test_history() as history_id:

From f2176d395478fe4c4ad41ae3f03747b3967abc6d Mon Sep 17 00:00:00 2001
From: djcomlab <djcomlab@gmail.com>
Date: Fri, 29 Jun 2018 07:18:14 -0700
Subject: [PATCH 24/26] Update isa-rwval version as new update fixes a bug

---
 .../dependencies/pipfiles/default/pinned-requirements.txt       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
index 6af205df7b6..62c276000f3 100644
--- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
@@ -41,7 +41,7 @@ h5py==2.6.0
 html5lib==1.0.1
 idna==2.6
 ipaddress==1.0.19; python_version < '3.3'
-isa-rwval==0.10.3
+isa-rwval==0.10.4
 iso8601==0.1.12
 jmespath==0.9.3
 jsonpatch==1.23

From 35093e38fcfe4f9efc3a2914d694cf4a11f24ace Mon Sep 17 00:00:00 2001
From: Pierrick ROGER <pk.roger@icloud.com>
Date: Thu, 18 Oct 2018 11:16:29 +0200
Subject: [PATCH 25/26] Remove spaces in blank line.

---
 test/api/test_tools_upload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/api/test_tools_upload.py b/test/api/test_tools_upload.py
index c5eb868114f..c8296c4df80 100644
--- a/test/api/test_tools_upload.py
+++ b/test/api/test_tools_upload.py
@@ -247,7 +247,7 @@ class ToolsUploadTestCase(api.ApiTestCase):
         assert details["state"] == "ok"
         assert details["file_ext"] == "isa-tab", details
         assert details["file_size"] == 85, details
-            
+
     def test_upload_dbkey(self):
         with self.dataset_populator.test_history() as history_id:
             payload = self.dataset_populator.upload_payload(history_id, "Test123", dbkey="hg19")

From a9eb587a3e38849c5044700f9f01c47d53f55a87 Mon Sep 17 00:00:00 2001
From: Pierrick ROGER <pk.roger@icloud.com>
Date: Thu, 18 Oct 2018 11:37:04 +0200
Subject: [PATCH 26/26] Various corrections for py34-lint

Add second blank line.
Remove useless imports.
---
 lib/galaxy/datatypes/isa.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py
index f39c24e4b74..7b1cfa6f44a 100644
--- a/lib/galaxy/datatypes/isa.py
+++ b/lib/galaxy/datatypes/isa.py
@@ -14,11 +14,8 @@ import os.path
 import re
 import shutil
 import sys
-import tarfile
 import tempfile
-import zipfile
 from cgi import escape
-from io import BytesIO
 from json import dumps  # noqa: F401
 
 # Imports isatab after turning off warnings inside logger settings to avoid pandas warning making uploads fail.
@@ -53,6 +50,7 @@ logger = logging.getLogger(__name__)
 # Function for opening correctly a CSV file for csv.reader() for both Python 2 and 3 {{{1
 ################################################################
 
+
 def utf8_text_file_open(path):
     if sys.version_info[0] < 3:
         fp = open(path, 'rb')