Merge pull request #5108 from mvdbeek/merge_forward_17_09

Merge forward 17 09

config/datatypes_conf.xml.sample

@@ -498,6 +498,7 @@
     <!-- Annotation Datatypes -->
     <datatype extension="snaphmm" type="galaxy.datatypes.annotation:SnapHmm" display_in_upload="true" />
     <datatype extension="augustus" type="galaxy.datatypes.annotation:Augustus" display_in_upload="true" />
+    <datatype extension="icm" type="galaxy.datatypes.binary:ICM" display_in_upload="true"/>
     <!-- MSA Datatypes -->
     <datatype extension="hmm2" type="galaxy.datatypes.msa:Hmmer2" display_in_upload="true" />
     <datatype extension="hmm3" type="galaxy.datatypes.msa:Hmmer3" display_in_upload="true" />
@@ -704,6 +705,7 @@
     <sniffer type="galaxy.datatypes.binary:Fast5ArchiveBz2" />
     <sniffer type="galaxy.datatypes.binary:Fast5Archive" />
     <sniffer type="galaxy.datatypes.binary:PostgresqlArchive"/>
+    <sniffer type="galaxy.datatypes.annotation:ICM" />
     <sniffer type="galaxy.datatypes.annotation:Augustus" />
     <sniffer type="galaxy.datatypes.triples:Rdf"/>
     <sniffer type="galaxy.datatypes.blast:BlastXml"/>

lib/galaxy/datatypes/binary.py

@@ -1793,6 +1793,29 @@ class DMND(Binary):
             return False
 
 
+class ICM(Binary):
+    """
+    Class describing an ICM (interpolated context model) file, used by Glimmer
+    """
+    file_ext = "icm"
+    edam_data = "data_0950"
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        if not dataset.dataset.purged:
+            dataset.peek = "Binary ICM (interpolated context model) file"
+            dataset.blurb = nice_size(dataset.get_size())
+        else:
+            dataset.peek = 'file does not exist'
+            dataset.blurb = 'file purged from disk'
+
+    def sniff(self, dataset):
+        line = open(dataset).read(100)
+        if '>ver = ' in line and 'len = ' in line and 'depth = ' in line and 'periodicity =' in line and 'nodes = ' in line:
+            return True
+
+        return False
+
+
 if __name__ == '__main__':
     import doctest
     doctest.testmod(sys.modules[__name__])

lib/galaxy/datatypes/mothur.py

@@ -26,6 +26,24 @@ class Otu(Text):
         super(Otu, self).__init__(**kwd)
 
     def set_meta(self, dataset, overwrite=True, **kwd):
+        """
+        Set metadata for Otu files.
+
+        >>> from galaxy.datatypes.sniff import get_test_fname
+        >>> from galaxy.util.bunch import Bunch
+        >>> dataset = Bunch()
+        >>> dataset.metadata = Bunch
+        >>> otu = Otu()
+        >>> dataset.file_name = get_test_fname( 'mothur_datatypetest_true.mothur.otu' )
+        >>> dataset.has_data = lambda: True
+        >>> otu.set_meta(dataset)
+        >>> dataset.metadata.columns
+        100
+        >>> len(dataset.metadata.labels) == 37
+        True
+        >>> len(dataset.metadata.otulabels) == 98
+        True
+        """
         super(Otu, self).set_meta(dataset, overwrite=overwrite, **kwd)
 
         if dataset.has_data():
@@ -37,6 +55,8 @@ class Otu(Text):
 
             headers = iter_headers(dataset.file_name, sep='\t', count=-1)
             first_line = get_headers(dataset.file_name, sep='\t', count=1)
+            if first_line:
+                first_line = first_line[0]
             # set otulabels
             if len(first_line) > 2:
                 otulabel_names = first_line[2:]

lib/galaxy/jobs/runners/condor.py

@@ -210,7 +210,7 @@ class CondorJobRunner(AsynchronousJobRunner):
         external_id = job.job_runner_external_id
         failure_message = condor_stop(external_id)
         if failure_message:
-            log.debug("(%s/%s). Failed to stop condor %s" % (external_id, failure_message))
+            log.debug("(%s). Failed to stop condor %s" % (external_id, failure_message))
 
     def recover(self, job, job_wrapper):
         """Recovers jobs stuck in the queued/running state when Galaxy started"""

lib/galaxy/tools/actions/__init__.py

@@ -257,7 +257,7 @@ class DefaultToolAction(object):
         # Add the dbkey to the incoming parameters
         incoming["dbkey"] = input_dbkey
         # wrapped params are used by change_format action and by output.label; only perform this wrapping once, as needed
-        wrapped_params = self._wrapped_params(trans, tool, incoming)
+        wrapped_params = self._wrapped_params(trans, tool, incoming, inp_data)
 
         out_data = odict()
         input_collections = dict((k, v[0][0]) for k, v in inp_dataset_collections.items())
@@ -518,8 +518,8 @@ class DefaultToolAction(object):
             trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id)
             return job, out_data
 
-    def _wrapped_params(self, trans, tool, incoming):
-        wrapped_params = WrappedParameters(trans, tool, incoming)
+    def _wrapped_params(self, trans, tool, incoming, input_datasets=None):
+        wrapped_params = WrappedParameters(trans, tool, incoming, input_datasets=input_datasets)
         return wrapped_params
 
     def _get_on_text(self, inp_data):

lib/galaxy/tools/error_reports/plugins/sentry.py

@@ -65,8 +65,7 @@ class SentryPlugin(ErrorPlugin):
                 extra['email'] = unicodify(kwargs['email'])
 
             # User submitted message
-            if 'message' in kwargs:
-                extra['message'] = unicodify(kwargs['message'])
+            extra['message'] = unicodify(kwargs.get('message', ''))
 
             # Construct the error message to send to sentry. The first line
             # will be the issue title, everything after that becomes the

lib/galaxy/tools/evaluation.py

@@ -26,6 +26,7 @@ from galaxy.tools.wrappers import (
     DatasetCollectionWrapper,
     DatasetFilenameWrapper,
     DatasetListWrapper,
+    ElementIdentifierMapper,
     InputValueWrapper,
     RawObjectWrapper,
     SelectToolParameterWrapper,
@@ -233,11 +234,9 @@ class ToolEvaluator(object):
                     real_path = dataset.file_name
                     if real_path in input_dataset_paths:
                         wrapper_kwds["dataset_path"] = input_dataset_paths[real_path]
-                identifier_key = identifier_key_dict.get(dataset, None)
-                if identifier_key:
-                    element_identifier = param_dict.get(identifier_key, None)
-                    if element_identifier:
-                        wrapper_kwds["identifier"] = element_identifier
+                element_identifier = element_identifier_mapper.identifier(dataset, param_dict)
+                if element_identifier:
+                    wrapper_kwds["identifier"] = element_identifier
                 input_values[input.name] = \
                     DatasetFilenameWrapper(dataset, **wrapper_kwds)
             elif isinstance(input, DataCollectionToolParameter):
@@ -265,7 +264,7 @@ class ToolEvaluator(object):
         #       tools where the inputs don't even get passed through. These
         #       tools (e.g. UCSC) should really be handled in a special way.
         if self.tool.check_values:
-            identifier_key_dict = dict((v, "%s|__identifier__" % k) for k, v in input_datasets.items())  # allows lookup of identifier through HDA.
+            element_identifier_mapper = ElementIdentifierMapper(input_datasets)
             self.__walk_inputs(self.tool.inputs, param_dict, wrap_input)
 
     def __populate_input_dataset_wrappers(self, param_dict, input_datasets, input_dataset_paths):

lib/galaxy/tools/parameters/__init__.py

@@ -41,26 +41,78 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre
     >>> g = BooleanToolParameter(None, XML('<param name="g"/>'))
     >>> h = TextToolParameter(None, XML('<param name="h"/>'))
     >>> i = TextToolParameter(None, XML('<param name="i"/>'))
-    >>> b.name = 'b'
-    >>> b.inputs = odict([('c', c), ('d', d)])
-    >>> d.name = 'd'
-    >>> d.inputs = odict([('e', e), ('f', f)])
+    >>> j = TextToolParameter(None, XML('<param name="j"/>'))
+    >>> b.name = b.title = 'b'
+    >>> b.inputs = odict([ ('c', c), ('d', d) ])
+    >>> d.name = d.title = 'd'
+    >>> d.inputs = odict([ ('e', e), ('f', f) ])
     >>> f.test_param = g
     >>> f.name = 'f'
-    >>> f.cases = [Bunch(value='true', inputs= { 'h': h }), Bunch(value='false', inputs= { 'i': i })]
+    >>> f.cases = [Bunch(value='true', inputs= {'h': h}), Bunch(value='false', inputs= { 'i': i })]
     >>>
-    >>> def visitor(input, value, prefix, prefixed_name, **kwargs):
-    ...     print('name=%s, prefix=%s, prefixed_name=%s, value=%s' % (input.name, prefix, prefixed_name, value))
-    >>> inputs = odict([('a',a),('b',b)])
-    >>> nested = odict([('a', 1), ('b', [odict([('c', 3), ('d', [odict([('e', 5), ('f', odict([('g', True), ('h', 7)]))])])])])])
+    >>> def visitor(input, value, prefix, prefixed_name, prefixed_label, error, **kwargs):
+    ...     print('name=%s, prefix=%s, prefixed_name=%s, prefixed_label=%s, value=%s' % (input.name, prefix, prefixed_name, prefixed_label, value))
+    ...     if error:
+    ...         print(error)
+    >>> inputs = odict([('a', a),('b', b)])
+    >>> nested = odict([('a', 1), ('b', [odict([('c', 3), ('d', [odict([ ('e', 5), ('f', odict([ ('g', True), ('h', 7)]))])])])])])
     >>> visit_input_values(inputs, nested, visitor)
-    name=a, prefix=, prefixed_name=a, value=1
-    name=c, prefix=b_0|, prefixed_name=b_0|c, value=3
-    name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, value=5
-    name=g, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|g, value=True
-    name=h, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|h, value=7
+    name=a, prefix=, prefixed_name=a, prefixed_label=a, value=1
+    name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=3
+    name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e, value=5
+    name=g, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|g, prefixed_label=b 1 > d 1 > g, value=True
+    name=h, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|h, prefixed_label=b 1 > d 1 > h, value=7
     >>> params_from_strings(inputs, params_to_strings(inputs, nested, None), None)['b'][0]['d'][0]['f']['g'] is True
     True
+
+    >>> # Conditional test parameter value does not match any case, warning is shown and child values are not visited
+    >>> f.test_param = j
+    >>> nested['b'][0]['d'][0]['f']['j'] = 'j'
+    >>> visit_input_values(inputs, nested, visitor)
+    name=a, prefix=, prefixed_name=a, prefixed_label=a, value=1
+    name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=3
+    name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e, value=5
+    name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|j, prefixed_label=b 1 > d 1 > j, value=j
+    The selected case is unavailable/invalid.
+
+    >>> # Test parameter missing in state, value error
+    >>> del nested['b'][0]['d'][0]['f']['j']
+    >>> visit_input_values(inputs, nested, visitor)
+    name=a, prefix=, prefixed_name=a, prefixed_label=a, value=1
+    name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=3
+    name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e, value=5
+    name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|j, prefixed_label=b 1 > d 1 > j, value=None
+    No value found for 'b 1 > d 1 > j'.
+
+    >>> # Conditional parameter missing in state, value error
+    >>> del nested['b'][0]['d'][0]['f']
+    >>> visit_input_values(inputs, nested, visitor)
+    name=a, prefix=, prefixed_name=a, prefixed_label=a, value=1
+    name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=3
+    name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e, value=5
+    name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|j, prefixed_label=b 1 > d 1 > j, value=None
+    No value found for 'b 1 > d 1 > j'.
+
+    >>> # Conditional input name has changed e.g. due to tool changes, key error
+    >>> f.name = 'f_1'
+    >>> visit_input_values(inputs, nested, visitor)
+    name=a, prefix=, prefixed_name=a, prefixed_label=a, value=1
+    name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=3
+    name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e, value=5
+    name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f_1|j, prefixed_label=b 1 > d 1 > j, value=None
+    No value found for 'b 1 > d 1 > j'.
+
+    >>> # Other parameters are missing in state
+    >>> nested = odict([('b', [odict([( 'd', [odict([('f', odict([('g', True), ('h', 7)]))])])])])])
+    >>> visit_input_values(inputs, nested, visitor)
+    name=a, prefix=, prefixed_name=a, prefixed_label=a, value=None
+    No value found for 'a'.
+    name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=None
+    No value found for 'b 1 > c'.
+    name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e, value=None
+    No value found for 'b 1 > d 1 > e'.
+    name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f_1|j, prefixed_label=b 1 > d 1 > j, value=None
+    No value found for 'b 1 > d 1 > j'.
     """
     def callback_helper(input, input_values, name_prefix, label_prefix, parent_prefix, context=None, error=None):
         args = {
@@ -83,6 +135,12 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre
         if replace:
             input_values[input.name] = new_value
 
+    def get_current_case(input, input_values):
+        try:
+            return input.get_current_case(input_values[input.test_param.name])
+        except (KeyError, ValueError):
+            return -1
+
     context = ExpressionContext(input_values, context)
     payload = {'context': context, 'no_replacement_value': no_replacement_value}
     for input in inputs.values():
@@ -96,15 +154,11 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre
         elif isinstance(input, Conditional):
             values = input_values[input.name] = input_values.get(input.name, {})
             new_name_prefix = name_prefix + input.name + '|'
-            case_error = None
-            try:
-                input.get_current_case(values[input.test_param.name])
-            except Exception:
-                case_error = 'The selected case is unavailable/invalid.'
-                pass
+            case_error = None if get_current_case(input, values) >= 0 else 'The selected case is unavailable/invalid.'
             callback_helper(input.test_param, values, new_name_prefix, label_prefix, parent_prefix=name_prefix, context=context, error=case_error)
-            values['__current_case__'] = input.get_current_case(values[input.test_param.name])
-            visit_input_values(input.cases[values['__current_case__']].inputs, values, callback, new_name_prefix, label_prefix, parent_prefix=name_prefix, **payload)
+            values['__current_case__'] = get_current_case(input, values)
+            if values['__current_case__'] >= 0:
+                visit_input_values(input.cases[values['__current_case__']].inputs, values, callback, new_name_prefix, label_prefix, parent_prefix=name_prefix, **payload)
         elif isinstance(input, Section):
             values = input_values[input.name] = input_values.get(input.name, {})
             new_name_prefix = name_prefix + input.name + '|'

lib/galaxy/tools/parameters/wrapped.py

@@ -12,6 +12,7 @@ from galaxy.tools.wrappers import (
     DatasetCollectionWrapper,
     DatasetFilenameWrapper,
     DatasetListWrapper,
+    ElementIdentifierMapper,
     InputValueWrapper,
     SelectToolParameterWrapper
 )
@@ -21,11 +22,12 @@ PARAMS_UNWRAPPED = object()
 
 class WrappedParameters(object):
 
-    def __init__(self, trans, tool, incoming):
+    def __init__(self, trans, tool, incoming, input_datasets=None):
         self.trans = trans
         self.tool = tool
         self.incoming = incoming
         self._params = PARAMS_UNWRAPPED
+        self._input_datasets = input_datasets
 
     @property
     def params(self):
@@ -40,6 +42,8 @@ class WrappedParameters(object):
         tool = self.tool
         incoming = self.incoming
 
+        element_identifier_mapper = ElementIdentifierMapper(self._input_datasets)
+
         # Wrap tool inputs as necessary
         for input in inputs.values():
             if input.name not in input_values and skip_missing_values:
@@ -64,11 +68,16 @@ class WrappedParameters(object):
                                        tool=tool,
                                        name=input.name)
             elif isinstance(input, DataToolParameter):
-                input_values[input.name] = \
-                    DatasetFilenameWrapper(value,
-                                           datatypes_registry=trans.app.datatypes_registry,
-                                           tool=tool,
-                                           name=input.name)
+                wrapper_kwds = dict(
+                    datatypes_registry=trans.app.datatypes_registry,
+                    tool=tool,
+                    name=input.name
+                )
+                element_identifier = element_identifier_mapper.identifier(value, input_values)
+                if element_identifier:
+                    wrapper_kwds["identifier"] = element_identifier
+
+                input_values[input.name] = DatasetFilenameWrapper(value, **wrapper_kwds)
             elif isinstance(input, SelectToolParameter):
                 input_values[input.name] = SelectToolParameterWrapper(input, input_values[input.name], other_values=incoming)
             elif isinstance(input, DataCollectionToolParameter):

lib/galaxy/tools/toolbox/integrated_panel.py

@@ -103,4 +103,9 @@ class ManagesIntegratedToolPanelMixin:
             shutil.copy(filename, filename + ".copy")
             filename = filename + ".copy"
         shutil.move(filename, destination)
-        os.chmod(self._integrated_tool_panel_config, 0o644)
+        try:
+            os.chmod(destination, 0o644)
+        except OSError:
+            # That can happen if multiple threads are simultaneously moving/chmod'ing this file
+            # Should be harmless, though this race condition should be avoided.
+            pass

lib/galaxy/tools/wrappers.py

@@ -430,3 +430,21 @@ class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets):
         # not specified or if resulting collection is empty.
         return self.__input_supplied and bool(self.__element_instance_list)
     __nonzero__ = __bool__
+
+
+class ElementIdentifierMapper(object):
+    """Track mapping of dataset collection elements datasets to element identifiers."""
+
+    def __init__(self, input_datasets=None):
+        if input_datasets is not None:
+            self.identifier_key_dict = dict((v, "%s|__identifier__" % k) for k, v in input_datasets.items())
+        else:
+            self.identifier_key_dict = {}
+
+    def identifier(self, dataset_value, input_values):
+        identifier_key = self.identifier_key_dict.get(dataset_value, None)
+        element_identifier = None
+        if identifier_key:
+            element_identifier = input_values.get(identifier_key, None)
+
+        return element_identifier

lib/galaxy/work/context.py

@@ -28,8 +28,6 @@ class WorkRequestContext(ProvidesAppContext, ProvidesUserContext, ProvidesHistor
         self.workflow_building_mode = workflow_building_mode
 
     def get_history(self, create=False):
-        if create:
-            raise NotImplementedError("Cannot create histories from a work request context.")
         return self.__history
 
     def set_history(self):

test/api/test_tools.py

@@ -969,6 +969,36 @@ class ToolsTestCase(api.ApiTestCase):
         output1_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output1)
         self.assertEquals(output1_content.strip(), '\n'.join([d['name'] for d in element_identifiers]))
 
+    @skip_without_tool("identifier_in_actions")
+    def test_identifier_in_actions(self):
+        history_id = self.dataset_populator.new_history()
+
+        element_identifiers = self.dataset_collection_populator.list_identifiers(history_id, contents=["1\t2"])
+
+        payload = dict(
+            instance_type="history",
+            history_id=history_id,
+            element_identifiers=json.dumps(element_identifiers),
+            collection_type="list",
+        )
+
+        create_response = self._post("dataset_collections", payload)
+        dataset_collection = create_response.json()
+
+        inputs = {
+            "input": {'batch': True, 'values': [{'src': 'hdca', 'id': dataset_collection['id']}]},
+        }
+
+        self.dataset_populator.wait_for_history(history_id, assert_ok=True)
+        create_response = self._run("identifier_in_actions", history_id, inputs)
+        self._assert_status_code_is(create_response, 200)
+        create = create_response.json()
+        outputs = create['outputs']
+        output1 = outputs[0]
+
+        output_details = self.dataset_populator.get_history_dataset_details(history_id, dataset=output1)
+        assert output_details["metadata_column_names"][1] == "data1", output_details
+
     @skip_without_tool("cat1")
     def test_map_over_nested_collections(self):
         history_id = self.dataset_populator.new_history()

test/functional/tools/identifier_in_actions.xml

@@ -0,0 +1,23 @@
+<tool id="identifier_in_actions" name="identifier_in_actions" version="1.0.0">
+    <command><![CDATA[
+        cp '$input' '$output'
+    ]]></command>
+    <inputs>
+        <param name="input" type="data" multiple="false" />
+    </inputs>
+    <outputs>
+        <data format="tabular" name="output">
+            <actions>
+                <action name="column_names" type="metadata" default="First,${input.element_identifier}" />
+            </actions>
+        </data>
+    </outputs>
+    <tests>
+        <test>
+            <param name="input" value="2.tabular" />
+            <output name="output">
+                <metadata name="column_names" value="First,2.tabular"/>
+            </output>
+        </test>
+    </tests>
+</tool>

test/functional/tools/samples_tool_conf.xml

@@ -95,6 +95,7 @@
   <tool file="identifier_multiple_in_conditional.xml" />
   <tool file="identifier_multiple_in_repeat.xml" />
   <tool file="identifier_collection.xml" />
+  <tool file="identifier_in_actions.xml" />
   <tool file="tool_directory.xml" />
   <tool file="output_action_change_format.xml" />
   <tool file="collection_paired_test.xml" />