diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample
index 1e90f112bf5..d471a03b207 100644
--- a/config/datatypes_conf.xml.sample
+++ b/config/datatypes_conf.xml.sample
@@ -498,6 +498,7 @@
+
@@ -704,6 +705,7 @@
+
diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py
index 3d7703f4280..71e3f12a32e 100644
--- a/lib/galaxy/datatypes/binary.py
+++ b/lib/galaxy/datatypes/binary.py
@@ -1793,6 +1793,29 @@ class DMND(Binary):
return False
+class ICM(Binary):
+ """
+ Class describing an ICM (interpolated context model) file, used by Glimmer
+ """
+ file_ext = "icm"
+ edam_data = "data_0950"
+
+ def set_peek(self, dataset, is_multi_byte=False):
+ if not dataset.dataset.purged:
+ dataset.peek = "Binary ICM (interpolated context model) file"
+ dataset.blurb = nice_size(dataset.get_size())
+ else:
+ dataset.peek = 'file does not exist'
+ dataset.blurb = 'file purged from disk'
+
+ def sniff(self, dataset):
+ line = open(dataset).read(100)
+ if '>ver = ' in line and 'len = ' in line and 'depth = ' in line and 'periodicity =' in line and 'nodes = ' in line:
+ return True
+
+ return False
+
+
if __name__ == '__main__':
import doctest
doctest.testmod(sys.modules[__name__])
diff --git a/lib/galaxy/datatypes/mothur.py b/lib/galaxy/datatypes/mothur.py
index c13b7d2adc9..89fa0d0154a 100644
--- a/lib/galaxy/datatypes/mothur.py
+++ b/lib/galaxy/datatypes/mothur.py
@@ -26,6 +26,24 @@ class Otu(Text):
super(Otu, self).__init__(**kwd)
def set_meta(self, dataset, overwrite=True, **kwd):
+ """
+ Set metadata for Otu files.
+
+ >>> from galaxy.datatypes.sniff import get_test_fname
+ >>> from galaxy.util.bunch import Bunch
+ >>> dataset = Bunch()
+ >>> dataset.metadata = Bunch
+ >>> otu = Otu()
+ >>> dataset.file_name = get_test_fname( 'mothur_datatypetest_true.mothur.otu' )
+ >>> dataset.has_data = lambda: True
+ >>> otu.set_meta(dataset)
+ >>> dataset.metadata.columns
+ 100
+ >>> len(dataset.metadata.labels) == 37
+ True
+ >>> len(dataset.metadata.otulabels) == 98
+ True
+ """
super(Otu, self).set_meta(dataset, overwrite=overwrite, **kwd)
if dataset.has_data():
@@ -37,6 +55,8 @@ class Otu(Text):
headers = iter_headers(dataset.file_name, sep='\t', count=-1)
first_line = get_headers(dataset.file_name, sep='\t', count=1)
+ if first_line:
+ first_line = first_line[0]
# set otulabels
if len(first_line) > 2:
otulabel_names = first_line[2:]
diff --git a/lib/galaxy/jobs/runners/condor.py b/lib/galaxy/jobs/runners/condor.py
index 5217034c7f6..b8c49c499a6 100644
--- a/lib/galaxy/jobs/runners/condor.py
+++ b/lib/galaxy/jobs/runners/condor.py
@@ -210,7 +210,7 @@ class CondorJobRunner(AsynchronousJobRunner):
external_id = job.job_runner_external_id
failure_message = condor_stop(external_id)
if failure_message:
- log.debug("(%s/%s). Failed to stop condor %s" % (external_id, failure_message))
+ log.debug("(%s). Failed to stop condor %s" % (external_id, failure_message))
def recover(self, job, job_wrapper):
"""Recovers jobs stuck in the queued/running state when Galaxy started"""
diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py
index bfc467fb727..b6f7e52fe3e 100644
--- a/lib/galaxy/tools/actions/__init__.py
+++ b/lib/galaxy/tools/actions/__init__.py
@@ -257,7 +257,7 @@ class DefaultToolAction(object):
# Add the dbkey to the incoming parameters
incoming["dbkey"] = input_dbkey
# wrapped params are used by change_format action and by output.label; only perform this wrapping once, as needed
- wrapped_params = self._wrapped_params(trans, tool, incoming)
+ wrapped_params = self._wrapped_params(trans, tool, incoming, inp_data)
out_data = odict()
input_collections = dict((k, v[0][0]) for k, v in inp_dataset_collections.items())
@@ -518,8 +518,8 @@ class DefaultToolAction(object):
trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id)
return job, out_data
- def _wrapped_params(self, trans, tool, incoming):
- wrapped_params = WrappedParameters(trans, tool, incoming)
+ def _wrapped_params(self, trans, tool, incoming, input_datasets=None):
+ wrapped_params = WrappedParameters(trans, tool, incoming, input_datasets=input_datasets)
return wrapped_params
def _get_on_text(self, inp_data):
diff --git a/lib/galaxy/tools/error_reports/plugins/sentry.py b/lib/galaxy/tools/error_reports/plugins/sentry.py
index 00dd96ca719..c686d01a469 100644
--- a/lib/galaxy/tools/error_reports/plugins/sentry.py
+++ b/lib/galaxy/tools/error_reports/plugins/sentry.py
@@ -65,8 +65,7 @@ class SentryPlugin(ErrorPlugin):
extra['email'] = unicodify(kwargs['email'])
# User submitted message
- if 'message' in kwargs:
- extra['message'] = unicodify(kwargs['message'])
+ extra['message'] = unicodify(kwargs.get('message', ''))
# Construct the error message to send to sentry. The first line
# will be the issue title, everything after that becomes the
diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py
index 5cd92aa913f..b4f3a44f4f4 100644
--- a/lib/galaxy/tools/evaluation.py
+++ b/lib/galaxy/tools/evaluation.py
@@ -26,6 +26,7 @@ from galaxy.tools.wrappers import (
DatasetCollectionWrapper,
DatasetFilenameWrapper,
DatasetListWrapper,
+ ElementIdentifierMapper,
InputValueWrapper,
RawObjectWrapper,
SelectToolParameterWrapper,
@@ -233,11 +234,9 @@ class ToolEvaluator(object):
real_path = dataset.file_name
if real_path in input_dataset_paths:
wrapper_kwds["dataset_path"] = input_dataset_paths[real_path]
- identifier_key = identifier_key_dict.get(dataset, None)
- if identifier_key:
- element_identifier = param_dict.get(identifier_key, None)
- if element_identifier:
- wrapper_kwds["identifier"] = element_identifier
+ element_identifier = element_identifier_mapper.identifier(dataset, param_dict)
+ if element_identifier:
+ wrapper_kwds["identifier"] = element_identifier
input_values[input.name] = \
DatasetFilenameWrapper(dataset, **wrapper_kwds)
elif isinstance(input, DataCollectionToolParameter):
@@ -265,7 +264,7 @@ class ToolEvaluator(object):
# tools where the inputs don't even get passed through. These
# tools (e.g. UCSC) should really be handled in a special way.
if self.tool.check_values:
- identifier_key_dict = dict((v, "%s|__identifier__" % k) for k, v in input_datasets.items()) # allows lookup of identifier through HDA.
+ element_identifier_mapper = ElementIdentifierMapper(input_datasets)
self.__walk_inputs(self.tool.inputs, param_dict, wrap_input)
def __populate_input_dataset_wrappers(self, param_dict, input_datasets, input_dataset_paths):
diff --git a/lib/galaxy/tools/parameters/__init__.py b/lib/galaxy/tools/parameters/__init__.py
index 0d0d90e1d77..c1630c38e57 100644
--- a/lib/galaxy/tools/parameters/__init__.py
+++ b/lib/galaxy/tools/parameters/__init__.py
@@ -41,26 +41,78 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre
>>> g = BooleanToolParameter(None, XML(''))
>>> h = TextToolParameter(None, XML(''))
>>> i = TextToolParameter(None, XML(''))
- >>> b.name = 'b'
- >>> b.inputs = odict([('c', c), ('d', d)])
- >>> d.name = 'd'
- >>> d.inputs = odict([('e', e), ('f', f)])
+ >>> j = TextToolParameter(None, XML(''))
+ >>> b.name = b.title = 'b'
+ >>> b.inputs = odict([ ('c', c), ('d', d) ])
+ >>> d.name = d.title = 'd'
+ >>> d.inputs = odict([ ('e', e), ('f', f) ])
>>> f.test_param = g
>>> f.name = 'f'
- >>> f.cases = [Bunch(value='true', inputs= { 'h': h }), Bunch(value='false', inputs= { 'i': i })]
+ >>> f.cases = [Bunch(value='true', inputs= {'h': h}), Bunch(value='false', inputs= { 'i': i })]
>>>
- >>> def visitor(input, value, prefix, prefixed_name, **kwargs):
- ... print('name=%s, prefix=%s, prefixed_name=%s, value=%s' % (input.name, prefix, prefixed_name, value))
- >>> inputs = odict([('a',a),('b',b)])
- >>> nested = odict([('a', 1), ('b', [odict([('c', 3), ('d', [odict([('e', 5), ('f', odict([('g', True), ('h', 7)]))])])])])])
+ >>> def visitor(input, value, prefix, prefixed_name, prefixed_label, error, **kwargs):
+ ... print('name=%s, prefix=%s, prefixed_name=%s, prefixed_label=%s, value=%s' % (input.name, prefix, prefixed_name, prefixed_label, value))
+ ... if error:
+ ... print(error)
+ >>> inputs = odict([('a', a),('b', b)])
+ >>> nested = odict([('a', 1), ('b', [odict([('c', 3), ('d', [odict([ ('e', 5), ('f', odict([ ('g', True), ('h', 7)]))])])])])])
>>> visit_input_values(inputs, nested, visitor)
- name=a, prefix=, prefixed_name=a, value=1
- name=c, prefix=b_0|, prefixed_name=b_0|c, value=3
- name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, value=5
- name=g, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|g, value=True
- name=h, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|h, value=7
+ name=a, prefix=, prefixed_name=a, prefixed_label=a, value=1
+ name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=3
+ name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e, value=5
+ name=g, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|g, prefixed_label=b 1 > d 1 > g, value=True
+ name=h, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|h, prefixed_label=b 1 > d 1 > h, value=7
>>> params_from_strings(inputs, params_to_strings(inputs, nested, None), None)['b'][0]['d'][0]['f']['g'] is True
True
+
+ >>> # Conditional test parameter value does not match any case, warning is shown and child values are not visited
+ >>> f.test_param = j
+ >>> nested['b'][0]['d'][0]['f']['j'] = 'j'
+ >>> visit_input_values(inputs, nested, visitor)
+ name=a, prefix=, prefixed_name=a, prefixed_label=a, value=1
+ name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=3
+ name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e, value=5
+ name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|j, prefixed_label=b 1 > d 1 > j, value=j
+ The selected case is unavailable/invalid.
+
+ >>> # Test parameter missing in state, value error
+ >>> del nested['b'][0]['d'][0]['f']['j']
+ >>> visit_input_values(inputs, nested, visitor)
+ name=a, prefix=, prefixed_name=a, prefixed_label=a, value=1
+ name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=3
+ name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e, value=5
+ name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|j, prefixed_label=b 1 > d 1 > j, value=None
+ No value found for 'b 1 > d 1 > j'.
+
+ >>> # Conditional parameter missing in state, value error
+ >>> del nested['b'][0]['d'][0]['f']
+ >>> visit_input_values(inputs, nested, visitor)
+ name=a, prefix=, prefixed_name=a, prefixed_label=a, value=1
+ name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=3
+ name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e, value=5
+ name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|j, prefixed_label=b 1 > d 1 > j, value=None
+ No value found for 'b 1 > d 1 > j'.
+
+ >>> # Conditional input name has changed e.g. due to tool changes, key error
+ >>> f.name = 'f_1'
+ >>> visit_input_values(inputs, nested, visitor)
+ name=a, prefix=, prefixed_name=a, prefixed_label=a, value=1
+ name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=3
+ name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e, value=5
+ name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f_1|j, prefixed_label=b 1 > d 1 > j, value=None
+ No value found for 'b 1 > d 1 > j'.
+
+ >>> # Other parameters are missing in state
+ >>> nested = odict([('b', [odict([( 'd', [odict([('f', odict([('g', True), ('h', 7)]))])])])])])
+ >>> visit_input_values(inputs, nested, visitor)
+ name=a, prefix=, prefixed_name=a, prefixed_label=a, value=None
+ No value found for 'a'.
+ name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=None
+ No value found for 'b 1 > c'.
+ name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e, value=None
+ No value found for 'b 1 > d 1 > e'.
+ name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f_1|j, prefixed_label=b 1 > d 1 > j, value=None
+ No value found for 'b 1 > d 1 > j'.
"""
def callback_helper(input, input_values, name_prefix, label_prefix, parent_prefix, context=None, error=None):
args = {
@@ -83,6 +135,12 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre
if replace:
input_values[input.name] = new_value
+ def get_current_case(input, input_values):
+ try:
+ return input.get_current_case(input_values[input.test_param.name])
+ except (KeyError, ValueError):
+ return -1
+
context = ExpressionContext(input_values, context)
payload = {'context': context, 'no_replacement_value': no_replacement_value}
for input in inputs.values():
@@ -96,15 +154,11 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre
elif isinstance(input, Conditional):
values = input_values[input.name] = input_values.get(input.name, {})
new_name_prefix = name_prefix + input.name + '|'
- case_error = None
- try:
- input.get_current_case(values[input.test_param.name])
- except Exception:
- case_error = 'The selected case is unavailable/invalid.'
- pass
+ case_error = None if get_current_case(input, values) >= 0 else 'The selected case is unavailable/invalid.'
callback_helper(input.test_param, values, new_name_prefix, label_prefix, parent_prefix=name_prefix, context=context, error=case_error)
- values['__current_case__'] = input.get_current_case(values[input.test_param.name])
- visit_input_values(input.cases[values['__current_case__']].inputs, values, callback, new_name_prefix, label_prefix, parent_prefix=name_prefix, **payload)
+ values['__current_case__'] = get_current_case(input, values)
+ if values['__current_case__'] >= 0:
+ visit_input_values(input.cases[values['__current_case__']].inputs, values, callback, new_name_prefix, label_prefix, parent_prefix=name_prefix, **payload)
elif isinstance(input, Section):
values = input_values[input.name] = input_values.get(input.name, {})
new_name_prefix = name_prefix + input.name + '|'
diff --git a/lib/galaxy/tools/parameters/wrapped.py b/lib/galaxy/tools/parameters/wrapped.py
index c93b69e065b..f35555bad13 100644
--- a/lib/galaxy/tools/parameters/wrapped.py
+++ b/lib/galaxy/tools/parameters/wrapped.py
@@ -12,6 +12,7 @@ from galaxy.tools.wrappers import (
DatasetCollectionWrapper,
DatasetFilenameWrapper,
DatasetListWrapper,
+ ElementIdentifierMapper,
InputValueWrapper,
SelectToolParameterWrapper
)
@@ -21,11 +22,12 @@ PARAMS_UNWRAPPED = object()
class WrappedParameters(object):
- def __init__(self, trans, tool, incoming):
+ def __init__(self, trans, tool, incoming, input_datasets=None):
self.trans = trans
self.tool = tool
self.incoming = incoming
self._params = PARAMS_UNWRAPPED
+ self._input_datasets = input_datasets
@property
def params(self):
@@ -40,6 +42,8 @@ class WrappedParameters(object):
tool = self.tool
incoming = self.incoming
+ element_identifier_mapper = ElementIdentifierMapper(self._input_datasets)
+
# Wrap tool inputs as necessary
for input in inputs.values():
if input.name not in input_values and skip_missing_values:
@@ -64,11 +68,16 @@ class WrappedParameters(object):
tool=tool,
name=input.name)
elif isinstance(input, DataToolParameter):
- input_values[input.name] = \
- DatasetFilenameWrapper(value,
- datatypes_registry=trans.app.datatypes_registry,
- tool=tool,
- name=input.name)
+ wrapper_kwds = dict(
+ datatypes_registry=trans.app.datatypes_registry,
+ tool=tool,
+ name=input.name
+ )
+ element_identifier = element_identifier_mapper.identifier(value, input_values)
+ if element_identifier:
+ wrapper_kwds["identifier"] = element_identifier
+
+ input_values[input.name] = DatasetFilenameWrapper(value, **wrapper_kwds)
elif isinstance(input, SelectToolParameter):
input_values[input.name] = SelectToolParameterWrapper(input, input_values[input.name], other_values=incoming)
elif isinstance(input, DataCollectionToolParameter):
diff --git a/lib/galaxy/tools/toolbox/integrated_panel.py b/lib/galaxy/tools/toolbox/integrated_panel.py
index 555168fde7b..42e10fe5c14 100644
--- a/lib/galaxy/tools/toolbox/integrated_panel.py
+++ b/lib/galaxy/tools/toolbox/integrated_panel.py
@@ -103,4 +103,9 @@ class ManagesIntegratedToolPanelMixin:
shutil.copy(filename, filename + ".copy")
filename = filename + ".copy"
shutil.move(filename, destination)
- os.chmod(self._integrated_tool_panel_config, 0o644)
+ try:
+ os.chmod(destination, 0o644)
+ except OSError:
+ # That can happen if multiple threads are simultaneously moving/chmod'ing this file
+ # Should be harmless, though this race condition should be avoided.
+ pass
diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py
index 257909e3aef..b19761cbc9a 100644
--- a/lib/galaxy/tools/wrappers.py
+++ b/lib/galaxy/tools/wrappers.py
@@ -430,3 +430,21 @@ class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets):
# not specified or if resulting collection is empty.
return self.__input_supplied and bool(self.__element_instance_list)
__nonzero__ = __bool__
+
+
+class ElementIdentifierMapper(object):
+ """Track mapping of dataset collection elements datasets to element identifiers."""
+
+ def __init__(self, input_datasets=None):
+ if input_datasets is not None:
+ self.identifier_key_dict = dict((v, "%s|__identifier__" % k) for k, v in input_datasets.items())
+ else:
+ self.identifier_key_dict = {}
+
+ def identifier(self, dataset_value, input_values):
+ identifier_key = self.identifier_key_dict.get(dataset_value, None)
+ element_identifier = None
+ if identifier_key:
+ element_identifier = input_values.get(identifier_key, None)
+
+ return element_identifier
diff --git a/lib/galaxy/work/context.py b/lib/galaxy/work/context.py
index dd4fe4bebb6..ff7d38f1dd3 100644
--- a/lib/galaxy/work/context.py
+++ b/lib/galaxy/work/context.py
@@ -28,8 +28,6 @@ class WorkRequestContext(ProvidesAppContext, ProvidesUserContext, ProvidesHistor
self.workflow_building_mode = workflow_building_mode
def get_history(self, create=False):
- if create:
- raise NotImplementedError("Cannot create histories from a work request context.")
return self.__history
def set_history(self):
diff --git a/test/api/test_tools.py b/test/api/test_tools.py
index a1ea61d38fb..8591cc360c0 100644
--- a/test/api/test_tools.py
+++ b/test/api/test_tools.py
@@ -969,6 +969,36 @@ class ToolsTestCase(api.ApiTestCase):
output1_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output1)
self.assertEquals(output1_content.strip(), '\n'.join([d['name'] for d in element_identifiers]))
+ @skip_without_tool("identifier_in_actions")
+ def test_identifier_in_actions(self):
+ history_id = self.dataset_populator.new_history()
+
+ element_identifiers = self.dataset_collection_populator.list_identifiers(history_id, contents=["1\t2"])
+
+ payload = dict(
+ instance_type="history",
+ history_id=history_id,
+ element_identifiers=json.dumps(element_identifiers),
+ collection_type="list",
+ )
+
+ create_response = self._post("dataset_collections", payload)
+ dataset_collection = create_response.json()
+
+ inputs = {
+ "input": {'batch': True, 'values': [{'src': 'hdca', 'id': dataset_collection['id']}]},
+ }
+
+ self.dataset_populator.wait_for_history(history_id, assert_ok=True)
+ create_response = self._run("identifier_in_actions", history_id, inputs)
+ self._assert_status_code_is(create_response, 200)
+ create = create_response.json()
+ outputs = create['outputs']
+ output1 = outputs[0]
+
+ output_details = self.dataset_populator.get_history_dataset_details(history_id, dataset=output1)
+ assert output_details["metadata_column_names"][1] == "data1", output_details
+
@skip_without_tool("cat1")
def test_map_over_nested_collections(self):
history_id = self.dataset_populator.new_history()
diff --git a/test/functional/tools/identifier_in_actions.xml b/test/functional/tools/identifier_in_actions.xml
new file mode 100644
index 00000000000..ecccf112a47
--- /dev/null
+++ b/test/functional/tools/identifier_in_actions.xml
@@ -0,0 +1,23 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml
index aac89a43dd5..5bd62a7cbb8 100644
--- a/test/functional/tools/samples_tool_conf.xml
+++ b/test/functional/tools/samples_tool_conf.xml
@@ -95,6 +95,7 @@
+