diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 677f5b69a30..6c98ca43bca 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -711,7 +711,7 @@ class JobWrapper( object ): self.job_id = job.id self.session_id = job.session_id self.user_id = job.user_id - self.tool = queue.app.toolbox.get_tool( job.tool_id, exact=True ) + self.tool = queue.app.toolbox.get_tool( job.tool_id, job.tool_version, exact=True ) self.queue = queue self.app = queue.app self.sa_session = self.app.model.context diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 07a859d7f16..9504a26dbc3 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -119,7 +119,7 @@ class JobHandlerQueue( object ): & ( model.Job.handler == self.app.config.server_name ) ).all() for job in jobs_at_startup: - if not self.app.toolbox.has_tool( job.tool_id, exact=True ): + if not self.app.toolbox.has_tool( job.tool_id, job.tool_version, exact=True ): log.warning( "(%s) Tool '%s' removed from tool config, unable to recover job" % ( job.id, job.tool_id ) ) self.job_wrapper( job ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' ) elif job.job_runner_name is not None and job.job_runner_external_id is None: diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 964443a726d..b7fb93cb653 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -126,6 +126,10 @@ class ToolBox( object, Dictifiable ): # shed_tool_conf.xml file. self._dynamic_tool_confs = [] self._tools_by_id = {} + # Tool lineages can contain chains of related tools with different ids + # so each will be present once in the above dictionary. The following + # dictionary can instead hold multiple tools with different versions. + self._tool_versions_by_id = {} self._workflows_by_id = {} # In-memory dictionary that defines the layout of the tool panel. self._tool_panel = odict() @@ -352,7 +356,8 @@ class ToolBox( object, Dictifiable ): inserted = True if not inserted: # Check the tool's installed versions. - for lineage_id in tool.lineage_ids: + for tool_lineage_version in tool.lineage.get_versions(): + lineage_id = tool_lineage_version.id lineage_id_key = 'tool_%s' % lineage_id for index, integrated_panel_key in enumerate( self._integrated_tool_panel.keys() ): if lineage_id_key == integrated_panel_key: @@ -503,20 +508,26 @@ class ToolBox( object, Dictifiable ): def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=False ): """Attempt to locate a tool in the tool box.""" + if tool_version: + tool_version = str( tool_version ) + if get_all_versions and exact: raise AssertionError("Cannot specify get_tool with both get_all_versions and exact as True") if tool_id in self._tools_by_id and not get_all_versions: + if tool_version and tool_version in self._tool_versions_by_id[ tool_id ]: + return self._tool_versions_by_id[ tool_id ][ tool_version ] #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) return self._tools_by_id[ tool_id ] #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions rval = [] tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: - tool_version_ids = tool_lineage.get_version_ids( ) - for tool_version_id in tool_version_ids: - if tool_version_id in self._tools_by_id: - rval.append( self._tools_by_id[ tool_version_id ] ) + lineage_tool_versions = tool_lineage.get_versions( ) + for lineage_tool_version in lineage_tool_versions: + lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) + if lineage_tool: + rval.append( lineage_tool ) if not rval: #still no tool, do a deeper search and try to match by old ids for tool in self._tools_by_id.itervalues(): @@ -561,11 +572,12 @@ class ToolBox( object, Dictifiable ): """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: - tool_version_ids = tool_lineage.get_version_ids( ) + lineage_tool_versions = tool_lineage.get_versions( ) available_tool_versions = [] - for tool_version_id in tool_version_ids: - if tool_version_id in self._tools_by_id: - available_tool_versions.append( self._tools_by_id[ tool_version_id ] ) + for lineage_tool_version in lineage_tool_versions: + tool = self._tool_from_lineage_version( lineage_tool_version ) + if tool: + available_tool_versions.append( tool ) return available_tool_versions else: if tool_id in self._tools_by_id: @@ -746,7 +758,6 @@ class ToolBox( object, Dictifiable ): tool_lineage = self._lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) # Load the tool's lineage ids. tool.lineage = tool_lineage - tool.lineage_ids = tool_lineage.get_version_ids( ) self._tool_tag_manager.handle_tags( tool.id, elem ) self.__add_tool( tool, load_panel_dict, panel_dict ) # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. @@ -925,7 +936,20 @@ class ToolBox( object, Dictifiable ): return tool def register_tool( self, tool ): - self._tools_by_id[ tool.id ] = tool + tool_id = tool.id + version = tool.version or None + if tool_id not in self._tool_versions_by_id: + self._tool_versions_by_id[ tool_id ] = { version: tool } + else: + self._tool_versions_by_id[ tool_id ][ version ] = tool + if tool_id in self._tools_by_id: + related_tool = self._tools_by_id[ tool_id ] + # This one becomes the default un-versioned tool + # if newer. + if self._newer_tool( tool, related_tool ): + self._tools_by_id[ tool_id ] = tool + else: + self._tools_by_id[ tool_id ] = tool def package_tool( self, trans, tool_id ): """ @@ -1202,9 +1226,11 @@ class ToolBox( object, Dictifiable ): if not hasattr( tool, "lineage" ): return None tool_lineage = tool.lineage - lineage_ids = tool_lineage.get_version_ids( reverse=True ) - for lineage_id in lineage_ids: - if lineage_id in self._tools_by_id: + lineage_tool_versions = tool_lineage.get_versions( reverse=True ) + for lineage_tool_version in lineage_tool_versions: + lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) + if lineage_tool: + lineage_id = lineage_tool.id loaded_version_key = 'tool_%s' % lineage_id if loaded_version_key in panel_dict: return panel_dict[ loaded_version_key ] @@ -1214,7 +1240,22 @@ class ToolBox( object, Dictifiable ): """ Return True if tool1 is considered "newer" given its own lineage description. """ - return tool1.lineage_ids.index( tool1.id ) > tool1.lineage_ids.index( tool2.id ) + if not hasattr( tool1, "lineage" ): + return True + lineage_tool_versions = tool1.lineage.get_versions() + for lineage_tool_version in lineage_tool_versions: + lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) + if lineage_tool is tool1: + return False + if lineage_tool is tool2: + return True + return True + + def _tool_from_lineage_version( self, lineage_tool_version ): + if lineage_tool_version.id_based: + return self._tools_by_id.get( lineage_tool_version.id, None ) + else: + return self._tool_versions_by_id.get( lineage_tool_version.id, {} ).get( lineage_tool_version.version, None ) def _filter_for_panel( item, filters, context ): diff --git a/lib/galaxy/tools/toolbox/lineages/factory.py b/lib/galaxy/tools/toolbox/lineages/factory.py index 77efe944017..eb28ec06f11 100644 --- a/lib/galaxy/tools/toolbox/lineages/factory.py +++ b/lib/galaxy/tools/toolbox/lineages/factory.py @@ -1,4 +1,5 @@ from .tool_shed import ToolShedLineage +from .stock import StockLineage class LineageMap(object): @@ -13,7 +14,10 @@ class LineageMap(object): tool_id = tool.id if tool_id not in self.lineage_map: tool_shed_repository = kwds.get("tool_shed_repository", None) - lineage = ToolShedLineage.from_tool(self.app, tool, tool_shed_repository) + if tool_shed_repository: + lineage = ToolShedLineage.from_tool(self.app, tool, tool_shed_repository) + else: + lineage = StockLineage.from_tool( tool ) self.lineage_map[tool_id] = lineage return self.lineage_map[tool_id] diff --git a/lib/galaxy/tools/toolbox/lineages/interface.py b/lib/galaxy/tools/toolbox/lineages/interface.py index 1135213b8d1..46a49ecb87d 100644 --- a/lib/galaxy/tools/toolbox/lineages/interface.py +++ b/lib/galaxy/tools/toolbox/lineages/interface.py @@ -8,7 +8,35 @@ class ToolLineage(object): __metaclass__ = ABCMeta @abstractmethod - def get_version_ids( self, reverse=False ): - """ Return an ordered list of lineages in this chain, from - oldest to newest. + def get_versions( self, reverse=False ): + """ Return an ordered list of lineages (ToolLineageVersion) in this + chain, from oldest to newest. """ + + +class ToolLineageVersion(object): + """ Represents a single tool in a lineage. If lineage is based + around GUIDs that somehow encode the version (either using GUID + or a simple tool id and a version). """ + + def __init__(self, id, version): + self.id = id + self.version = version + + @staticmethod + def from_id_and_verion( id, version ): + assert version is not None + return ToolLineageVersion( id, version ) + + @staticmethod + def from_guid( guid ): + return ToolLineageVersion( guid, None ) + + @property + def id_based( self ): + """ Return True if the lineage is defined by GUIDs (in this + case the indexer of the tools (i.e. the ToolBox) should ignore + the tool_version (because it is encoded in the GUID and managed + externally). + """ + return self.version is None diff --git a/lib/galaxy/tools/toolbox/lineages/stock.py b/lib/galaxy/tools/toolbox/lineages/stock.py new file mode 100644 index 00000000000..c2db57bee7f --- /dev/null +++ b/lib/galaxy/tools/toolbox/lineages/stock.py @@ -0,0 +1,44 @@ +import threading + +from distutils.version import LooseVersion + +from .interface import ToolLineage +from .interface import ToolLineageVersion + + +class StockLineage(ToolLineage): + """ Simple tool's loaded directly from file system with lineage + determined solely by distutil's LooseVersion naming scheme. + """ + lineages_by_id = {} + lock = threading.Lock() + + def __init__(self, tool_id, **kwds): + self.tool_id = tool_id + self.tool_versions = set() + + @staticmethod + def from_tool( tool ): + tool_id = tool.id + lineages_by_id = StockLineage.lineages_by_id + with StockLineage.lock: + if tool_id not in lineages_by_id: + lineages_by_id[ tool_id ] = StockLineage( tool_id ) + lineage = lineages_by_id[ tool_id ] + lineage.register_version( tool.version ) + return lineage + + def register_version( self, tool_version ): + assert tool_version is not None + self.tool_versions.add( tool_version ) + + def get_versions( self, reverse=False ): + versions = [ ToolLineageVersion( self.tool_id, v ) for v in self.tool_versions ] + # Sort using LooseVersion which defines an appropriate __cmp__ + # method for comparing tool versions. + return sorted( versions, key=_to_loose_version ) + + +def _to_loose_version( tool_lineage_version ): + version = str( tool_lineage_version.version ) + return LooseVersion( version ) diff --git a/lib/galaxy/tools/toolbox/lineages/tool_shed.py b/lib/galaxy/tools/toolbox/lineages/tool_shed.py index 66b83e05798..7e261c367fb 100644 --- a/lib/galaxy/tools/toolbox/lineages/tool_shed.py +++ b/lib/galaxy/tools/toolbox/lineages/tool_shed.py @@ -1,4 +1,5 @@ from .interface import ToolLineage +from .interface import ToolLineageVersion from galaxy.model.tool_shed_install import ToolVersion @@ -32,6 +33,9 @@ class ToolShedLineage(ToolLineage): tool_version = self.app.install_model.context.query( ToolVersion ).get( self.tool_version_id ) return tool_version.get_version_ids( self.app, reverse=reverse ) + def get_versions( self, reverse=False ): + return map( ToolLineageVersion.from_guid, self.get_version_ids( reverse=reverse ) ) + def get_install_tool_version( app, tool_id ): return app.install_model.context.query( diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 253505f7dd9..c46536e27b2 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -467,10 +467,10 @@ class ToolModule( WorkflowModule ): type = "tool" - def __init__( self, trans, tool_id ): + def __init__( self, trans, tool_id, tool_version=None ): self.trans = trans self.tool_id = tool_id - self.tool = trans.app.toolbox.get_tool( tool_id ) + self.tool = trans.app.toolbox.get_tool( tool_id, tool_version=tool_version ) self.post_job_actions = {} self.workflow_outputs = [] self.state = None @@ -493,11 +493,14 @@ class ToolModule( WorkflowModule ): @classmethod def from_dict( Class, trans, d, secure=True ): tool_id = d[ 'tool_id' ] - module = Class( trans, tool_id ) + tool_version = str( d.get( 'tool_version', None ) ) + module = Class( trans, tool_id, tool_version=tool_version ) module.state = galaxy.tools.DefaultToolState() if module.tool is not None: if d.get('tool_version', 'Unspecified') != module.get_tool_version(): - module.version_changes.append( "%s: using version '%s' instead of version '%s' indicated in this workflow." % ( tool_id, d.get( 'tool_version', 'Unspecified' ), module.get_tool_version() ) ) + message = "%s: using version '%s' instead of version '%s' indicated in this workflow." % ( tool_id, d.get( 'tool_version', 'Unspecified' ), module.get_tool_version() ) + log.debug(message) + module.version_changes.append(message) module.state.decode( d[ "tool_state" ], module.tool, module.trans.app, secure=secure ) module.errors = d.get( "tool_errors", None ) module.post_job_actions = d.get( "post_job_actions", {} ) @@ -519,9 +522,12 @@ class ToolModule( WorkflowModule ): # This step has its state saved in the config field due to the # tool being previously unavailable. return module_factory.from_dict(trans, loads(step.config), secure=False) - module = Class( trans, tool_id ) + tool_version = step.tool_version + module = Class( trans, tool_id, tool_version=tool_version ) if step.tool_version and (step.tool_version != module.tool.version): - module.version_changes.append("%s: using version '%s' instead of version '%s' indicated in this workflow." % (tool_id, module.tool.version, step.tool_version)) + message = "%s: using version '%s' instead of version '%s' indicated in this workflow." % (tool_id, module.tool.version, step.tool_version) + log.debug(message) + module.version_changes.append(message) module.recover_state( step.tool_inputs ) module.errors = step.tool_errors module.workflow_outputs = step.workflow_outputs @@ -723,7 +729,7 @@ class ToolModule( WorkflowModule ): return state, step_errors def execute( self, trans, progress, invocation, step ): - tool = trans.app.toolbox.get_tool( step.tool_id ) + tool = trans.app.toolbox.get_tool( step.tool_id, tool_version=step.tool_version ) tool_state = step.state collections_to_match = self._find_collections_to_match( tool, progress, step ) diff --git a/test/api/test_tools.py b/test/api/test_tools.py index 6d02738796c..914ff517de4 100644 --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -165,6 +165,18 @@ class ToolsTestCase( api.ApiTestCase ): output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 ) self.assertEqual( output1_content.strip(), "Cat1Testlistified" ) + @skip_without_tool( "multiple_versions" ) + def test_run_by_versions( self ): + for version in ["0.1", "0.2"]: + # Run simple non-upload tool with an input data parameter. + history_id = self.dataset_populator.new_history() + inputs = dict() + outputs = self._run_and_get_outputs( tool_id="multiple_versions", history_id=history_id, inputs=inputs, tool_version=version ) + self.assertEquals( len( outputs ), 1 ) + output1 = outputs[ 0 ] + output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 ) + self.assertEqual( output1_content.strip(), "Version " + version ) + @skip_without_tool( "cat1" ) def test_run_cat1_single_meta_wrapper( self ): # Wrap input in a no-op meta parameter wrapper like Sam is planning to @@ -650,8 +662,8 @@ class ToolsTestCase( api.ApiTestCase ): def _cat1_outputs( self, history_id, inputs ): return self._run_outputs( self._run_cat1( history_id, inputs ) ) - def _run_and_get_outputs( self, tool_id, history_id, inputs ): - return self._run_outputs( self._run( tool_id, history_id, inputs ) ) + def _run_and_get_outputs( self, tool_id, history_id, inputs, tool_version=None ): + return self._run_outputs( self._run( tool_id, history_id, inputs, tool_version=tool_version ) ) def _run_outputs( self, create_response ): self._assert_status_code_is( create_response, 200 ) @@ -660,12 +672,14 @@ class ToolsTestCase( api.ApiTestCase ): def _run_cat1( self, history_id, inputs, assert_ok=False ): return self._run( 'cat1', history_id, inputs, assert_ok=assert_ok ) - def _run( self, tool_id, history_id, inputs, assert_ok=False ): + def _run( self, tool_id, history_id, inputs, assert_ok=False, tool_version=None ): payload = self.dataset_populator.run_tool_payload( tool_id=tool_id, inputs=inputs, history_id=history_id, ) + if tool_version is not None: + payload[ "tool_version" ] = tool_version create_response = self._post( "tools", data=payload ) if assert_ok: self._assert_status_code_is( create_response, 200 ) diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index b092a0284f5..696c2d722a7 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -444,6 +444,28 @@ class WorkflowsApiTestCase( BaseWorkflowsApiTestCase ): def test_run_workflow( self ): self.__run_cat_workflow( inputs_by='step_id' ) + @skip_without_tool( "multiple_versions" ) + def test_run_versioned_tools( self ): + history_01_id = self.dataset_populator.new_history() + workflow_version_01 = self._upload_yaml_workflow( """ +- tool_id: multiple_versions + tool_version: "0.1" + state: + inttest: 0 +""" ) + self.__invoke_workflow( history_01_id, workflow_version_01 ) + self.dataset_populator.wait_for_history( history_01_id, assert_ok=True ) + + history_02_id = self.dataset_populator.new_history() + workflow_version_02 = self._upload_yaml_workflow( """ +- tool_id: multiple_versions + tool_version: "0.2" + state: + inttest: 1 +""" ) + self.__invoke_workflow( history_02_id, workflow_version_02 ) + self.dataset_populator.wait_for_history( history_02_id, assert_ok=True ) + def __run_cat_workflow( self, inputs_by ): workflow = self.workflow_populator.load_workflow( name="test_for_run" ) workflow["steps"]["0"]["uuid"] = str(uuid4()) @@ -910,7 +932,7 @@ test_data: self._assert_status_code_is( hda_info_response, 200 ) self.assertEquals( hda_info_response.json()[ "metadata_data_lines" ], lines ) - def __invoke_workflow( self, history_id, workflow_id, inputs, assert_ok=True ): + def __invoke_workflow( self, history_id, workflow_id, inputs={}, assert_ok=True ): workflow_request = dict( history="hist_id=%s" % history_id, ) diff --git a/test/functional/tools/multiple_versions_v01.xml b/test/functional/tools/multiple_versions_v01.xml new file mode 100644 index 00000000000..8b05454529f --- /dev/null +++ b/test/functional/tools/multiple_versions_v01.xml @@ -0,0 +1,11 @@ + + + echo "Version 0.1" > $out_file1 + + + + + + + + diff --git a/test/functional/tools/multiple_versions_v02.xml b/test/functional/tools/multiple_versions_v02.xml new file mode 100644 index 00000000000..7a7a3a14937 --- /dev/null +++ b/test/functional/tools/multiple_versions_v02.xml @@ -0,0 +1,11 @@ + + + echo "Version 0.2" > $out_file1 + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 751eb627264..d0a1f8d1974 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -36,6 +36,9 @@ + + +