From c2897c211b1d9a3743171e3e9b05c235b79c50f7 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 26 May 2017 11:00:29 +0200
Subject: [PATCH 01/14] Replace ShedToolLineage with enhanced StockLineage

This means we don't rely on the Tool Shed metadata anymore and instead
deduce the tool lineage via the tool_id, which contains the version.
Version comparison then happens via LooseVersions.
This works for toolshed and non-toolshed tools, restoring proper
grouping in the tool panel and version switches, even for tool shed
tools whose lineage is broken.
---
 lib/galaxy/config.py                          |  2 -
 .../model/tool_shed_install/__init__.py       | 60 ++++--------
 lib/galaxy/queue_worker.py                    |  4 -
 lib/galaxy/tools/__init__.py                  |  8 +-
 lib/galaxy/tools/toolbox/base.py              | 15 +--
 lib/galaxy/tools/toolbox/lineages/__init__.py |  3 +-
 lib/galaxy/tools/toolbox/lineages/factory.py  | 42 ++++----
 .../tools/toolbox/lineages/interface.py       |  2 +-
 lib/galaxy/tools/toolbox/lineages/stock.py    | 17 +++-
 .../tools/toolbox/lineages/tool_shed.py       | 98 -------------------
 lib/galaxy/util/tool_version.py               | 19 ++++
 .../webapps/galaxy/controllers/admin.py       | 21 ++--
 lib/tool_shed/tools/tool_version_manager.py   |  2 -
 test/unit/tools/test_toolbox.py               |  5 +-
 14 files changed, 98 insertions(+), 200 deletions(-)
 delete mode 100644 lib/galaxy/tools/toolbox/lineages/tool_shed.py
 create mode 100644 lib/galaxy/util/tool_version.py

diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py
index e6a04cae318..72f535de62e 100644
--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -925,11 +925,9 @@ class ConfiguresGalaxyMixin:
         from galaxy import tools
         from galaxy.managers.citations import CitationsManager
         from galaxy.tools.deps import containers
-        from galaxy.tools.toolbox.lineages.tool_shed import ToolVersionCache
         import galaxy.tools.search
 
         self.citations_manager = CitationsManager( self )
-        self.tool_version_cache = ToolVersionCache(self)
 
         self._toolbox_lock = threading.RLock()
         # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file.
diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py
index ba0cc4b6c79..6ba9afb5746 100644
--- a/lib/galaxy/model/tool_shed_install/__init__.py
+++ b/lib/galaxy/model/tool_shed_install/__init__.py
@@ -589,51 +589,33 @@ class ToolVersion( object, Dictifiable ):
         self.tool_id = tool_id
         self.tool_shed_repository = tool_shed_repository
 
+    @property
+    def version_ids(self):
+        return self.get_version_ids()
+
     def get_previous_version( self, app ):
-        parent_id = app.tool_version_cache.tool_id_to_parent_id.get(self.id, None)
-        if parent_id:
-            return app.tool_version_cache.tool_version_by_id[parent_id]
-        else:
-            return None
+        version_ids = self.version_ids
+        index = version_ids.index(self.tool_id)
+        if index > 0:
+            tool = self.app.toolbox._tools_by_id.get(version_ids[index - 1])
+            if tool:
+                return tool.tool_version
+        return None
 
     def get_next_version( self, app ):
-        child_id = app.tool_version_cache.parent_id_to_tool_id.get(self.id, None)
-        if child_id:
-            return app.tool_version_cache.tool_version_by_id[child_id]
-        else:
-            return None
-
-    def get_versions( self, app ):
-        tool_versions = []
-
-        # Prepend ancestors.
-        def __ancestors( app, tool_version ):
-            # Should we handle multiple parents at each level?
-            previous_version = tool_version.get_previous_version( app )
-            if previous_version:
-                if previous_version not in tool_versions:
-                    tool_versions.insert( 0, previous_version )
-                    __ancestors( app, previous_version )
-
-        # Append descendants.
-        def __descendants( app, tool_version ):
-            # Should we handle multiple child siblings at each level?
-            next_version = tool_version.get_next_version( app )
-            if next_version:
-                if next_version not in tool_versions:
-                    tool_versions.append( next_version )
-                    __descendants( app, next_version )
-
-        __ancestors( app, self )
-        if self not in tool_versions:
-            tool_versions.append( self )
-        __descendants( app, self )
-        return tool_versions
+        version_ids = self.version_ids
+        index = version_ids.index(self.tool_id)
+        if len(version_ids) > index + 1:
+            tool = self.app.toolbox._tools_by_id.get(version_ids[index + 1])
+            if tool:
+                return tool.tool_version
+        return None
 
     def get_version_ids( self, app, reverse=False ):
-        version_ids = [ tool_version.tool_id for tool_version in self.get_versions( app ) ]
+        lineage = app.toolbox._lineage_map.get(self.tool_id)
+        version_ids = lineage.get_versions()
         if reverse:
-            version_ids.reverse()
+            version_ids = reversed(version_ids)
         return version_ids
 
     def to_dict( self, view='element' ):
diff --git a/lib/galaxy/queue_worker.py b/lib/galaxy/queue_worker.py
index 564ef962d59..b76ddfc7afc 100644
--- a/lib/galaxy/queue_worker.py
+++ b/lib/galaxy/queue_worker.py
@@ -102,10 +102,8 @@ def _get_new_toolbox(app):
     """
     from galaxy import tools
     from galaxy.tools.special_tools import load_lib_tools
-    from galaxy.tools.toolbox.lineages.tool_shed import ToolVersionCache
     if hasattr(app, 'tool_shed_repository_cache'):
                 app.tool_shed_repository_cache.rebuild()
-    app.tool_version_cache = ToolVersionCache(app)  # Load new tools into version cache
     tool_configs = app.config.tool_configs
     if app.config.migrated_tools_config not in tool_configs:
         tool_configs.append(app.config.migrated_tools_config)
@@ -122,7 +120,6 @@ def _get_new_toolbox(app):
 def reload_data_managers(app, **kwargs):
     reload_timer = util.ExecutionTimer()
     from galaxy.tools.data_manager.manager import DataManagers
-    from galaxy.tools.toolbox.lineages.tool_shed import ToolVersionCache
     log.debug("Executing data managers reload on '%s'", app.config.server_name)
     if hasattr(app, 'tool_shed_repository_cache'):
         app.tool_shed_repository_cache.rebuild()
@@ -131,7 +128,6 @@ def reload_data_managers(app, **kwargs):
     reload_count = app.data_managers._reload_count
     app.data_managers = DataManagers(app)
     app.data_managers._reload_count = reload_count + 1
-    app.tool_version_cache = ToolVersionCache(app)
     if hasattr(app, 'tool_cache'):
         app.tool_cache.reset_status()
     log.debug("Data managers reloaded %s", reload_timer)
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 4dbd611b169..4b1cbc90191 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -442,7 +442,7 @@ class Tool( object, Dictifiable ):
     @property
     def tool_version( self ):
         """Return a ToolVersion if one exists for our id"""
-        return self.app.tool_version_cache.tool_version_by_tool_id.get(self.id)
+        return self.app.toolbox._lineage_map.get(self.id)
 
     @property
     def tool_versions( self ):
@@ -1832,11 +1832,7 @@ class Tool( object, Dictifiable ):
             tool_help = unicodify( tool_help, 'utf-8' )
 
         # create tool versions
-        tool_versions = []
-        tools = self.app.toolbox.get_loaded_tools_by_lineage( self.id )
-        for t in tools:
-            if t.version not in tool_versions:
-                tool_versions.append( t.version )
+        tool_versions = list(self.tool_version.tool_versions)
 
         # update tool model
         tool_model.update({
diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py
index 12c08e38d7a..2144f08ae4e 100644
--- a/lib/galaxy/tools/toolbox/base.py
+++ b/lib/galaxy/tools/toolbox/base.py
@@ -427,8 +427,6 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
             # exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions
             rval = []
             tool_lineage = self._lineage_map.get( tool_id )
-            if not tool_lineage:
-                tool_lineage = self._lineage_map.get_versionless( tool_id )
             if tool_lineage:
                 lineage_tool_versions = tool_lineage.get_versions( )
                 for lineage_tool_version in lineage_tool_versions:
@@ -568,7 +566,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
                     tool.guid = guid
                     tool.version = item.elem.find( "version" ).text
                 # Make sure tools have a tool_version object.
-                tool_lineage = self._lineage_map.register( tool, from_toolshed=guid )
+                tool_lineage = self._lineage_map.register( tool )
                 tool.lineage = tool_lineage
                 if item.has_elem:
                     self._tool_tag_manager.handle_tags( tool.id, item.elem )
@@ -952,7 +950,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
             if not hasattr( tool, "lineage" ):
                 return None
             tool_lineage = tool.lineage
-        lineage_tool_versions = tool_lineage.get_versions( reverse=True )
+        lineage_tool_versions = reversed(tool_lineage.get_versions())
         for lineage_tool_version in lineage_tool_versions:
             lineage_tool = self._tool_from_lineage_version( lineage_tool_version )
             if lineage_tool:
@@ -967,14 +965,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
         """
         if not hasattr( tool1, "lineage" ):
             return True
-        lineage_tool_versions = tool1.lineage.get_versions()
-        for lineage_tool_version in lineage_tool_versions:
-            lineage_tool = self._tool_from_lineage_version( lineage_tool_version )
-            if lineage_tool is tool1:
-                return False
-            if lineage_tool is tool2:
-                return True
-        return True
+        return tool1.version_object > tool2.version_object
 
     def _tool_from_lineage_version( self, lineage_tool_version ):
         if lineage_tool_version.id_based:
diff --git a/lib/galaxy/tools/toolbox/lineages/__init__.py b/lib/galaxy/tools/toolbox/lineages/__init__.py
index f79c808b79b..d4e823400de 100644
--- a/lib/galaxy/tools/toolbox/lineages/__init__.py
+++ b/lib/galaxy/tools/toolbox/lineages/__init__.py
@@ -1,6 +1,5 @@
 from .factory import LineageMap
 from .interface import ToolLineage
-from .tool_shed import ToolVersionCache
 
 
-__all__ = ("LineageMap", "ToolLineage", "ToolVersionCache")
+__all__ = ("LineageMap", "ToolLineage")
diff --git a/lib/galaxy/tools/toolbox/lineages/factory.py b/lib/galaxy/tools/toolbox/lineages/factory.py
index 2da533471c6..0e9080039eb 100644
--- a/lib/galaxy/tools/toolbox/lineages/factory.py
+++ b/lib/galaxy/tools/toolbox/lineages/factory.py
@@ -1,15 +1,6 @@
+from galaxy.util.tool_version import remove_version_from_guid
+
 from .stock import StockLineage
-from .tool_shed import ToolShedLineage
-
-
-def remove_version_from_guid( guid ):
-    """
-    Removes version from toolshed-derived tool_id(=guid).
-    """
-    if "/repos/" not in guid:
-        return None
-    last_slash = guid.rfind('/')
-    return guid[:last_slash]
 
 
 class LineageMap(object):
@@ -20,28 +11,39 @@ class LineageMap(object):
         self.lineage_map = {}
         self.app = app
 
-    def register(self, tool, from_toolshed=False):
+    def register(self, tool):
         tool_id = tool.id
-        versionless_tool_id = remove_version_from_guid( tool_id )
-        if from_toolshed:
-            lineage = ToolShedLineage.from_tool(self.app, tool)
-        else:
+        versionless_tool_id = remove_version_from_guid( tool_id)
+        lineage = self.lineage_map.get(versionless_tool_id)
+        if not lineage:
             lineage = StockLineage.from_tool( tool )
         if versionless_tool_id and versionless_tool_id not in self.lineage_map:
             self.lineage_map[versionless_tool_id] = lineage
         if tool_id not in self.lineage_map:
             self.lineage_map[tool_id] = lineage
+        lineage.register_version(tool.version)
         return self.lineage_map[tool_id]
 
     def get(self, tool_id):
+        """
+        Get lineage for `tool_id`.
+
+        By preference the lineage for a version-agnostic tool_id is returned.
+        Falls back to fetching the lineage only when this fails.
+        This happens when the tool_id does not contain a version.
+        """
+        lineage = self._get_versionless(tool_id)
+        if lineage:
+            return lineage
         if tool_id not in self.lineage_map:
-            lineage = ToolShedLineage.from_tool_id( self.app, tool_id )
+            tool = self.app.toolbox._tools_by_id.get(tool_id)
+            if tool:
+                lineage = StockLineage.from_tool( tool )
             if lineage:
                 self.lineage_map[tool_id] = lineage
+        return self.lineage_map.get(tool_id)
 
-        return self.lineage_map.get(tool_id, None)
-
-    def get_versionless(self, tool_id):
+    def _get_versionless(self, tool_id):
         versionless_tool_id = remove_version_from_guid(tool_id)
         return self.lineage_map.get(versionless_tool_id, None)
 
diff --git a/lib/galaxy/tools/toolbox/lineages/interface.py b/lib/galaxy/tools/toolbox/lineages/interface.py
index 8b82af33692..a36fe5445a8 100644
--- a/lib/galaxy/tools/toolbox/lineages/interface.py
+++ b/lib/galaxy/tools/toolbox/lineages/interface.py
@@ -12,7 +12,7 @@ class ToolLineage(object):
     """
 
     @abstractmethod
-    def get_versions( self, reverse=False ):
+    def get_versions( self ):
         """ Return an ordered list of lineages (ToolLineageVersion) in this
         chain, from oldest to newest.
         """
diff --git a/lib/galaxy/tools/toolbox/lineages/stock.py b/lib/galaxy/tools/toolbox/lineages/stock.py
index b6ed17348a0..ad5491d9d8e 100644
--- a/lib/galaxy/tools/toolbox/lineages/stock.py
+++ b/lib/galaxy/tools/toolbox/lineages/stock.py
@@ -2,6 +2,8 @@ import threading
 
 from distutils.version import LooseVersion
 
+from galaxy.util.tool_version import remove_version_from_guid
+
 from .interface import ToolLineage
 from .interface import ToolLineageVersion
 
@@ -17,6 +19,12 @@ class StockLineage(ToolLineage):
         self.tool_id = tool_id
         self.tool_versions = set()
 
+    @property
+    def tool_ids(self):
+        versionless_tool_id = remove_version_from_guid(self.tool_id)
+        tool_id = versionless_tool_id or self.tool_id
+        return ["%s/%s" % (tool_id, version) for version in self.tool_versions]
+
     @staticmethod
     def from_tool( tool ):
         tool_id = tool.id
@@ -32,11 +40,14 @@ class StockLineage(ToolLineage):
         assert tool_version is not None
         self.tool_versions.add( tool_version )
 
-    def get_versions( self, reverse=False ):
-        versions = [ ToolLineageVersion( self.tool_id, v ) for v in self.tool_versions ]
+    def get_versions( self ):
+        versions = [ ToolLineageVersion( tool_id, tool_version ) for tool_id, tool_version in zip(self.tool_ids, self.tool_versions) ]
         # Sort using LooseVersion which defines an appropriate __cmp__
         # method for comparing tool versions.
-        return sorted( versions, key=_to_loose_version, reverse=reverse )
+        return sorted( versions, key=_to_loose_version )
+
+    def get_version_ids(self):
+        return self.tool_ids
 
     def to_dict(self):
         return dict(
diff --git a/lib/galaxy/tools/toolbox/lineages/tool_shed.py b/lib/galaxy/tools/toolbox/lineages/tool_shed.py
deleted file mode 100644
index 720d45f279e..00000000000
--- a/lib/galaxy/tools/toolbox/lineages/tool_shed.py
+++ /dev/null
@@ -1,98 +0,0 @@
-from .interface import ToolLineage, ToolLineageVersion
-
-try:
-    from galaxy.model.tool_shed_install import ToolVersion
-except ImportError:
-    ToolVersion = None
-
-
-class ToolVersionCache(object):
-    """
-    Instances of this class allow looking up tool_version objects from memory
-    (instead of querying the database) using the tool_version id or the tool_id.
-    This is used to lookup parent tool_version ids using child tool_id, or the
-    inverse, and getting all previous/next tool versions without numerous
-    database requests.
-    """
-    def __init__(self, app):
-        self.app = app
-        self.tool_version_by_id, self.tool_version_by_tool_id = self.get_tool_versions()
-        self.tool_id_to_parent_id, self.parent_id_to_tool_id = self.get_tva_map()
-
-    def get_tva_map(self):
-        """
-        Retrieves all ToolVersionAssociation objects from the database, and builds
-        dictionaries that can be used to either get a tools' parent tool_version id
-        (which can be used to get the parent's tool_version object), or to get the
-        child's tool id using the parent's tool_version id.
-        """
-        tvas = self.app.install_model.context.query(self.app.install_model.ToolVersionAssociation).all()
-        tool_id_to_parent_id = {tva.tool_id: tva.parent_id for tva in tvas}
-        parent_id_to_tool_id = {tva.parent_id: tva.tool_id for tva in tvas}
-        return tool_id_to_parent_id, parent_id_to_tool_id
-
-    def get_tool_versions(self):
-        """
-        Get all tool_version objects from the database and build 2 dictionaries,
-        with tool_version id or tool_id as key and the tool_version object as value.
-        """
-        tool_versions = self.app.install_model.context.query(self.app.install_model.ToolVersion).all()
-        tool_version_by_id = {tv.id: tv for tv in tool_versions}
-        tool_version_by_tool_id = {tv.tool_id: tv for tv in tool_versions}
-        return tool_version_by_id, tool_version_by_tool_id
-
-
-class ToolShedLineage(ToolLineage):
-    """ Representation of tool lineage derived from tool shed repository
-    installations. """
-
-    def __init__(self, app, tool_version, tool_shed_repository=None):
-        if ToolVersion is None:
-            raise Exception("Tool shed models not present, can't create tool shed lineages.")
-        self.app = app
-        self.tool_version_id = tool_version.id
-        # Only used for logging
-        self._tool_shed_repository = tool_shed_repository
-
-    @staticmethod
-    def from_tool( app, tool ):
-        # Make sure the tool has a tool_version.
-        if not get_installed_tool_version( app, tool.id ):
-            tool_version = ToolVersion( tool_id=tool.id, tool_shed_repository=tool.tool_shed_repository )
-            app.install_model.context.add( tool_version )
-            app.install_model.context.flush()
-            app.tool_version_cache = ToolVersionCache(app)
-        return ToolShedLineage( app, tool.tool_version )
-
-    @staticmethod
-    def from_tool_id( app, tool_id ):
-        tool_version = get_installed_tool_version( app, tool_id )
-        if tool_version:
-            return ToolShedLineage( app, tool_version )
-        else:
-            return None
-
-    def get_version_ids( self, reverse=False ):
-        tool_version = self.app.install_model.context.query( ToolVersion ).get( self.tool_version_id )
-        result = tool_version.get_version_ids( self.app, reverse=reverse )
-        return result
-
-    def get_versions( self, reverse=False ):
-        return [ ToolLineageVersion.from_guid(_) for _ in self.get_version_ids( reverse=reverse ) ]
-
-    def to_dict(self):
-        tool_shed_repository = self._tool_shed_repository
-        rval = dict(
-            tool_version_id=self.tool_version_id,
-            tool_versions=[v.to_dict() for v in self.get_versions()],
-            tool_shed_repository=tool_shed_repository if tool_shed_repository is not None else None,
-            lineage_type='tool_shed',
-        )
-        return rval
-
-
-def get_installed_tool_version( app, tool_id ):
-    return app.tool_version_cache.tool_version_by_tool_id.get(tool_id, None)
-
-
-__all__ = ( "ToolShedLineage", )
diff --git a/lib/galaxy/util/tool_version.py b/lib/galaxy/util/tool_version.py
new file mode 100644
index 00000000000..3b602a23a04
--- /dev/null
+++ b/lib/galaxy/util/tool_version.py
@@ -0,0 +1,19 @@
+def remove_version_from_guid(guid):
+
+    """
+    Removes version from toolshed-derived tool_id(=guid).
+    """
+    if "/" not in guid:
+        return None
+    last_slash = guid.rfind('/')
+    return guid[:last_slash]
+
+
+def get_version_from_guid(guid):
+    """
+    Removes version from toolshed-derived tool_id(=guid).
+    """
+    if "/" not in guid:
+        return None
+    last_slash = guid.rfind('/')
+    return guid[last_slash + 1:]
diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py
index f29a9a907ad..b79fedbb64b 100644
--- a/lib/galaxy/webapps/galaxy/controllers/admin.py
+++ b/lib/galaxy/webapps/galaxy/controllers/admin.py
@@ -455,13 +455,20 @@ class ToolVersionListGrid( grids.Grid ):
         def get_value( self, trans, grid, tool_version ):
             tool_ids_str = ''
             toolbox = trans.app.toolbox
-            for tool_id in tool_version.get_version_ids( trans.app ):
-                if toolbox.has_tool( tool_id, exact=True ):
-                    link = url_for( controller='tool_runner', tool_id=tool_id )
-                    link_str = '<a target="_blank" href="%s">' % link
-                    tool_ids_str += '<div class="count-box state-color-ok">%s%s</a></div><br/>' % ( link_str, tool_id )
-                else:
-                    tool_ids_str += '%s<br/>' % tool_id
+            tool = toolbox._tools_by_id.get(tool_version.tool_id)
+            if tool:
+                tool_id = tool.id
+                lineage = trans.app.toolbox._lineage_map.get(tool_id)
+                if lineage:
+                    for tool_id in lineage.tool_ids:
+                        if toolbox.has_tool( tool_id, exact=True ):
+                            link = url_for( controller='tool_runner', tool_id=tool_id )
+                            link_str = '<a target="_blank" href="%s">' % link
+                            tool_ids_str += '<div class="count-box state-color-ok">%s%s</a></div><br/>' % ( link_str, tool_id )
+                        else:
+                            tool_ids_str += '%s<br/>' % tool_version.tool_id
+            else:
+                tool_ids_str += '%s<br/>' % tool_version.tool_id
             return tool_ids_str
 
     # Grid definition
diff --git a/lib/tool_shed/tools/tool_version_manager.py b/lib/tool_shed/tools/tool_version_manager.py
index 7a99ccf806d..6b9c593e86c 100644
--- a/lib/tool_shed/tools/tool_version_manager.py
+++ b/lib/tool_shed/tools/tool_version_manager.py
@@ -2,7 +2,6 @@ import logging
 
 from sqlalchemy import and_, or_
 
-from galaxy.tools.toolbox.lineages.tool_shed import ToolVersionCache
 from tool_shed.util import hg_util
 from tool_shed.util import metadata_util
 from tool_shed.util import repository_util
@@ -112,4 +111,3 @@ class ToolVersionManager( object ):
                                                                        parent_id=tool_version_using_parent_id.id )
                     context.add( tool_version_association )
                     context.flush()
-        self.app.tool_version_cache = ToolVersionCache(self.app)
diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py
index 6a64976ff5e..df00b0bc1cd 100644
--- a/test/unit/tools/test_toolbox.py
+++ b/test/unit/tools/test_toolbox.py
@@ -12,7 +12,6 @@ from galaxy.model import tool_shed_install
 from galaxy.model.tool_shed_install import mapping
 from galaxy.tools import ToolBox
 from galaxy.tools.cache import ToolCache
-from galaxy.tools.toolbox.lineages.tool_shed import ToolVersionCache
 from galaxy.tools.toolbox.watcher import get_tool_conf_watcher
 from galaxy.webapps.galaxy.config_watchers import ConfigWatchers
 
@@ -79,7 +78,6 @@ class BaseToolBoxTestCase(  unittest.TestCase, tools_support.UsesApp, tools_supp
         repository.uninstalled = False
         self.app.install_model.context.add( repository )
         self.app.install_model.context.flush( )
-        self.app.tool_version_cache = ToolVersionCache(self.app)
         return repository
 
     def _setup_two_versions( self ):
@@ -104,7 +102,6 @@ class BaseToolBoxTestCase(  unittest.TestCase, tools_support.UsesApp, tools_supp
 
         self.app.install_model.context.add( version_association )
         self.app.install_model.context.flush( )
-        self.app.tool_version_cache = ToolVersionCache(self.app)
 
     def _setup_two_versions_in_config( self, section=False ):
         if section:
@@ -269,7 +266,7 @@ class ToolBoxTestCase( BaseToolBoxTestCase ):
         ]
         assert self.toolbox.get_tool_id( "github.com/galaxyproject/example/test_tool/0.1" ) == "github.com/galaxyproject/example/test_tool/0.1"
         assert self.toolbox.get_tool_id( "github.com/galaxyproject/example/test_tool/0.2" ) == "github.com/galaxyproject/example/test_tool/0.2"
-        assert self.toolbox.get_tool_id( "github.com/galaxyproject/example/test_tool/0.3" ) is None
+        assert self.toolbox.get_tool_id( "github.com/galaxyproject/example/test_tool/0.3" ) != "github.com/galaxyproject/example/test_tool/0.3"
 
     def test_tool_dir( self ):
         self._init_tool()

From 0017d167f100f84df95967a05b0551250828f7a4 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 29 May 2017 14:51:00 +0200
Subject: [PATCH 02/14] Use correct tool version for
 ToolPanelManager.test_add_twice test to pass

---
 test/unit/shed_unit/test_tool_panel_manager.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/unit/shed_unit/test_tool_panel_manager.py b/test/unit/shed_unit/test_tool_panel_manager.py
index b33d77d0b8f..0f4b3bb1b19 100644
--- a/test/unit/shed_unit/test_tool_panel_manager.py
+++ b/test/unit/shed_unit/test_tool_panel_manager.py
@@ -78,8 +78,8 @@ class ToolPanelManagerTestCase( BaseToolBoxTestCase ):
         for v in "1", "2", "3":
             self.__toolbox = self.get_new_toolbox()
             changeset = "0123456789abcde%s" % v
-            guid = DEFAULT_GUID + ("v%s" % v)
-            tool = self._init_ts_tool( guid=guid, filename="tool_v%s.xml" % v )
+            guid = DEFAULT_GUID + ("v/%s" % v)
+            tool = self._init_ts_tool( guid=guid, filename="tool_v%s.xml" % v, version=v )
             tool_path = self._tool_path( name="tool_v%s.xml" % v )
             new_tools = [{"guid": guid, "tool_config": tool_path}]
             tool_shed_repository = self._repo_install( changeset )
@@ -195,6 +195,7 @@ class ToolPanelManagerTestCase( BaseToolBoxTestCase ):
     def _init_ts_tool( self, guid=DEFAULT_GUID, **kwds ):
         tool = self._init_tool( **kwds )
         tool.guid = guid
+        tool.version = kwds.get('version', '1.0')
         return tool
 
     @property

From db5d0095a6a426efcd587eca6808f350a6e126cf Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 29 May 2017 15:26:32 +0200
Subject: [PATCH 03/14] Fix grouping tests by setting up correct versions

---
 test/unit/tools/test_toolbox.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py
index df00b0bc1cd..9bca497488f 100644
--- a/test/unit/tools/test_toolbox.py
+++ b/test/unit/tools/test_toolbox.py
@@ -84,6 +84,7 @@ class BaseToolBoxTestCase(  unittest.TestCase, tools_support.UsesApp, tools_supp
         repository1 = self._repo_install( changeset="1" )
         version1 = tool_shed_install.ToolVersion()
         version1.tool_id = "github.com/galaxyproject/example/test_tool/0.1"
+        version1.version = "0.1"
         version1.repository = repository1
         self.app.install_model.context.add( version1 )
         self.app.install_model.context.flush( )
@@ -91,6 +92,7 @@ class BaseToolBoxTestCase(  unittest.TestCase, tools_support.UsesApp, tools_supp
         repository2 = self._repo_install( changeset="2" )
         version2 = tool_shed_install.ToolVersion()
         version2.tool_id = "github.com/galaxyproject/example/test_tool/0.2"
+        version2.version = "0.2"
         version2.repository = repository2
 
         self.app.install_model.context.add( version2 )

From aabb933fdf69719adcb6fb9e9ba82d795a3b9b74 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 29 May 2017 18:12:55 +0200
Subject: [PATCH 04/14] Keep StockLineage versions sorted

---
 lib/galaxy/tools/__init__.py               |  2 +-
 lib/galaxy/tools/toolbox/lineages/stock.py | 18 +++++++-----------
 test/unit/tools/test_toolbox.py            |  2 --
 3 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 4b1cbc90191..a5fff42ffb3 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -1832,7 +1832,7 @@ class Tool( object, Dictifiable ):
             tool_help = unicodify( tool_help, 'utf-8' )
 
         # create tool versions
-        tool_versions = list(self.tool_version.tool_versions)
+        tool_versions = self.tool_version.tool_versions
 
         # update tool model
         tool_model.update({
diff --git a/lib/galaxy/tools/toolbox/lineages/stock.py b/lib/galaxy/tools/toolbox/lineages/stock.py
index ad5491d9d8e..44850cb35f7 100644
--- a/lib/galaxy/tools/toolbox/lineages/stock.py
+++ b/lib/galaxy/tools/toolbox/lineages/stock.py
@@ -17,7 +17,11 @@ class StockLineage(ToolLineage):
 
     def __init__(self, tool_id, **kwds):
         self.tool_id = tool_id
-        self.tool_versions = set()
+        self._tool_versions = set()
+
+    @property
+    def tool_versions(self):
+        return sorted(self._tool_versions, key=LooseVersion)
 
     @property
     def tool_ids(self):
@@ -38,13 +42,10 @@ class StockLineage(ToolLineage):
 
     def register_version( self, tool_version ):
         assert tool_version is not None
-        self.tool_versions.add( tool_version )
+        self._tool_versions.add( str(tool_version) )
 
     def get_versions( self ):
-        versions = [ ToolLineageVersion( tool_id, tool_version ) for tool_id, tool_version in zip(self.tool_ids, self.tool_versions) ]
-        # Sort using LooseVersion which defines an appropriate __cmp__
-        # method for comparing tool versions.
-        return sorted( versions, key=_to_loose_version )
+        return [ ToolLineageVersion( tool_id, tool_version ) for tool_id, tool_version in zip(self.tool_ids, self.tool_versions) ]
 
     def get_version_ids(self):
         return self.tool_ids
@@ -55,8 +56,3 @@ class StockLineage(ToolLineage):
             tool_versions=list(self.tool_versions),
             lineage_type='stock',
         )
-
-
-def _to_loose_version( tool_lineage_version ):
-    version = str( tool_lineage_version.version )
-    return LooseVersion( version )
diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py
index 9bca497488f..df00b0bc1cd 100644
--- a/test/unit/tools/test_toolbox.py
+++ b/test/unit/tools/test_toolbox.py
@@ -84,7 +84,6 @@ class BaseToolBoxTestCase(  unittest.TestCase, tools_support.UsesApp, tools_supp
         repository1 = self._repo_install( changeset="1" )
         version1 = tool_shed_install.ToolVersion()
         version1.tool_id = "github.com/galaxyproject/example/test_tool/0.1"
-        version1.version = "0.1"
         version1.repository = repository1
         self.app.install_model.context.add( version1 )
         self.app.install_model.context.flush( )
@@ -92,7 +91,6 @@ class BaseToolBoxTestCase(  unittest.TestCase, tools_support.UsesApp, tools_supp
         repository2 = self._repo_install( changeset="2" )
         version2 = tool_shed_install.ToolVersion()
         version2.tool_id = "github.com/galaxyproject/example/test_tool/0.2"
-        version2.version = "0.2"
         version2.repository = repository2
 
         self.app.install_model.context.add( version2 )

From 77e016bbbd5de566484d1c74c4f5679354edf360 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 30 May 2017 09:51:17 +0200
Subject: [PATCH 05/14] Remove unused next/previous version methods

---
 .../model/tool_shed_install/__init__.py       | 22 -------------------
 1 file changed, 22 deletions(-)

diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py
index 6ba9afb5746..7704e98bb76 100644
--- a/lib/galaxy/model/tool_shed_install/__init__.py
+++ b/lib/galaxy/model/tool_shed_install/__init__.py
@@ -589,28 +589,6 @@ class ToolVersion( object, Dictifiable ):
         self.tool_id = tool_id
         self.tool_shed_repository = tool_shed_repository
 
-    @property
-    def version_ids(self):
-        return self.get_version_ids()
-
-    def get_previous_version( self, app ):
-        version_ids = self.version_ids
-        index = version_ids.index(self.tool_id)
-        if index > 0:
-            tool = self.app.toolbox._tools_by_id.get(version_ids[index - 1])
-            if tool:
-                return tool.tool_version
-        return None
-
-    def get_next_version( self, app ):
-        version_ids = self.version_ids
-        index = version_ids.index(self.tool_id)
-        if len(version_ids) > index + 1:
-            tool = self.app.toolbox._tools_by_id.get(version_ids[index + 1])
-            if tool:
-                return tool.tool_version
-        return None
-
     def get_version_ids( self, app, reverse=False ):
         lineage = app.toolbox._lineage_map.get(self.tool_id)
         version_ids = lineage.get_versions()

From 6864187725b0af03e2196d12722f78f71d84977e Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 20 Jun 2017 20:53:34 +0200
Subject: [PATCH 06/14] ToolVersion to StockLineage, drop ToolVersionCache

remnant from unittests (thx @nsoranzo!).
---
 lib/galaxy/tools/__init__.py | 2 +-
 test/unit/tools_support.py   | 5 -----
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index a5fff42ffb3..818df6d8e0a 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -441,7 +441,7 @@ class Tool( object, Dictifiable ):
 
     @property
     def tool_version( self ):
-        """Return a ToolVersion if one exists for our id"""
+        """Return a StockLineage if one exists for our id."""
         return self.app.toolbox._lineage_map.get(self.id)
 
     @property
diff --git a/test/unit/tools_support.py b/test/unit/tools_support.py
index 3a6fe71c00b..5a666f97922 100644
--- a/test/unit/tools_support.py
+++ b/test/unit/tools_support.py
@@ -154,11 +154,6 @@ class MockApp( object ):
         self.container_finder = NullContainerFinder()
         self.name = "galaxy"
         self._toolbox_lock = MockLock()
-        self.tool_version_cache = Bunch(app=self,
-                                        tool_version_by_id={},
-                                        tool_version_by_tool_id={},
-                                        tool_id_to_parent_id={},
-                                        parent_id_to_tool_id={})
 
     def wait_for_toolbox_reload(self, toolbox):
         # TODO: If the tpm test case passes, does the operation really

From 81e73969dce5102dec799e3ca8b2a3056969f6a3 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 22 Jun 2017 00:22:15 +0200
Subject: [PATCH 07/14] Remove duplication between ToolLineage and TVM

Getting tool versions and tool ids is now done through the tool lineage.
Also assume that the lineage can always be found for a given loaded tool in
the ToolVersionListGrid class. Thanks @nsoranzo for these suggestions!
---
 lib/galaxy/model/tool_shed_install/__init__.py   |  7 -------
 lib/galaxy/tools/toolbox/lineages/factory.py     |  2 +-
 lib/galaxy/tools/toolbox/lineages/stock.py       |  4 +++-
 lib/galaxy/util/tool_version.py                  |  1 -
 lib/galaxy/webapps/galaxy/controllers/admin.py   | 16 +++++++---------
 .../webapps/galaxy/controllers/admin_toolshed.py |  4 +---
 6 files changed, 12 insertions(+), 22 deletions(-)

diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py
index 7704e98bb76..c69691c09c4 100644
--- a/lib/galaxy/model/tool_shed_install/__init__.py
+++ b/lib/galaxy/model/tool_shed_install/__init__.py
@@ -589,13 +589,6 @@ class ToolVersion( object, Dictifiable ):
         self.tool_id = tool_id
         self.tool_shed_repository = tool_shed_repository
 
-    def get_version_ids( self, app, reverse=False ):
-        lineage = app.toolbox._lineage_map.get(self.tool_id)
-        version_ids = lineage.get_versions()
-        if reverse:
-            version_ids = reversed(version_ids)
-        return version_ids
-
     def to_dict( self, view='element' ):
         rval = super( ToolVersion, self ).to_dict( view=view )
         rval[ 'tool_name' ] = self.tool_id
diff --git a/lib/galaxy/tools/toolbox/lineages/factory.py b/lib/galaxy/tools/toolbox/lineages/factory.py
index 0e9080039eb..e97f08dc714 100644
--- a/lib/galaxy/tools/toolbox/lineages/factory.py
+++ b/lib/galaxy/tools/toolbox/lineages/factory.py
@@ -13,7 +13,7 @@ class LineageMap(object):
 
     def register(self, tool):
         tool_id = tool.id
-        versionless_tool_id = remove_version_from_guid( tool_id)
+        versionless_tool_id = remove_version_from_guid(tool_id)
         lineage = self.lineage_map.get(versionless_tool_id)
         if not lineage:
             lineage = StockLineage.from_tool( tool )
diff --git a/lib/galaxy/tools/toolbox/lineages/stock.py b/lib/galaxy/tools/toolbox/lineages/stock.py
index 44850cb35f7..92d4f377ea9 100644
--- a/lib/galaxy/tools/toolbox/lineages/stock.py
+++ b/lib/galaxy/tools/toolbox/lineages/stock.py
@@ -47,7 +47,9 @@ class StockLineage(ToolLineage):
     def get_versions( self ):
         return [ ToolLineageVersion( tool_id, tool_version ) for tool_id, tool_version in zip(self.tool_ids, self.tool_versions) ]
 
-    def get_version_ids(self):
+    def get_version_ids(self, reverse=False):
+        if reverse:
+            return list(reversed(self.tool_ids))
         return self.tool_ids
 
     def to_dict(self):
diff --git a/lib/galaxy/util/tool_version.py b/lib/galaxy/util/tool_version.py
index 3b602a23a04..5442b0a2c7f 100644
--- a/lib/galaxy/util/tool_version.py
+++ b/lib/galaxy/util/tool_version.py
@@ -1,5 +1,4 @@
 def remove_version_from_guid(guid):
-
     """
     Removes version from toolshed-derived tool_id(=guid).
     """
diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py
index b79fedbb64b..1cfeadd0a55 100644
--- a/lib/galaxy/webapps/galaxy/controllers/admin.py
+++ b/lib/galaxy/webapps/galaxy/controllers/admin.py
@@ -458,15 +458,13 @@ class ToolVersionListGrid( grids.Grid ):
             tool = toolbox._tools_by_id.get(tool_version.tool_id)
             if tool:
                 tool_id = tool.id
-                lineage = trans.app.toolbox._lineage_map.get(tool_id)
-                if lineage:
-                    for tool_id in lineage.tool_ids:
-                        if toolbox.has_tool( tool_id, exact=True ):
-                            link = url_for( controller='tool_runner', tool_id=tool_id )
-                            link_str = '<a target="_blank" href="%s">' % link
-                            tool_ids_str += '<div class="count-box state-color-ok">%s%s</a></div><br/>' % ( link_str, tool_id )
-                        else:
-                            tool_ids_str += '%s<br/>' % tool_version.tool_id
+                for tool_id in tool.lineage.tool_ids:
+                    if toolbox.has_tool( tool_id, exact=True ):
+                        link = url_for( controller='tool_runner', tool_id=tool_id )
+                        link_str = '<a target="_blank" href="%s">' % link
+                        tool_ids_str += '<div class="count-box state-color-ok">%s%s</a></div><br/>' % ( link_str, tool_id )
+                    else:
+                        tool_ids_str += '%s<br/>' % tool_version.tool_id
             else:
                 tool_ids_str += '%s<br/>' % tool_version.tool_id
             return tool_ids_str
diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
index b02553419a5..5f9be86e152 100644
--- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
+++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
@@ -2061,9 +2061,7 @@ class AdminToolshed( AdminGalaxy ):
                         tool_config = os.path.join( shed_config_dict.get( 'tool_path' ), tool_config )
                     tool = trans.app.toolbox.load_tool( os.path.abspath( tool_config ), guid=tool_metadata[ 'guid' ] )
                     if tool:
-                        tvm = tool_version_manager.ToolVersionManager( trans.app )
-                        tool_version = tvm.get_tool_version( str( tool.id ) )
-                        tool_lineage = tool_version.get_version_ids( trans.app, reverse=True )
+                        tool_lineage = tool.tool_version.get_version_ids(reverse=True)
                     break
         return trans.fill_template( "/admin/tool_shed_repository/view_tool_metadata.mako",
                                     repository=repository,

From 0830ea2e4172e3be51ae081f9578c2c582f12a7e Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 22 Jun 2017 01:16:16 +0200
Subject: [PATCH 08/14] Remove unused `get_version_from_guid` function

---
 lib/galaxy/util/tool_version.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/lib/galaxy/util/tool_version.py b/lib/galaxy/util/tool_version.py
index 5442b0a2c7f..03ef6c7a3b3 100644
--- a/lib/galaxy/util/tool_version.py
+++ b/lib/galaxy/util/tool_version.py
@@ -6,13 +6,3 @@ def remove_version_from_guid(guid):
         return None
     last_slash = guid.rfind('/')
     return guid[:last_slash]
-
-
-def get_version_from_guid(guid):
-    """
-    Removes version from toolshed-derived tool_id(=guid).
-    """
-    if "/" not in guid:
-        return None
-    last_slash = guid.rfind('/')
-    return guid[last_slash + 1:]

From db5e41c186d4c4b452ee4b7e83a0b26474d72873 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 22 Jun 2017 11:02:00 +0200
Subject: [PATCH 09/14] Remove unused tool_id variable

---
 lib/galaxy/webapps/galaxy/controllers/admin.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py
index 1cfeadd0a55..228b4d88b77 100644
--- a/lib/galaxy/webapps/galaxy/controllers/admin.py
+++ b/lib/galaxy/webapps/galaxy/controllers/admin.py
@@ -457,7 +457,6 @@ class ToolVersionListGrid( grids.Grid ):
             toolbox = trans.app.toolbox
             tool = toolbox._tools_by_id.get(tool_version.tool_id)
             if tool:
-                tool_id = tool.id
                 for tool_id in tool.lineage.tool_ids:
                     if toolbox.has_tool( tool_id, exact=True ):
                         link = url_for( controller='tool_runner', tool_id=tool_id )

From 0257248152db3edb493bfd4420f105a19000ae89 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 22 Jun 2017 11:21:36 +0200
Subject: [PATCH 10/14] Rename StockLineage to ToolLineage and merge stock.py
 contents into interface.py

---
 lib/galaxy/tools/__init__.py                  |  2 +-
 lib/galaxy/tools/toolbox/lineages/factory.py  |  6 +-
 .../tools/toolbox/lineages/interface.py       | 75 +++++++++++++++----
 lib/galaxy/tools/toolbox/lineages/stock.py    | 60 ---------------
 4 files changed, 63 insertions(+), 80 deletions(-)
 delete mode 100644 lib/galaxy/tools/toolbox/lineages/stock.py

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 818df6d8e0a..1d74d27d449 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -441,7 +441,7 @@ class Tool( object, Dictifiable ):
 
     @property
     def tool_version( self ):
-        """Return a StockLineage if one exists for our id."""
+        """Return a ToolLineage if one exists for our id."""
         return self.app.toolbox._lineage_map.get(self.id)
 
     @property
diff --git a/lib/galaxy/tools/toolbox/lineages/factory.py b/lib/galaxy/tools/toolbox/lineages/factory.py
index e97f08dc714..abe5e81177c 100644
--- a/lib/galaxy/tools/toolbox/lineages/factory.py
+++ b/lib/galaxy/tools/toolbox/lineages/factory.py
@@ -1,6 +1,6 @@
 from galaxy.util.tool_version import remove_version_from_guid
 
-from .stock import StockLineage
+from .interface import ToolLineage
 
 
 class LineageMap(object):
@@ -16,7 +16,7 @@ class LineageMap(object):
         versionless_tool_id = remove_version_from_guid(tool_id)
         lineage = self.lineage_map.get(versionless_tool_id)
         if not lineage:
-            lineage = StockLineage.from_tool( tool )
+            lineage = ToolLineage.from_tool( tool )
         if versionless_tool_id and versionless_tool_id not in self.lineage_map:
             self.lineage_map[versionless_tool_id] = lineage
         if tool_id not in self.lineage_map:
@@ -38,7 +38,7 @@ class LineageMap(object):
         if tool_id not in self.lineage_map:
             tool = self.app.toolbox._tools_by_id.get(tool_id)
             if tool:
-                lineage = StockLineage.from_tool( tool )
+                lineage = ToolLineage.from_tool( tool )
             if lineage:
                 self.lineage_map[tool_id] = lineage
         return self.lineage_map.get(tool_id)
diff --git a/lib/galaxy/tools/toolbox/lineages/interface.py b/lib/galaxy/tools/toolbox/lineages/interface.py
index a36fe5445a8..c97657070b9 100644
--- a/lib/galaxy/tools/toolbox/lineages/interface.py
+++ b/lib/galaxy/tools/toolbox/lineages/interface.py
@@ -1,21 +1,8 @@
-from abc import (
-    ABCMeta,
-    abstractmethod
-)
+import threading
 
-import six
+from distutils.version import LooseVersion
 
-
-@six.add_metaclass(ABCMeta)
-class ToolLineage(object):
-    """
-    """
-
-    @abstractmethod
-    def get_versions( self ):
-        """ Return an ordered list of lineages (ToolLineageVersion) in this
-        chain, from oldest to newest.
-        """
+from galaxy.util.tool_version import remove_version_from_guid
 
 
 class ToolLineageVersion(object):
@@ -50,3 +37,59 @@ class ToolLineageVersion(object):
             id=self.id,
             version=self.version,
         )
+
+
+class ToolLineage:
+    """ Simple tool's loaded directly from file system with lineage
+    determined solely by distutil's LooseVersion naming scheme.
+    """
+    lineages_by_id = {}
+    lock = threading.Lock()
+
+    def __init__(self, tool_id, **kwds):
+        self.tool_id = tool_id
+        self._tool_versions = set()
+
+    @property
+    def tool_versions(self):
+        return sorted(self._tool_versions, key=LooseVersion)
+
+    @property
+    def tool_ids(self):
+        versionless_tool_id = remove_version_from_guid(self.tool_id)
+        tool_id = versionless_tool_id or self.tool_id
+        return ["%s/%s" % (tool_id, version) for version in self.tool_versions]
+
+    @staticmethod
+    def from_tool( tool ):
+        tool_id = tool.id
+        lineages_by_id = ToolLineage.lineages_by_id
+        with ToolLineage.lock:
+            if tool_id not in lineages_by_id:
+                lineages_by_id[ tool_id ] = ToolLineage( tool_id )
+        lineage = lineages_by_id[ tool_id ]
+        lineage.register_version( tool.version )
+        return lineage
+
+    def register_version( self, tool_version ):
+        assert tool_version is not None
+        self._tool_versions.add( str(tool_version) )
+
+    def get_versions( self ):
+        """
+        Return an ordered list of lineages (ToolLineageVersion) in this
+        chain, from oldest to newest.
+        """
+        return [ ToolLineageVersion( tool_id, tool_version ) for tool_id, tool_version in zip(self.tool_ids, self.tool_versions) ]
+
+    def get_version_ids(self, reverse=False):
+        if reverse:
+            return list(reversed(self.tool_ids))
+        return self.tool_ids
+
+    def to_dict(self):
+        return dict(
+            tool_id=self.tool_id,
+            tool_versions=list(self.tool_versions),
+            lineage_type='stock',
+        )
diff --git a/lib/galaxy/tools/toolbox/lineages/stock.py b/lib/galaxy/tools/toolbox/lineages/stock.py
deleted file mode 100644
index 92d4f377ea9..00000000000
--- a/lib/galaxy/tools/toolbox/lineages/stock.py
+++ /dev/null
@@ -1,60 +0,0 @@
-import threading
-
-from distutils.version import LooseVersion
-
-from galaxy.util.tool_version import remove_version_from_guid
-
-from .interface import ToolLineage
-from .interface import ToolLineageVersion
-
-
-class StockLineage(ToolLineage):
-    """ Simple tool's loaded directly from file system with lineage
-    determined solely by distutil's LooseVersion naming scheme.
-    """
-    lineages_by_id = {}
-    lock = threading.Lock()
-
-    def __init__(self, tool_id, **kwds):
-        self.tool_id = tool_id
-        self._tool_versions = set()
-
-    @property
-    def tool_versions(self):
-        return sorted(self._tool_versions, key=LooseVersion)
-
-    @property
-    def tool_ids(self):
-        versionless_tool_id = remove_version_from_guid(self.tool_id)
-        tool_id = versionless_tool_id or self.tool_id
-        return ["%s/%s" % (tool_id, version) for version in self.tool_versions]
-
-    @staticmethod
-    def from_tool( tool ):
-        tool_id = tool.id
-        lineages_by_id = StockLineage.lineages_by_id
-        with StockLineage.lock:
-            if tool_id not in lineages_by_id:
-                lineages_by_id[ tool_id ] = StockLineage( tool_id )
-        lineage = lineages_by_id[ tool_id ]
-        lineage.register_version( tool.version )
-        return lineage
-
-    def register_version( self, tool_version ):
-        assert tool_version is not None
-        self._tool_versions.add( str(tool_version) )
-
-    def get_versions( self ):
-        return [ ToolLineageVersion( tool_id, tool_version ) for tool_id, tool_version in zip(self.tool_ids, self.tool_versions) ]
-
-    def get_version_ids(self, reverse=False):
-        if reverse:
-            return list(reversed(self.tool_ids))
-        return self.tool_ids
-
-    def to_dict(self):
-        return dict(
-            tool_id=self.tool_id,
-            tool_versions=list(self.tool_versions),
-            lineage_type='stock',
-        )

From 68981c16cc708514e879941f152be9a60f7ad290 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 22 Jun 2017 13:30:53 +0200
Subject: [PATCH 11/14] Remove unused lineage_ids from Tool class

---
 lib/galaxy/tools/__init__.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 1d74d27d449..ce40427f973 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -415,8 +415,6 @@ class Tool( object, Dictifiable ):
         self.old_id = None
         self.version = None
         self.dependencies = []
-        # Enable easy access to this tool's version lineage.
-        self.lineage_ids = []
         # populate toolshed repository info, if available
         self.populate_tool_shed_info()
         # add tool resource parameters

From e9658e8adb5d0bf0f8db42faa4d14cb50d4c90ce Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 22 Jun 2017 16:48:20 +0200
Subject: [PATCH 12/14] Clarify when and why tool version is being registered

---
 lib/galaxy/tools/toolbox/lineages/factory.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/toolbox/lineages/factory.py b/lib/galaxy/tools/toolbox/lineages/factory.py
index abe5e81177c..9bfd9aca5c6 100644
--- a/lib/galaxy/tools/toolbox/lineages/factory.py
+++ b/lib/galaxy/tools/toolbox/lineages/factory.py
@@ -17,11 +17,15 @@ class LineageMap(object):
         lineage = self.lineage_map.get(versionless_tool_id)
         if not lineage:
             lineage = ToolLineage.from_tool( tool )
+        else:
+            # A lineage for a tool with the same versionless_tool_id exists,
+            # but this lineage may not have the current tools' version,
+            # so we add tool.version to the lineage
+            lineage.register_version(tool.version)
         if versionless_tool_id and versionless_tool_id not in self.lineage_map:
             self.lineage_map[versionless_tool_id] = lineage
         if tool_id not in self.lineage_map:
             self.lineage_map[tool_id] = lineage
-        lineage.register_version(tool.version)
         return self.lineage_map[tool_id]
 
     def get(self, tool_id):

From 41d739bdb4d1ac622fb7fd78f8b3dbc9acd978ad Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 23 Jun 2017 10:44:59 +0200
Subject: [PATCH 13/14] Move tool.tool_version to tool.lineage

This change makes it obvious that a Lineage object will be returned.

Also removes two unused methods from ToolLineageVersion.
---
 lib/galaxy/tools/__init__.py                    | 17 +++++++++--------
 lib/galaxy/tools/toolbox/base.py                | 13 +++----------
 lib/galaxy/tools/toolbox/lineages/interface.py  |  9 ---------
 .../galaxy/controllers/admin_toolshed.py        |  9 ++++++---
 4 files changed, 18 insertions(+), 30 deletions(-)

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index ce40427f973..e559c8a1f8c 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -414,6 +414,7 @@ class Tool( object, Dictifiable ):
         self.guid = guid
         self.old_id = None
         self.version = None
+        self._lineage = None
         self.dependencies = []
         # populate toolshed repository info, if available
         self.populate_tool_shed_info()
@@ -438,17 +439,17 @@ class Tool( object, Dictifiable ):
         return self.app.model.context
 
     @property
-    def tool_version( self ):
-        """Return a ToolLineage if one exists for our id."""
-        return self.app.toolbox._lineage_map.get(self.id)
+    def lineage(self):
+        """Return ToolLineage for this tool."""
+        return self._lineage
 
     @property
     def tool_versions( self ):
         # If we have versions, return them.
-        tool_version = self.tool_version
-        if tool_version:
-            return tool_version.get_versions( self.app )
-        return []
+        if self.lineage:
+            return self.lineage.get_versions()
+        else:
+            return []
 
     @property
     def tool_shed_repository( self ):
@@ -1830,7 +1831,7 @@ class Tool( object, Dictifiable ):
             tool_help = unicodify( tool_help, 'utf-8' )
 
         # create tool versions
-        tool_versions = self.tool_version.tool_versions
+        tool_versions = self.lineage.tool_versions
 
         # update tool model
         tool_model.update({
diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py
index 2144f08ae4e..5ef80220b02 100644
--- a/lib/galaxy/tools/toolbox/base.py
+++ b/lib/galaxy/tools/toolbox/base.py
@@ -289,9 +289,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
                 inserted = True
             if not inserted:
                 # Check the tool's installed versions.
-                versions = []
-                if hasattr(tool, 'lineage'):
-                    versions = tool.lineage.get_versions()
+                versions = tool.lineage.get_versions()
                 for tool_lineage_version in versions:
                     lineage_id = tool_lineage_version.id
                     index = self._integrated_tool_panel.index_of_tool_id(lineage_id)
@@ -565,9 +563,8 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
                     tool.installed_changeset_revision = tool_shed_repository.installed_changeset_revision
                     tool.guid = guid
                     tool.version = item.elem.find( "version" ).text
-                # Make sure tools have a tool_version object.
-                tool_lineage = self._lineage_map.register( tool )
-                tool.lineage = tool_lineage
+                # Make sure tools are registered in self._lineage_map.
+                tool._lineage = self._lineage_map.register( tool )
                 if item.has_elem:
                     self._tool_tag_manager.handle_tags( tool.id, item.elem )
                 self.__add_tool( tool, load_panel_dict, panel_dict )
@@ -947,8 +944,6 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
         """
         if tool_lineage is None:
             assert tool is not None
-            if not hasattr( tool, "lineage" ):
-                return None
             tool_lineage = tool.lineage
         lineage_tool_versions = reversed(tool_lineage.get_versions())
         for lineage_tool_version in lineage_tool_versions:
@@ -963,8 +958,6 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
         """ Return True if tool1 is considered "newer" given its own lineage
         description.
         """
-        if not hasattr( tool1, "lineage" ):
-            return True
         return tool1.version_object > tool2.version_object
 
     def _tool_from_lineage_version( self, lineage_tool_version ):
diff --git a/lib/galaxy/tools/toolbox/lineages/interface.py b/lib/galaxy/tools/toolbox/lineages/interface.py
index c97657070b9..6a691a359e8 100644
--- a/lib/galaxy/tools/toolbox/lineages/interface.py
+++ b/lib/galaxy/tools/toolbox/lineages/interface.py
@@ -14,15 +14,6 @@ class ToolLineageVersion(object):
         self.id = id
         self.version = version
 
-    @staticmethod
-    def from_id_and_verion( id, version ):
-        assert version is not None
-        return ToolLineageVersion( id, version )
-
-    @staticmethod
-    def from_guid( guid ):
-        return ToolLineageVersion( guid, None )
-
     @property
     def id_based( self ):
         """ Return True if the lineage is defined by GUIDs (in this
diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
index 5f9be86e152..cd4124afb12 100644
--- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
+++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
@@ -2059,9 +2059,12 @@ class AdminToolshed( AdminGalaxy ):
                     tool_config = tool_metadata[ 'tool_config' ]
                     if shed_config_dict and shed_config_dict.get( 'tool_path' ):
                         tool_config = os.path.join( shed_config_dict.get( 'tool_path' ), tool_config )
-                    tool = trans.app.toolbox.load_tool( os.path.abspath( tool_config ), guid=tool_metadata[ 'guid' ] )
-                    if tool:
-                        tool_lineage = tool.tool_version.get_version_ids(reverse=True)
+                    tool = trans.app.toolbox.get_tool(tool_id=tool_metadata[ 'guid' ], exact=True)
+                    if not tool:
+                        tool = trans.app.toolbox.load_tool( os.path.abspath( tool_config ), guid=tool_metadata[ 'guid' ] )
+                        if tool:
+                            tool._lineage = trans.app.toolbox._lineage_map.register( tool )
+                    tool_lineage = tool.lineage.get_version_ids(reverse=True)
                     break
         return trans.fill_template( "/admin/tool_shed_repository/view_tool_metadata.mako",
                                     repository=repository,

From 84e0e7e01e732b28dedebdf8a3fbea8225fece9a Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sat, 24 Jun 2017 13:59:16 +0200
Subject: [PATCH 14/14] Access tool.lineage only if tool was loaded

---
 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
index cd4124afb12..886c95cac27 100644
--- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
+++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
@@ -2064,7 +2064,8 @@ class AdminToolshed( AdminGalaxy ):
                         tool = trans.app.toolbox.load_tool( os.path.abspath( tool_config ), guid=tool_metadata[ 'guid' ] )
                         if tool:
                             tool._lineage = trans.app.toolbox._lineage_map.register( tool )
-                    tool_lineage = tool.lineage.get_version_ids(reverse=True)
+                    if tool:
+                        tool_lineage = tool.lineage.get_version_ids(reverse=True)
                     break
         return trans.fill_template( "/admin/tool_shed_repository/view_tool_metadata.mako",
                                     repository=repository,