From 0e509a98ed79daf250fdd486f827d14335e88486 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 8 Jan 2017 18:46:10 +0100 Subject: [PATCH 1/2] Create merged environments at install time Currently tested with UI only --- lib/galaxy/tools/deps/__init__.py | 13 +++++++++++++ lib/galaxy/tools/deps/resolvers/conda.py | 11 ++++++++++- lib/galaxy/tools/deps/views.py | 5 +++++ lib/galaxy/webapps/galaxy/api/tools.py | 6 ++---- lib/tool_shed/galaxy_install/install_manager.py | 15 ++++++--------- 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 117b5b49b87..902b4124efe 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -118,6 +118,19 @@ class DependencyManager( object ): return requirement_to_dependency + def install_all(self, requirements): + """ + Attempt to install all requirements at once. + If sucessfull returns True, else returns False + """ + for resolver in self.dependency_resolvers: + if hasattr(resolver, 'get_targets') and hasattr(resolver, 'install_all'): + targets = resolver.get_targets(requirements) + is_installed = resolver.install_all(targets) + if is_installed: + return True + return False + def _requirements_to_dependencies_dict(self, requirements, **kwds): """Build simple requirements to dependencies dict for resolution.""" requirement_to_dependency = OrderedDict() diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index 56e368db41b..e87c1727cc4 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -120,7 +120,7 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In return is_installed - def resolve_all(self, requirements, **kwds): + def _get_conda_targets(self, requirements): if len(requirements) == 0: return False @@ -138,6 +138,15 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In version = None conda_targets.append(CondaTarget(requirement.name, version=version)) + return conda_targets + + def get_targets(self, requirements): + return self._get_conda_targets(requirements) + + def resolve_all(self, requirements, **kwds): + conda_targets = self.get_targets(requirements) + if not conda_targets: + return False preserve_python_environment = kwds.get("preserve_python_environment", False) diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py index 8f23335d7df..e35235631cb 100644 --- a/lib/galaxy/tools/deps/views.py +++ b/lib/galaxy/tools/deps/views.py @@ -45,6 +45,11 @@ class DependencyResolversView(object): def resolver_dependency(self, index, **kwds): return self._dependency(**kwds) + def install_dependencies(self, index, requirements): + is_installed = self._dependency_manager.install_all(requirements) + if not is_installed: + return [self.install_dependency(index, req) for req in requirements] + def install_dependency(self, index=None, **payload): """ Installs dependency using highest priority resolver that supports dependency installation diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index b76dd9a3f40..efa6314f5de 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -141,11 +141,9 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): force_rebuild: If true and chache dir exists, attempts to delete cache dir """ tool = self._get_tool(id) - [tool._view.install_dependency(id=None, **req.to_dict()) for req in tool.requirements] - if kwds.get('build_dependency_cache'): - tool.build_dependency_cache(**kwds) + tool._view.install_dependencies(None, tool.requirements) # TODO: rework resolver install system to log and report what has been done. - # _view.install_dependency should return a dict with stdout, stderr and success status + # _view.install_dependencies should return a dict with stdout, stderr and success status return tool.tool_requirements_status @expose_api diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index 7da8c6e40e7..d79383b7a2c 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -905,15 +905,12 @@ class InstallRepositoryManager( object ): if 'tools' in metadata and install_resolver_dependencies: self.update_tool_shed_repository_status( tool_shed_repository, self.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) - requirements = suc.get_unique_requirements_from_repository(tool_shed_repository) - [self._view.install_dependency(id=None, **req) for req in requirements] - if self.app.config.use_cached_dependency_manager: - cached_requirements = [] - for tool_d in metadata['tools']: - tool = self.app.toolbox._tools_by_id.get(tool_d['guid'], None) - if tool and tool.requirements not in cached_requirements: - cached_requirements.append(tool.requirements) - tool.build_dependency_cache() + installed_requirements = [] + for tool_d in metadata['tools']: + tool = self.app.toolbox._tools_by_id.get(tool_d['guid'], None) + if tool and tool.requirements not in installed_requirements: + installed_requirements.append(tool.requirements) + self._view.install_dependencies(None, tool.requirements) if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata: work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itsr" ) # Install tool dependencies. From ad92bbf9bebafed22395e0199efd35a838ac1f4a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 8 Jan 2017 19:15:10 +0100 Subject: [PATCH 2/2] Restore the cached environments for now ... better to completely eliminate them later --- lib/galaxy/webapps/galaxy/api/tools.py | 2 ++ lib/tool_shed/galaxy_install/install_manager.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index efa6314f5de..d9e15895a78 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -142,6 +142,8 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): """ tool = self._get_tool(id) tool._view.install_dependencies(None, tool.requirements) + if kwds.get('build_dependency_cache'): + tool.build_dependency_cache(**kwds) # TODO: rework resolver install system to log and report what has been done. # _view.install_dependencies should return a dict with stdout, stderr and success status return tool.tool_requirements_status diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index d79383b7a2c..5f668105a18 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -911,6 +911,8 @@ class InstallRepositoryManager( object ): if tool and tool.requirements not in installed_requirements: installed_requirements.append(tool.requirements) self._view.install_dependencies(None, tool.requirements) + if self.app.config.use_cached_dependency_manager: + tool.build_dependency_cache() if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata: work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itsr" ) # Install tool dependencies.