opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-21 13:50:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15467 from jdavcs/dev_toolbox_refactor

Browse Source 
Move database access code out of tool_util
This commit is contained in:
John Chilton
2023-02-01 09:22:10 -05:00
committed by GitHub
parent ffa2fb3e92 d91db472b8
commit dc9e603ab3
4 changed files with 144 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/tool_util/toolbox/__init__.py
+2 -2
View File
@@ -2,7 +2,7 @@
from .base import (
AbstractToolBox,
BaseGalaxyToolBox,
AbstractToolTagManager,
)
from .panel import (
panel_item_types,
@@ -12,7 +12,7 @@ from .panel import (
__all__ = (
"AbstractToolBox",
"BaseGalaxyToolBox",
"AbstractToolTagManager",
"panel_item_types",
"ToolSection",
"ToolSectionLabel",
lib/galaxy/tool_util/toolbox/base.py
+18 -85
View File
@@ -24,11 +24,6 @@ from galaxy.exceptions import (
ObjectNotFound,
RequestParameterInvalidException,
)
from galaxy.tool_util.deps import (
build_dependency_manager,
NullDependencyManager,
)
from galaxy.tool_util.loader_directory import looks_like_a_tool
from galaxy.util import (
etree,
ExecutionTimer,
@@ -52,7 +47,6 @@ from .parser import (
ensure_tool_conf_item,
get_toolbox_parser,
)
from .tags import tool_tag_manager
from .views.edam import (
EdamPanelMode,
EdamToolPanelView,
@@ -116,6 +110,16 @@ class ToolBoxRegistryImpl(ToolBoxRegistry):
DynamicToolConfDict = Dict[str, Any]
class AbstractToolTagManager(metaclass=abc.ABCMeta):
@abc.abstractmethod
def reset_tags(self):
"""Starting to load tool panels, reset all tags."""
@abc.abstractmethod
def handle_tags(self, tool_id, tool_definition_source):
"""Parse out tags and persist them."""
class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
"""
Abstract container for managing a ToolPanel - containing tools and
@@ -167,7 +171,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
self._tool_watcher = self.app.watchers.tool_watcher
self._tool_config_watcher = self.app.watchers.tool_config_watcher
self._filter_factory = FilterFactory(self)
self._tool_tag_manager = tool_tag_manager(app)
self._tool_tag_manager = self.tool_tag_manager()
self._init_tools_from_configs(config_filenames)
if self.app.name == "galaxy" and self._integrated_tool_panel_config_has_contents:
@@ -228,6 +232,13 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
def can_load_config_file(self, config_filename):
return True
def _load_workflow(self, workflow_id):
raise NotImplementedError()
def tool_tag_manager(self):
"""Build a tool tag manager according to app's configuration and return it."""
raise NotImplementedError()
def _init_tools_from_configs(self, config_filenames):
"""Read through all tool config files and initialize tools in each
with init_tools_from_config below.
@@ -1245,15 +1256,6 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
tool_panel_section_id = ""
return tool_panel_section_id
def _load_workflow(self, workflow_id):
"""
Return an instance of 'Workflow' identified by `id`,
which is encoded in the tool panel.
"""
id = self.app.security.decode_id(workflow_id)
stored = self.app.model.context.query(self.app.model.StoredWorkflow).get(id)
return stored.latest_workflow
def tool_panel_contents(self, trans, view=None, **kwds):
"""Filter tool_panel contents for displaying for user."""
if view is None:
@@ -1410,72 +1412,3 @@ def _filter_for_panel(item, item_type, filters, context):
return copy
return None
class BaseGalaxyToolBox(AbstractToolBox):
"""
Extend the AbstractToolBox with more Galaxy tooling-specific
functionality. Adds dependencies on dependency resolution and
tool loading modules, that an abstract description of panels
shouldn't really depend on.
"""
def __init__(
self,
config_filenames,
tool_root_dir,
app,
view_sources=None,
default_panel_view=None,
save_integrated_tool_panel=True,
):
super().__init__(
config_filenames, tool_root_dir, app, view_sources, default_panel_view, save_integrated_tool_panel
)
old_toolbox = getattr(app, "toolbox", None)
if old_toolbox:
self.dependency_manager = old_toolbox.dependency_manager
else:
self._init_dependency_manager()
@property
def sa_session(self):
"""
Returns a SQLAlchemy session
"""
return self.app.model.context
def _looks_like_a_tool(self, path):
return looks_like_a_tool(path, enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False))
def _init_dependency_manager(self):
use_tool_dependency_resolution = getattr(self.app, "use_tool_dependency_resolution", True)
if not use_tool_dependency_resolution:
self.dependency_manager = NullDependencyManager()
return
app_config_dict = self.app.config.config_dict
conf_file = app_config_dict.get("dependency_resolvers_config_file")
default_tool_dependency_dir = os.path.join(
self.app.config.data_dir, self.app.config.schema.defaults["tool_dependency_dir"]
)
self.dependency_manager = build_dependency_manager(
app_config_dict=app_config_dict,
conf_file=conf_file,
default_tool_dependency_dir=default_tool_dependency_dir,
)
def reload_dependency_manager(self):
self._init_dependency_manager()
def load_builtin_converters(self):
id = "builtin_converters"
section = ToolSection({"name": "Built-in Converters", "id": id})
self._tool_panel[id] = section
converters = self.app.datatypes_registry.datatype_converters
for source, targets in converters.items():
for target, tool in targets.items():
tool.name = f"{source}-to-{target}"
tool.description = "converter"
tool.hidden = False
section.elems.append_tool(tool)
lib/galaxy/tool_util/toolbox/tags.py
-74
View File
@@ -1,74 +0,0 @@
""" Handle details of tool tagging - perhaps a deprecated feature.
"""
import logging
from abc import (
ABCMeta,
abstractmethod,
)
log = logging.getLogger(__name__)
def tool_tag_manager(app):
"""Build a tool tag manager according to app's configuration
and return it.
"""
if hasattr(app.config, "get_bool") and app.config.get_bool("enable_tool_tags", False):
return PersistentToolTagManager(app)
else:
return NullToolTagManager()
class AbstractToolTagManager(metaclass=ABCMeta):
@abstractmethod
def reset_tags(self):
"""Starting to load tool panels, reset all tags."""
@abstractmethod
def handle_tags(self, tool_id, tool_definition_source):
"""Parse out tags and persist them."""
class NullToolTagManager(AbstractToolTagManager):
def reset_tags(self):
return None
def handle_tags(self, tool_id, tool_definition_source):
return None
class PersistentToolTagManager(AbstractToolTagManager):
def __init__(self, app):
self.app = app
self.sa_session = app.model.context
def reset_tags(self):
log.info(
f"removing all tool tag associations ({str(self.sa_session.query(self.app.model.ToolTagAssociation).count())})"
)
self.sa_session.query(self.app.model.ToolTagAssociation).delete()
self.sa_session.flush()
def handle_tags(self, tool_id, tool_definition_source):
elem = tool_definition_source
if self.app.config.get_bool("enable_tool_tags", False):
tag_names = elem.get("tags", "").split(",")
for tag_name in tag_names:
if tag_name == "":
continue
tag = self.sa_session.query(self.app.model.Tag).filter_by(name=tag_name).first()
if not tag:
tag = self.app.model.Tag(name=tag_name)
self.sa_session.add(tag)
self.sa_session.flush()
tta = self.app.model.ToolTagAssociation(tool_id=tool_id, tag_id=tag.id)
self.sa_session.add(tta)
self.sa_session.flush()
else:
for tagged_tool in tag.tagged_tools:
if tagged_tool.tool_id == tool_id:
break
else:
tta = self.app.model.ToolTagAssociation(tool_id=tool_id, tag_id=tag.id)
self.sa_session.add(tta)
self.sa_session.flush()
lib/galaxy/tools/__init__.py
+124 -5
View File
@@ -39,13 +39,18 @@ from galaxy.job_execution import output_collect
from galaxy.metadata import get_metadata_compute_strategy
from galaxy.tool_shed.util.repository_util import get_installed_repository
from galaxy.tool_shed.util.shed_util_common import set_image_paths
from galaxy.tool_util.deps import CachedDependencyManager
from galaxy.tool_util.deps import (
build_dependency_manager,
CachedDependencyManager,
NullDependencyManager,
)
from galaxy.tool_util.fetcher import ToolLocationFetcher
from galaxy.tool_util.loader import (
imported_macro_paths,
raw_tool_xml_tree,
template_macro_params,
)
from galaxy.tool_util.loader_directory import looks_like_a_tool
from galaxy.tool_util.ontologies.ontology_data import (
biotools_reference,
expand_ontology_data,
@@ -67,7 +72,11 @@ from galaxy.tool_util.parser.xml import (
XmlToolSource,
)
from galaxy.tool_util.provided_metadata import parse_tool_provided_metadata
from galaxy.tool_util.toolbox import BaseGalaxyToolBox
from galaxy.tool_util.toolbox import (
AbstractToolBox,
AbstractToolTagManager,
ToolSection,
)
from galaxy.tool_util.toolbox.views.sources import StaticToolBoxViewSources
from galaxy.tools import expressions
from galaxy.tools.actions import (
@@ -314,9 +323,56 @@ def create_tool_from_representation(
return create_tool_from_source(app, tool_source=tool_source, tool_dir=tool_dir)
class ToolBox(BaseGalaxyToolBox):
"""A derivative of AbstractToolBox with knowledge about Tool internals -
how to construct them, action types, dependency management, etc....
class NullToolTagManager(AbstractToolTagManager):
def reset_tags(self):
return None
def handle_tags(self, tool_id, tool_definition_source):
return None
class PersistentToolTagManager(AbstractToolTagManager):
def __init__(self, app):
self.app = app
self.sa_session = app.model.context
def reset_tags(self):
log.info(
f"removing all tool tag associations ({str(self.sa_session.query(self.app.model.ToolTagAssociation).count())})"
)
self.sa_session.query(self.app.model.ToolTagAssociation).delete()
self.sa_session.flush()
def handle_tags(self, tool_id, tool_definition_source):
elem = tool_definition_source
if self.app.config.get_bool("enable_tool_tags", False):
tag_names = elem.get("tags", "").split(",")
for tag_name in tag_names:
if tag_name == "":
continue
tag = self.sa_session.query(self.app.model.Tag).filter_by(name=tag_name).first()
if not tag:
tag = self.app.model.Tag(name=tag_name)
self.sa_session.add(tag)
self.sa_session.flush()
tta = self.app.model.ToolTagAssociation(tool_id=tool_id, tag_id=tag.id)
self.sa_session.add(tta)
self.sa_session.flush()
else:
for tagged_tool in tag.tagged_tools:
if tagged_tool.tool_id == tool_id:
break
else:
tta = self.app.model.ToolTagAssociation(tool_id=tool_id, tag_id=tag.id)
self.sa_session.add(tta)
self.sa_session.flush()
class ToolBox(AbstractToolBox):
"""
A derivative of AbstractToolBox with Galaxy tooling-specific functionality
and knowledge about Tool internals - how to construct them, action types,
dependency management, etc.
"""
def __init__(self, config_filenames, tool_root_dir, app, save_integrated_tool_panel=True):
@@ -333,6 +389,7 @@ class ToolBox(BaseGalaxyToolBox):
view_dicts=app.config.panel_views,
)
default_panel_view = app.config.default_panel_view
super().__init__(
config_filenames=config_filenames,
tool_root_dir=tool_root_dir,
@@ -342,6 +399,40 @@ class ToolBox(BaseGalaxyToolBox):
save_integrated_tool_panel=save_integrated_tool_panel,
)
old_toolbox = getattr(app, "toolbox", None)
if old_toolbox:
self.dependency_manager = old_toolbox.dependency_manager
else:
self._init_dependency_manager()
def tool_tag_manager(self):
if hasattr(self.app.config, "get_bool") and self.app.config.get_bool("enable_tool_tags", False):
return PersistentToolTagManager(self.app)
else:
return NullToolTagManager()
@property
def sa_session(self):
"""
Returns a SQLAlchemy session
"""
return self.app.model.context
def reload_dependency_manager(self):
self._init_dependency_manager()
def load_builtin_converters(self):
id = "builtin_converters"
section = ToolSection({"name": "Built-in Converters", "id": id})
self._tool_panel[id] = section
converters = self.app.datatypes_registry.datatype_converters
for source, targets in converters.items():
for target, tool in targets.items():
tool.name = f"{source}-to-{target}"
tool.description = "converter"
tool.hidden = False
section.elems.append_tool(tool)
def persist_cache(self, register_postfork=False):
"""
Persists any modified tool cache files to disk.
@@ -490,6 +581,34 @@ class ToolBox(BaseGalaxyToolBox):
from_cache=True,
)
def _looks_like_a_tool(self, path):
return looks_like_a_tool(path, enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False))
def _init_dependency_manager(self):
use_tool_dependency_resolution = getattr(self.app, "use_tool_dependency_resolution", True)
if not use_tool_dependency_resolution:
self.dependency_manager = NullDependencyManager()
return
app_config_dict = self.app.config.config_dict
conf_file = app_config_dict.get("dependency_resolvers_config_file")
default_tool_dependency_dir = os.path.join(
self.app.config.data_dir, self.app.config.schema.defaults["tool_dependency_dir"]
)
self.dependency_manager = build_dependency_manager(
app_config_dict=app_config_dict,
conf_file=conf_file,
default_tool_dependency_dir=default_tool_dependency_dir,
)
def _load_workflow(self, workflow_id):
"""
Return an instance of 'Workflow' identified by `id`,
which is encoded in the tool panel.
"""
id = self.app.security.decode_id(workflow_id)
stored = self.app.model.context.query(self.app.model.StoredWorkflow).get(id)
return stored.latest_workflow
def __build_tool_version_select_field(self, tools, tool_id, set_selected):
"""Build a SelectField whose options are the ids for the received list of tools."""
options: List[Tuple[str, str]] = []