mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Merge pull request #14617 from jmchilton/remove_propertiary_datatypes
Remove legacy tool shed datatype handling
This commit is contained in:
Björn Grüning
+1
-14
@@ -327,19 +327,9 @@ class ConfiguresGalaxyMixin:
|
||||
if exc.errno != errno.ENOENT or self.config.is_set("shed_tool_data_table_config"):
|
||||
raise
|
||||
|
||||
def _configure_datatypes_registry(
|
||||
self, installed_repository_manager=None, use_display_applications=True, use_converters=True
|
||||
):
|
||||
def _configure_datatypes_registry(self, use_display_applications=True, use_converters=True):
|
||||
# Create an empty datatypes registry.
|
||||
self.datatypes_registry = Registry(self.config)
|
||||
if installed_repository_manager and self.config.load_tool_shed_datatypes:
|
||||
# Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories. We
|
||||
# load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some
|
||||
# generic sniffers (eg text,xml) which catch anything, so it's impossible for proprietary sniffers to be used.
|
||||
# However, if there is a conflict (2 datatypes with the same extension) between a proprietary datatype and a datatype
|
||||
# in the Galaxy distribution, the datatype in the Galaxy distribution will take precedence. If there is a conflict
|
||||
# between 2 proprietary datatypes, the datatype from the repository that was installed earliest will take precedence.
|
||||
installed_repository_manager.load_proprietary_datatypes()
|
||||
# Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config.
|
||||
datatypes_configs = self.config.datatypes_config
|
||||
for datatypes_config in listify(datatypes_configs):
|
||||
@@ -583,7 +573,6 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
|
||||
)
|
||||
self.dynamic_tool_manager = self._register_singleton(DynamicToolManager)
|
||||
self._configure_datatypes_registry(
|
||||
self.installed_repository_manager,
|
||||
use_converters=use_converters,
|
||||
use_display_applications=use_display_applications,
|
||||
)
|
||||
@@ -659,8 +648,6 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
|
||||
self.update_repository_manager = self._register_singleton(
|
||||
UpdateRepositoryManager, UpdateRepositoryManager(self)
|
||||
)
|
||||
# Load proprietary datatype converters and display applications.
|
||||
self.installed_repository_manager.load_proprietary_converters_and_display_applications()
|
||||
# Load datatype display applications defined in local datatypes_conf.xml
|
||||
self.datatypes_registry.load_display_applications(self)
|
||||
# Load datatype converters defined in local datatypes_conf.xml
|
||||
|
||||
@@ -466,15 +466,6 @@ galaxy:
|
||||
# <config_dir>.
|
||||
#tool_sheds_config_file: tool_sheds_conf.xml
|
||||
|
||||
# This option controls whether legacy datatypes are loaded from
|
||||
# installed tool shed repositories. We're are in the process of
|
||||
# disabling Tool Shed datatypes. This option with a default of true
|
||||
# will be added in 22.01, we will disable the datatypes on the big
|
||||
# public servers during that release. This option will be switched to
|
||||
# False by default in 22.05 and this broken functionality will be
|
||||
# removed all together during some future release.
|
||||
#load_tool_shed_datatypes: true
|
||||
|
||||
# Monitor the tools and tool directories listed in any tool config
|
||||
# file specified in tool_config_file option. If changes are found,
|
||||
# tools are automatically reloaded. Watchdog (
|
||||
|
||||
@@ -443,17 +443,6 @@ mapping:
|
||||
File containing the Galaxy Tool Sheds that should be made available to
|
||||
install from in the admin interface (.sample used if default does not exist).
|
||||
|
||||
load_tool_shed_datatypes:
|
||||
type: bool
|
||||
default: true
|
||||
required: false
|
||||
desc: |
|
||||
This option controls whether legacy datatypes are loaded from installed tool shed repositories.
|
||||
We're are in the process of disabling Tool Shed datatypes. This option with a default of true
|
||||
will be added in 22.01, we will disable the datatypes on the big public servers during that
|
||||
release. This option will be switched to False by default in 22.05 and this broken functionality
|
||||
will be removed all together during some future release.
|
||||
|
||||
watch_tools:
|
||||
type: str
|
||||
default: 'false'
|
||||
|
||||
+237
-457
@@ -68,8 +68,6 @@ class Registry:
|
||||
# Converters defined in local datatypes_conf.xml
|
||||
self.converters = []
|
||||
self.converter_tools = set()
|
||||
# Converters defined in datatypes_conf.xml included in installed tool shed repositories.
|
||||
self.proprietary_converters = []
|
||||
self.converter_deps = {}
|
||||
self.available_tracks = []
|
||||
self.set_external_metadata_tool = None
|
||||
@@ -77,9 +75,6 @@ class Registry:
|
||||
self.upload_file_formats = []
|
||||
# Datatype elements defined in local datatypes_conf.xml that contain display applications.
|
||||
self.display_app_containers = []
|
||||
# Datatype elements in datatypes_conf.xml included in installed
|
||||
# tool shed repositories that contain display applications.
|
||||
self.proprietary_display_app_containers = []
|
||||
# Map a display application id to a display application
|
||||
self.display_applications: Dict[str, DisplayApplication] = {}
|
||||
# The following 2 attributes are used in the to_xml_file()
|
||||
@@ -91,8 +86,6 @@ class Registry:
|
||||
# The 'default' display_path defined in local datatypes_conf.xml
|
||||
self.display_applications_path = None
|
||||
self.inherit_display_application_by_class = []
|
||||
# Keep a list of imported proprietary datatype class modules.
|
||||
self.imported_modules = []
|
||||
self.datatype_elems = []
|
||||
self.datatype_info_dicts = []
|
||||
self.sniffer_elems = []
|
||||
@@ -109,7 +102,6 @@ class Registry:
|
||||
self,
|
||||
root_dir=None,
|
||||
config=None,
|
||||
deactivate=False,
|
||||
override=True,
|
||||
use_converters=True,
|
||||
use_display_applications=True,
|
||||
@@ -117,10 +109,7 @@ class Registry:
|
||||
):
|
||||
"""
|
||||
Parse a datatypes XML file located at root_dir/config (if processing the Galaxy distributed config) or contained within
|
||||
an installed Tool Shed repository. If deactivate is True, an installed Tool Shed repository that includes custom datatypes
|
||||
is being deactivated or uninstalled, so appropriate loaded datatypes will be removed from the registry. The value of
|
||||
override will be False when a Tool Shed repository is being installed. Since installation is occurring after the datatypes
|
||||
registry has been initialized at server startup, its contents cannot be overridden by newly introduced conflicting data types.
|
||||
an installed Tool Shed repository.
|
||||
"""
|
||||
|
||||
def __import_module(full_path: str, datatype_module: str):
|
||||
@@ -134,24 +123,13 @@ class Registry:
|
||||
return module
|
||||
|
||||
if root_dir and config:
|
||||
# If handling_proprietary_datatypes is determined as True below, we'll have an elem that looks something like this:
|
||||
# <datatype display_in_upload="true"
|
||||
# extension="blastxml"
|
||||
# mimetype="application/xml"
|
||||
# proprietary_datatype_module="blast"
|
||||
# proprietary_path="[cloned repository path]"
|
||||
# type="galaxy.datatypes.blast:BlastXml" />
|
||||
compressed_sniffers = {}
|
||||
handling_proprietary_datatypes = False
|
||||
if isinstance(config, (str, Path)):
|
||||
# Parse datatypes_conf.xml
|
||||
tree = galaxy.util.parse_xml(config)
|
||||
root = tree.getroot()
|
||||
# Load datatypes and converters from config
|
||||
if deactivate:
|
||||
self.log.debug(f"Deactivating datatypes from {config}")
|
||||
else:
|
||||
self.log.debug(f"Loading datatypes from {config}")
|
||||
self.log.debug(f"Loading datatypes from {config}")
|
||||
else:
|
||||
root = config
|
||||
registration = root.find("registration")
|
||||
@@ -182,15 +160,7 @@ class Registry:
|
||||
os.path.join(os.path.dirname(__file__), "display_applications", "configs")
|
||||
)
|
||||
self.display_applications_path = self.display_path_attr
|
||||
# Proprietary datatype's <registration> tag may have special attributes, proprietary_converter_path and proprietary_display_path.
|
||||
proprietary_converter_path = registration.get("proprietary_converter_path", None)
|
||||
proprietary_display_path = registration.get("proprietary_display_path", None)
|
||||
if (
|
||||
proprietary_converter_path is not None
|
||||
or proprietary_display_path is not None
|
||||
and not handling_proprietary_datatypes
|
||||
):
|
||||
handling_proprietary_datatypes = True
|
||||
|
||||
for elem in registration.findall("datatype"):
|
||||
# Keep a status of the process steps to enable stopping the process of handling the datatype if necessary.
|
||||
ok = True
|
||||
@@ -217,263 +187,192 @@ class Registry:
|
||||
if edam_data and not make_subclass:
|
||||
self.log.warning("Cannot specify edam_data without setting subclass to True, skipping datatype.")
|
||||
continue
|
||||
# Proprietary datatypes included in installed tool shed repositories will include two special attributes
|
||||
# (proprietary_path and proprietary_datatype_module) if they depend on proprietary datatypes classes.
|
||||
# The value of proprietary_path is the path to the cloned location of the tool shed repository's contained
|
||||
# datatypes_conf.xml file.
|
||||
proprietary_path = elem.get("proprietary_path", None)
|
||||
proprietary_datatype_module = elem.get("proprietary_datatype_module", None)
|
||||
if (
|
||||
proprietary_path is not None
|
||||
or proprietary_datatype_module is not None
|
||||
and not handling_proprietary_datatypes
|
||||
):
|
||||
handling_proprietary_datatypes = True
|
||||
if deactivate:
|
||||
# We are deactivating or uninstalling an installed tool shed repository, so eliminate the datatype
|
||||
# elem from the in-memory list of datatype elems.
|
||||
for in_memory_elem in self.datatype_elems:
|
||||
in_memory_extension = in_memory_elem.get("extension", None)
|
||||
if in_memory_extension == extension:
|
||||
in_memory_dtype = elem.get("type", None)
|
||||
in_memory_type_extension = elem.get("type_extension", None)
|
||||
in_memory_mimetype = elem.get("mimetype", None)
|
||||
in_memory_display_in_upload = galaxy.util.string_as_bool(
|
||||
elem.get("display_in_upload", False)
|
||||
)
|
||||
in_memory_make_subclass = galaxy.util.string_as_bool(elem.get("subclass", False))
|
||||
if (
|
||||
in_memory_dtype == dtype
|
||||
and in_memory_type_extension == type_extension
|
||||
and in_memory_mimetype == mimetype
|
||||
and in_memory_display_in_upload == display_in_upload
|
||||
and in_memory_make_subclass == make_subclass
|
||||
):
|
||||
self.datatype_elems.remove(in_memory_elem)
|
||||
if extension is not None and extension in self.datatypes_by_extension:
|
||||
# We are deactivating or uninstalling an installed tool shed repository, so eliminate the datatype
|
||||
# from the registry. TODO: Handle deactivating datatype converters, etc before removing from
|
||||
# self.datatypes_by_extension.
|
||||
del self.datatypes_by_extension[extension]
|
||||
if extension in self.upload_file_formats:
|
||||
self.upload_file_formats.remove(extension)
|
||||
self.log.debug(f"Removed datatype with extension '{extension}' from the registry.")
|
||||
else:
|
||||
# We are loading new datatype, so we'll make sure it is correctly defined before proceeding.
|
||||
can_process_datatype = False
|
||||
if extension is not None:
|
||||
if dtype is not None or type_extension is not None:
|
||||
if override or extension not in self.datatypes_by_extension:
|
||||
can_process_datatype = True
|
||||
if can_process_datatype:
|
||||
if dtype is not None:
|
||||
try:
|
||||
fields = dtype.split(":")
|
||||
datatype_module = fields[0]
|
||||
datatype_class_name = fields[1]
|
||||
except Exception:
|
||||
self.log.exception("Error parsing datatype definition for dtype %s", str(dtype))
|
||||
ok = False
|
||||
if ok:
|
||||
datatype_class = None
|
||||
if proprietary_path and proprietary_datatype_module and datatype_class_name:
|
||||
# TODO: previously comments suggested this needs to be locked because it modifies
|
||||
# the sys.path, probably true but the previous lock wasn't doing that.
|
||||
try:
|
||||
imported_module = __import_module(proprietary_path, proprietary_datatype_module)
|
||||
if imported_module not in self.imported_modules:
|
||||
self.imported_modules.append(imported_module)
|
||||
if hasattr(imported_module, datatype_class_name):
|
||||
datatype_class = getattr(imported_module, datatype_class_name)
|
||||
except Exception as e:
|
||||
full_path = os.path.join(proprietary_path, proprietary_datatype_module)
|
||||
self.log.debug(
|
||||
"Exception importing proprietary code file %s: %s",
|
||||
full_path,
|
||||
e,
|
||||
)
|
||||
# Either the above exception was thrown because the proprietary_datatype_module is not derived from a class
|
||||
# in the repository, or we are loading Galaxy's datatypes. In either case we'll look in the registry.
|
||||
if datatype_class is None:
|
||||
try:
|
||||
# The datatype class name must be contained in one of the datatype modules in the Galaxy distribution.
|
||||
fields = datatype_module.split(".")[1:]
|
||||
module = __import__(datatype_module)
|
||||
for mod in fields:
|
||||
module = getattr(module, mod)
|
||||
datatype_class = getattr(module, datatype_class_name)
|
||||
self.log.debug(
|
||||
f"Retrieved datatype module {str(datatype_module)}:{datatype_class_name} from the datatype registry for extension {extension}."
|
||||
)
|
||||
except Exception:
|
||||
self.log.exception("Error importing datatype module %s", str(datatype_module))
|
||||
ok = False
|
||||
elif type_extension is not None:
|
||||
try:
|
||||
datatype_class = self.datatypes_by_extension[type_extension].__class__
|
||||
self.log.debug(
|
||||
f"Retrieved datatype module {str(datatype_class.__name__)} from type_extension {type_extension} for extension {extension}."
|
||||
)
|
||||
except Exception:
|
||||
self.log.exception(
|
||||
"Error determining datatype_class for type_extension %s", str(type_extension)
|
||||
)
|
||||
ok = False
|
||||
|
||||
# We are loading new datatype, so we'll make sure it is correctly defined before proceeding.
|
||||
can_process_datatype = False
|
||||
if extension is not None:
|
||||
if dtype is not None or type_extension is not None:
|
||||
if override or extension not in self.datatypes_by_extension:
|
||||
can_process_datatype = True
|
||||
if can_process_datatype:
|
||||
if dtype is not None:
|
||||
try:
|
||||
fields = dtype.split(":")
|
||||
datatype_module = fields[0]
|
||||
datatype_class_name = fields[1]
|
||||
except Exception:
|
||||
self.log.exception("Error parsing datatype definition for dtype %s", str(dtype))
|
||||
ok = False
|
||||
if ok:
|
||||
if not deactivate:
|
||||
# A new tool shed repository that contains custom datatypes is being installed, and since installation is
|
||||
# occurring after the datatypes registry has been initialized at server startup, its contents cannot be
|
||||
# overridden by new introduced conflicting data types unless the value of override is True.
|
||||
if extension in self.datatypes_by_extension:
|
||||
# Because of the way that the value of can_process_datatype was set above, we know that the value of
|
||||
# override is True.
|
||||
datatype_class = None
|
||||
if datatype_class is None:
|
||||
try:
|
||||
# The datatype class name must be contained in one of the datatype modules in the Galaxy distribution.
|
||||
fields = datatype_module.split(".")[1:]
|
||||
module = __import__(datatype_module)
|
||||
for mod in fields:
|
||||
module = getattr(module, mod)
|
||||
datatype_class = getattr(module, datatype_class_name)
|
||||
self.log.debug(
|
||||
"Overriding conflicting datatype with extension '%s', using datatype from %s."
|
||||
% (str(extension), str(config))
|
||||
f"Retrieved datatype module {str(datatype_module)}:{datatype_class_name} from the datatype registry for extension {extension}."
|
||||
)
|
||||
if make_subclass:
|
||||
datatype_class = type(datatype_class_name, (datatype_class,), {})
|
||||
if edam_format:
|
||||
datatype_class.edam_format = edam_format
|
||||
if edam_data:
|
||||
datatype_class.edam_data = edam_data
|
||||
datatype_class.is_subclass = make_subclass
|
||||
description = elem.get("description", None)
|
||||
description_url = elem.get("description_url", None)
|
||||
datatype_instance = datatype_class()
|
||||
self.datatypes_by_extension[extension] = datatype_instance
|
||||
if mimetype is None:
|
||||
# Use default mimetype per datatype specification.
|
||||
mimetype = self.datatypes_by_extension[extension].get_mime()
|
||||
self.mimetypes_by_extension[extension] = mimetype
|
||||
if datatype_class.track_type:
|
||||
self.available_tracks.append(extension)
|
||||
if display_in_upload and extension not in self.upload_file_formats:
|
||||
self.upload_file_formats.append(extension)
|
||||
# Max file size cut off for setting optional metadata.
|
||||
self.datatypes_by_extension[extension].max_optional_metadata_filesize = elem.get(
|
||||
"max_optional_metadata_filesize", None
|
||||
)
|
||||
infer_from_suffixes = []
|
||||
# read from element instead of attribute so we can customize references to
|
||||
# compressed files in the future (e.g. maybe some day faz will be a compressed fasta
|
||||
# or something along those lines)
|
||||
for infer_from in elem.findall("infer_from"):
|
||||
suffix = infer_from.get("suffix", None)
|
||||
if suffix is None:
|
||||
raise Exception("Failed to parse infer_from datatype element")
|
||||
infer_from_suffixes.append(suffix)
|
||||
self.datatypes_by_suffix_inferences[suffix] = datatype_instance
|
||||
for converter in elem.findall("converter"):
|
||||
# Build the list of datatype converters which will later be loaded into the calling app's toolbox.
|
||||
converter_config = converter.get("file", None)
|
||||
target_datatype = converter.get("target_datatype", None)
|
||||
depends_on = converter.get("depends_on", None)
|
||||
if depends_on is not None and target_datatype is not None:
|
||||
if extension not in self.converter_deps:
|
||||
self.converter_deps[extension] = {}
|
||||
self.converter_deps[extension][target_datatype] = depends_on.split(",")
|
||||
if converter_config and target_datatype:
|
||||
if proprietary_converter_path:
|
||||
self.proprietary_converters.append(
|
||||
(converter_config, extension, target_datatype)
|
||||
)
|
||||
else:
|
||||
self.converters.append((converter_config, extension, target_datatype))
|
||||
# Add composite files.
|
||||
for composite_file in elem.findall("composite_file"):
|
||||
name = composite_file.get("name", None)
|
||||
if name is None:
|
||||
self.log.warning(
|
||||
f"You must provide a name for your composite_file ({composite_file})."
|
||||
)
|
||||
optional = composite_file.get("optional", False)
|
||||
mimetype = composite_file.get("mimetype", None)
|
||||
self.datatypes_by_extension[extension].add_composite_file(
|
||||
name, optional=optional, mimetype=mimetype
|
||||
)
|
||||
for _display_app in elem.findall("display"):
|
||||
if proprietary_display_path:
|
||||
if elem not in self.proprietary_display_app_containers:
|
||||
self.proprietary_display_app_containers.append(elem)
|
||||
else:
|
||||
if elem not in self.display_app_containers:
|
||||
self.display_app_containers.append(elem)
|
||||
datatype_info_dict = {
|
||||
except Exception:
|
||||
self.log.exception("Error importing datatype module %s", str(datatype_module))
|
||||
ok = False
|
||||
elif type_extension is not None:
|
||||
try:
|
||||
datatype_class = self.datatypes_by_extension[type_extension].__class__
|
||||
self.log.debug(
|
||||
f"Retrieved datatype module {str(datatype_class.__name__)} from type_extension {type_extension} for extension {extension}."
|
||||
)
|
||||
except Exception:
|
||||
self.log.exception(
|
||||
"Error determining datatype_class for type_extension %s", str(type_extension)
|
||||
)
|
||||
ok = False
|
||||
if ok:
|
||||
# A new tool shed repository that contains custom datatypes is being installed, and since installation is
|
||||
# occurring after the datatypes registry has been initialized at server startup, its contents cannot be
|
||||
# overridden by new introduced conflicting data types unless the value of override is True.
|
||||
if extension in self.datatypes_by_extension:
|
||||
# Because of the way that the value of can_process_datatype was set above, we know that the value of
|
||||
# override is True.
|
||||
self.log.debug(
|
||||
"Overriding conflicting datatype with extension '%s', using datatype from %s."
|
||||
% (str(extension), str(config))
|
||||
)
|
||||
if make_subclass:
|
||||
datatype_class = type(datatype_class_name, (datatype_class,), {})
|
||||
if edam_format:
|
||||
datatype_class.edam_format = edam_format
|
||||
if edam_data:
|
||||
datatype_class.edam_data = edam_data
|
||||
datatype_class.is_subclass = make_subclass
|
||||
description = elem.get("description", None)
|
||||
description_url = elem.get("description_url", None)
|
||||
datatype_instance = datatype_class()
|
||||
self.datatypes_by_extension[extension] = datatype_instance
|
||||
if mimetype is None:
|
||||
# Use default mimetype per datatype specification.
|
||||
mimetype = self.datatypes_by_extension[extension].get_mime()
|
||||
self.mimetypes_by_extension[extension] = mimetype
|
||||
if datatype_class.track_type:
|
||||
self.available_tracks.append(extension)
|
||||
if display_in_upload and extension not in self.upload_file_formats:
|
||||
self.upload_file_formats.append(extension)
|
||||
# Max file size cut off for setting optional metadata.
|
||||
self.datatypes_by_extension[extension].max_optional_metadata_filesize = elem.get(
|
||||
"max_optional_metadata_filesize", None
|
||||
)
|
||||
infer_from_suffixes = []
|
||||
# read from element instead of attribute so we can customize references to
|
||||
# compressed files in the future (e.g. maybe some day faz will be a compressed fasta
|
||||
# or something along those lines)
|
||||
for infer_from in elem.findall("infer_from"):
|
||||
suffix = infer_from.get("suffix", None)
|
||||
if suffix is None:
|
||||
raise Exception("Failed to parse infer_from datatype element")
|
||||
infer_from_suffixes.append(suffix)
|
||||
self.datatypes_by_suffix_inferences[suffix] = datatype_instance
|
||||
for converter in elem.findall("converter"):
|
||||
# Build the list of datatype converters which will later be loaded into the calling app's toolbox.
|
||||
converter_config = converter.get("file", None)
|
||||
target_datatype = converter.get("target_datatype", None)
|
||||
depends_on = converter.get("depends_on", None)
|
||||
if depends_on is not None and target_datatype is not None:
|
||||
if extension not in self.converter_deps:
|
||||
self.converter_deps[extension] = {}
|
||||
self.converter_deps[extension][target_datatype] = depends_on.split(",")
|
||||
if converter_config and target_datatype:
|
||||
self.converters.append((converter_config, extension, target_datatype))
|
||||
# Add composite files.
|
||||
for composite_file in elem.findall("composite_file"):
|
||||
name = composite_file.get("name", None)
|
||||
if name is None:
|
||||
self.log.warning(f"You must provide a name for your composite_file ({composite_file}).")
|
||||
optional = composite_file.get("optional", False)
|
||||
mimetype = composite_file.get("mimetype", None)
|
||||
self.datatypes_by_extension[extension].add_composite_file(
|
||||
name, optional=optional, mimetype=mimetype
|
||||
)
|
||||
for _display_app in elem.findall("display"):
|
||||
if elem not in self.display_app_containers:
|
||||
self.display_app_containers.append(elem)
|
||||
datatype_info_dict = {
|
||||
"display_in_upload": display_in_upload,
|
||||
"extension": extension,
|
||||
"description": description,
|
||||
"description_url": description_url,
|
||||
}
|
||||
composite_files = datatype_instance.get_composite_files()
|
||||
if composite_files:
|
||||
_composite_files = []
|
||||
for name, composite_file in composite_files.items():
|
||||
_composite_file = composite_file.dict()
|
||||
_composite_file["name"] = name
|
||||
_composite_files.append(_composite_file)
|
||||
datatype_info_dict["composite_files"] = _composite_files
|
||||
self.datatype_info_dicts.append(datatype_info_dict)
|
||||
|
||||
for auto_compressed_type in auto_compressed_types:
|
||||
compressed_extension = f"{extension}.{auto_compressed_type}"
|
||||
upper_compressed_type = auto_compressed_type[0].upper() + auto_compressed_type[1:]
|
||||
auto_compressed_type_name = datatype_class_name + upper_compressed_type
|
||||
attributes = {}
|
||||
if auto_compressed_type == "gz":
|
||||
dynamic_parent = binary.GzDynamicCompressedArchive
|
||||
elif auto_compressed_type == "bz2":
|
||||
dynamic_parent = binary.Bz2DynamicCompressedArchive
|
||||
else:
|
||||
raise Exception(f"Unknown auto compression type [{auto_compressed_type}]")
|
||||
attributes["file_ext"] = compressed_extension
|
||||
attributes["uncompressed_datatype_instance"] = datatype_instance
|
||||
compressed_datatype_class = type(
|
||||
auto_compressed_type_name,
|
||||
(
|
||||
datatype_class,
|
||||
dynamic_parent,
|
||||
),
|
||||
attributes,
|
||||
)
|
||||
if edam_format:
|
||||
compressed_datatype_class.edam_format = edam_format
|
||||
if edam_data:
|
||||
compressed_datatype_class.edam_data = edam_data
|
||||
compressed_datatype_instance = compressed_datatype_class()
|
||||
self.datatypes_by_extension[compressed_extension] = compressed_datatype_instance
|
||||
for suffix in infer_from_suffixes:
|
||||
self.datatypes_by_suffix_inferences[
|
||||
f"{suffix}.{auto_compressed_type}"
|
||||
] = compressed_datatype_instance
|
||||
if display_in_upload and compressed_extension not in self.upload_file_formats:
|
||||
self.upload_file_formats.append(compressed_extension)
|
||||
self.datatype_info_dicts.append(
|
||||
{
|
||||
"display_in_upload": display_in_upload,
|
||||
"extension": extension,
|
||||
"extension": compressed_extension,
|
||||
"description": description,
|
||||
"description_url": description_url,
|
||||
}
|
||||
composite_files = datatype_instance.get_composite_files()
|
||||
if composite_files:
|
||||
_composite_files = []
|
||||
for name, composite_file in composite_files.items():
|
||||
_composite_file = composite_file.dict()
|
||||
_composite_file["name"] = name
|
||||
_composite_files.append(_composite_file)
|
||||
datatype_info_dict["composite_files"] = _composite_files
|
||||
self.datatype_info_dicts.append(datatype_info_dict)
|
||||
|
||||
for auto_compressed_type in auto_compressed_types:
|
||||
compressed_extension = f"{extension}.{auto_compressed_type}"
|
||||
upper_compressed_type = auto_compressed_type[0].upper() + auto_compressed_type[1:]
|
||||
auto_compressed_type_name = datatype_class_name + upper_compressed_type
|
||||
attributes = {}
|
||||
if auto_compressed_type == "gz":
|
||||
dynamic_parent = binary.GzDynamicCompressedArchive
|
||||
elif auto_compressed_type == "bz2":
|
||||
dynamic_parent = binary.Bz2DynamicCompressedArchive
|
||||
else:
|
||||
raise Exception(f"Unknown auto compression type [{auto_compressed_type}]")
|
||||
attributes["file_ext"] = compressed_extension
|
||||
attributes["uncompressed_datatype_instance"] = datatype_instance
|
||||
compressed_datatype_class = type(
|
||||
auto_compressed_type_name,
|
||||
(
|
||||
datatype_class,
|
||||
dynamic_parent,
|
||||
),
|
||||
attributes,
|
||||
)
|
||||
if auto_compressed_type == "gz":
|
||||
self.converters.append(
|
||||
(
|
||||
f"uncompressed_to_{auto_compressed_type}.xml",
|
||||
extension,
|
||||
compressed_extension,
|
||||
)
|
||||
if edam_format:
|
||||
compressed_datatype_class.edam_format = edam_format
|
||||
if edam_data:
|
||||
compressed_datatype_class.edam_data = edam_data
|
||||
compressed_datatype_instance = compressed_datatype_class()
|
||||
self.datatypes_by_extension[compressed_extension] = compressed_datatype_instance
|
||||
for suffix in infer_from_suffixes:
|
||||
self.datatypes_by_suffix_inferences[
|
||||
f"{suffix}.{auto_compressed_type}"
|
||||
] = compressed_datatype_instance
|
||||
if display_in_upload and compressed_extension not in self.upload_file_formats:
|
||||
self.upload_file_formats.append(compressed_extension)
|
||||
self.datatype_info_dicts.append(
|
||||
{
|
||||
"display_in_upload": display_in_upload,
|
||||
"extension": compressed_extension,
|
||||
"description": description,
|
||||
"description_url": description_url,
|
||||
}
|
||||
)
|
||||
if auto_compressed_type == "gz":
|
||||
self.converters.append(
|
||||
(
|
||||
f"uncompressed_to_{auto_compressed_type}.xml",
|
||||
extension,
|
||||
compressed_extension,
|
||||
)
|
||||
)
|
||||
self.converters.append(
|
||||
(f"{auto_compressed_type}_to_uncompressed.xml", compressed_extension, extension)
|
||||
)
|
||||
if datatype_class not in compressed_sniffers:
|
||||
compressed_sniffers[datatype_class] = []
|
||||
if sniff_compressed_types:
|
||||
compressed_sniffers[datatype_class].append(compressed_datatype_instance)
|
||||
# Processing the new datatype elem is now complete, so make sure the element defining it is retained by appending
|
||||
# the new datatype to the in-memory list of datatype elems to enable persistence.
|
||||
self.datatype_elems.append(elem)
|
||||
)
|
||||
self.converters.append(
|
||||
(f"{auto_compressed_type}_to_uncompressed.xml", compressed_extension, extension)
|
||||
)
|
||||
if datatype_class not in compressed_sniffers:
|
||||
compressed_sniffers[datatype_class] = []
|
||||
if sniff_compressed_types:
|
||||
compressed_sniffers[datatype_class].append(compressed_datatype_instance)
|
||||
# Processing the new datatype elem is now complete, so make sure the element defining it is retained by appending
|
||||
# the new datatype to the in-memory list of datatype elems to enable persistence.
|
||||
self.datatype_elems.append(elem)
|
||||
else:
|
||||
if extension is not None:
|
||||
if dtype is not None or type_extension is not None:
|
||||
@@ -488,8 +387,6 @@ class Registry:
|
||||
# since sniffers are not tightly coupled with datatypes.
|
||||
self.load_datatype_sniffers(
|
||||
root,
|
||||
deactivate=deactivate,
|
||||
handling_proprietary_datatypes=handling_proprietary_datatypes,
|
||||
override=override,
|
||||
compressed_sniffers=compressed_sniffers,
|
||||
)
|
||||
@@ -570,16 +467,10 @@ class Registry:
|
||||
def get_display_sites(self, site_type):
|
||||
return self.display_sites.get(site_type, [])
|
||||
|
||||
def load_datatype_sniffers(
|
||||
self, root, deactivate=False, handling_proprietary_datatypes=False, override=False, compressed_sniffers=None
|
||||
):
|
||||
def load_datatype_sniffers(self, root, override=False, compressed_sniffers=None):
|
||||
"""
|
||||
Process the sniffers element from a parsed a datatypes XML file located at root_dir/config (if processing the Galaxy
|
||||
distributed config) or contained within an installed Tool Shed repository. If deactivate is True, an installed Tool
|
||||
Shed repository that includes custom sniffers is being deactivated or uninstalled, so appropriate loaded sniffers will
|
||||
be removed from the registry. The value of override will be False when a Tool Shed repository is being installed.
|
||||
Since installation is occurring after the datatypes registry has been initialized at server startup, its contents
|
||||
cannot be overridden by newly introduced conflicting sniffers.
|
||||
distributed config) or contained within an installed Tool Shed repository.
|
||||
"""
|
||||
sniffer_elem_classes = [e.attrib["type"] for e in self.sniffer_elems]
|
||||
sniffers = root.find("sniffers")
|
||||
@@ -598,12 +489,6 @@ class Registry:
|
||||
self.log.exception("Error determining datatype class or module for dtype %s", str(dtype))
|
||||
ok = False
|
||||
if ok:
|
||||
if handling_proprietary_datatypes:
|
||||
# See if one of the imported modules contains the datatype class name.
|
||||
for imported_module in self.imported_modules:
|
||||
if hasattr(imported_module, datatype_class_name):
|
||||
module = imported_module
|
||||
break
|
||||
if module is None:
|
||||
try:
|
||||
# The datatype class name must be contained in one of the datatype modules in the Galaxy distribution.
|
||||
@@ -622,47 +507,27 @@ class Registry:
|
||||
)
|
||||
ok = False
|
||||
if ok:
|
||||
if deactivate:
|
||||
# We are deactivating or uninstalling an installed Tool Shed repository, so eliminate the appropriate sniffers.
|
||||
sniffer_class = elem.get("type", None)
|
||||
if sniffer_class is not None:
|
||||
for index, s_e_c in enumerate(sniffer_elem_classes):
|
||||
if sniffer_class == s_e_c:
|
||||
del self.sniffer_elems[index]
|
||||
sniffer_elem_classes = [
|
||||
elem.attrib["type"] for elem in self.sniffer_elems
|
||||
]
|
||||
self.log.debug(f"Removed sniffer element for datatype '{str(dtype)}'")
|
||||
break
|
||||
for sniffer_class in self.sniff_order:
|
||||
if sniffer_class.__class__ == aclass.__class__:
|
||||
self.sniff_order.remove(sniffer_class)
|
||||
self.log.debug(
|
||||
f"Removed sniffer class for datatype '{str(dtype)}' from sniff order"
|
||||
)
|
||||
break
|
||||
else:
|
||||
# We are loading new sniffer, so see if we have a conflicting sniffer already loaded.
|
||||
conflict = False
|
||||
for conflict_loc, sniffer_class in enumerate(self.sniff_order):
|
||||
if sniffer_class.__class__ == aclass.__class__:
|
||||
# We have a conflicting sniffer, so replace the one previously loaded.
|
||||
conflict = True
|
||||
if override:
|
||||
del self.sniff_order[conflict_loc]
|
||||
self.log.debug(f"Removed conflicting sniffer for datatype '{dtype}'")
|
||||
break
|
||||
if not conflict or override:
|
||||
if compressed_sniffers and aclass.__class__ in compressed_sniffers:
|
||||
for compressed_sniffer in compressed_sniffers[aclass.__class__]:
|
||||
self.sniff_order.append(compressed_sniffer)
|
||||
self.sniff_order.append(aclass)
|
||||
self.log.debug(f"Loaded sniffer for datatype '{dtype}'")
|
||||
# Processing the new sniffer elem is now complete, so make sure the element defining it is loaded if necessary.
|
||||
sniffer_class = elem.get("type", None)
|
||||
if sniffer_class is not None:
|
||||
if sniffer_class not in sniffer_elem_classes:
|
||||
self.sniffer_elems.append(elem)
|
||||
# We are loading new sniffer, so see if we have a conflicting sniffer already loaded.
|
||||
conflict = False
|
||||
for conflict_loc, sniffer_class in enumerate(self.sniff_order):
|
||||
if sniffer_class.__class__ == aclass.__class__:
|
||||
# We have a conflicting sniffer, so replace the one previously loaded.
|
||||
conflict = True
|
||||
if override:
|
||||
del self.sniff_order[conflict_loc]
|
||||
self.log.debug(f"Removed conflicting sniffer for datatype '{dtype}'")
|
||||
break
|
||||
if not conflict or override:
|
||||
if compressed_sniffers and aclass.__class__ in compressed_sniffers:
|
||||
for compressed_sniffer in compressed_sniffers[aclass.__class__]:
|
||||
self.sniff_order.append(compressed_sniffer)
|
||||
self.sniff_order.append(aclass)
|
||||
self.log.debug(f"Loaded sniffer for datatype '{dtype}'")
|
||||
# Processing the new sniffer elem is now complete, so make sure the element defining it is loaded if necessary.
|
||||
sniffer_class = elem.get("type", None)
|
||||
if sniffer_class is not None:
|
||||
if sniffer_class not in sniffer_elem_classes:
|
||||
self.sniffer_elems.append(elem)
|
||||
|
||||
def get_datatype_from_filename(self, name):
|
||||
max_extension_parts = 3
|
||||
@@ -734,149 +599,64 @@ class Registry:
|
||||
data.init_meta(copy_from=data)
|
||||
return data
|
||||
|
||||
def load_datatype_converters(self, toolbox, installed_repository_dict=None, deactivate=False, use_cached=False):
|
||||
def load_datatype_converters(self, toolbox, use_cached=False):
|
||||
"""
|
||||
If deactivate is False, add datatype converters from self.converters or self.proprietary_converters
|
||||
to the calling app's toolbox. If deactivate is True, eliminates relevant converters from the calling
|
||||
app's toolbox.
|
||||
Add datatype converters from self.converters to the calling app's toolbox.
|
||||
"""
|
||||
if installed_repository_dict:
|
||||
# Load converters defined by datatypes_conf.xml included in installed tool shed repository.
|
||||
converters = self.proprietary_converters
|
||||
else:
|
||||
# Load converters defined by local datatypes_conf.xml.
|
||||
converters = self.converters
|
||||
# Load converters defined by local datatypes_conf.xml.
|
||||
converters = self.converters
|
||||
for elem in converters:
|
||||
tool_config = elem[0]
|
||||
source_datatype = elem[1]
|
||||
target_datatype = elem[2]
|
||||
if installed_repository_dict:
|
||||
converter_path = installed_repository_dict["converter_path"]
|
||||
else:
|
||||
converter_path = self.converters_path
|
||||
converter_path = self.converters_path
|
||||
try:
|
||||
config_path = os.path.join(converter_path, tool_config)
|
||||
converter = toolbox.load_tool(config_path, use_cached=use_cached)
|
||||
self.converter_tools.add(converter)
|
||||
if installed_repository_dict:
|
||||
# If the converter is included in an installed tool shed repository, set the tool
|
||||
# shed related tool attributes.
|
||||
converter.tool_shed = installed_repository_dict["tool_shed"]
|
||||
converter.repository_name = installed_repository_dict["repository_name"]
|
||||
converter.repository_owner = installed_repository_dict["repository_owner"]
|
||||
converter.installed_changeset_revision = installed_repository_dict["installed_changeset_revision"]
|
||||
converter.old_id = converter.id
|
||||
# The converter should be included in the list of tools defined in tool_dicts.
|
||||
tool_dicts = installed_repository_dict["tool_dicts"]
|
||||
for tool_dict in tool_dicts:
|
||||
if tool_dict["id"] == converter.id:
|
||||
converter.guid = tool_dict["guid"]
|
||||
converter.id = tool_dict["guid"]
|
||||
break
|
||||
if deactivate:
|
||||
toolbox.remove_tool_by_id(converter.id, remove_from_panel=False)
|
||||
if source_datatype in self.datatype_converters:
|
||||
if target_datatype in self.datatype_converters[source_datatype]:
|
||||
del self.datatype_converters[source_datatype][target_datatype]
|
||||
self.log.debug("Deactivated converter: %s", converter.id)
|
||||
else:
|
||||
toolbox.register_tool(converter)
|
||||
if source_datatype not in self.datatype_converters:
|
||||
self.datatype_converters[source_datatype] = {}
|
||||
self.datatype_converters[source_datatype][target_datatype] = converter
|
||||
if not hasattr(toolbox.app, "tool_cache") or converter.id in toolbox.app.tool_cache._new_tool_ids:
|
||||
self.log.debug("Loaded converter: %s", converter.id)
|
||||
toolbox.register_tool(converter)
|
||||
if source_datatype not in self.datatype_converters:
|
||||
self.datatype_converters[source_datatype] = {}
|
||||
self.datatype_converters[source_datatype][target_datatype] = converter
|
||||
if not hasattr(toolbox.app, "tool_cache") or converter.id in toolbox.app.tool_cache._new_tool_ids:
|
||||
self.log.debug("Loaded converter: %s", converter.id)
|
||||
except Exception:
|
||||
if deactivate:
|
||||
self.log.exception(f"Error deactivating converter from ({converter_path})")
|
||||
else:
|
||||
self.log.exception(f"Error loading converter ({converter_path})")
|
||||
self.log.exception(f"Error loading converter ({converter_path})")
|
||||
|
||||
def load_display_applications(self, app, installed_repository_dict=None, deactivate=False):
|
||||
def load_display_applications(self, app):
|
||||
"""
|
||||
If deactivate is False, add display applications from self.display_app_containers or
|
||||
self.proprietary_display_app_containers to appropriate datatypes. If deactivate is
|
||||
True, eliminates relevant display applications from appropriate datatypes.
|
||||
Add display applications from self.display_app_containers or to appropriate datatypes.
|
||||
"""
|
||||
if installed_repository_dict:
|
||||
# Load display applications defined by datatypes_conf.xml included in installed tool shed repository.
|
||||
datatype_elems = self.proprietary_display_app_containers
|
||||
else:
|
||||
# Load display applications defined by local datatypes_conf.xml.
|
||||
datatype_elems = self.display_app_containers
|
||||
# Load display applications defined by local datatypes_conf.xml.
|
||||
datatype_elems = self.display_app_containers
|
||||
for elem in datatype_elems:
|
||||
extension = self.get_extension(elem)
|
||||
for display_app in elem.findall("display"):
|
||||
display_file = display_app.get("file", None)
|
||||
if installed_repository_dict:
|
||||
display_path = installed_repository_dict["display_path"]
|
||||
display_file_head, display_file_tail = os.path.split(display_file)
|
||||
config_path = os.path.join(display_path, display_file_tail)
|
||||
else:
|
||||
config_path = os.path.join(self.display_applications_path, display_file)
|
||||
config_path = os.path.join(self.display_applications_path, display_file)
|
||||
try:
|
||||
inherit = galaxy.util.string_as_bool(display_app.get("inherit", "False"))
|
||||
display_app = DisplayApplication.from_file(config_path, app)
|
||||
if display_app:
|
||||
if display_app.id in self.display_applications:
|
||||
if deactivate:
|
||||
del self.display_applications[display_app.id]
|
||||
else:
|
||||
# If we already loaded this display application, we'll use the first one loaded.
|
||||
display_app = self.display_applications[display_app.id]
|
||||
elif installed_repository_dict:
|
||||
# If the display application is included in an installed tool shed repository,
|
||||
# set the tool shed related tool attributes.
|
||||
display_app.tool_shed = installed_repository_dict["tool_shed"]
|
||||
display_app.repository_name = installed_repository_dict["repository_name"]
|
||||
display_app.repository_owner = installed_repository_dict["repository_owner"]
|
||||
display_app.installed_changeset_revision = installed_repository_dict[
|
||||
"installed_changeset_revision"
|
||||
]
|
||||
display_app.old_id = display_app.id
|
||||
# The display application should be included in the list of tools defined in tool_dicts.
|
||||
tool_dicts = installed_repository_dict["tool_dicts"]
|
||||
for tool_dict in tool_dicts:
|
||||
if tool_dict["id"] == display_app.id:
|
||||
display_app.guid = tool_dict["guid"]
|
||||
display_app.id = tool_dict["guid"]
|
||||
break
|
||||
if deactivate:
|
||||
if display_app.id in self.display_applications:
|
||||
del self.display_applications[display_app.id]
|
||||
if extension in self.datatypes_by_extension:
|
||||
if display_app.id in self.datatypes_by_extension[extension].display_applications:
|
||||
del self.datatypes_by_extension[extension].display_applications[display_app.id]
|
||||
if (
|
||||
inherit
|
||||
and (self.datatypes_by_extension[extension], display_app)
|
||||
in self.inherit_display_application_by_class
|
||||
):
|
||||
self.inherit_display_application_by_class.remove(
|
||||
(self.datatypes_by_extension[extension], display_app)
|
||||
)
|
||||
self.log.debug(
|
||||
f"Deactivated display application '{display_app.id}' for datatype '{extension}'."
|
||||
)
|
||||
else:
|
||||
self.display_applications[display_app.id] = display_app
|
||||
self.datatypes_by_extension[extension].add_display_application(display_app)
|
||||
if (
|
||||
inherit
|
||||
and (self.datatypes_by_extension[extension], display_app)
|
||||
not in self.inherit_display_application_by_class
|
||||
):
|
||||
self.inherit_display_application_by_class.append(
|
||||
(self.datatypes_by_extension[extension], display_app)
|
||||
)
|
||||
self.log.debug(
|
||||
f"Loaded display application '{display_app.id}' for datatype '{extension}', inherit={inherit}."
|
||||
# If we already loaded this display application, we'll use the first one loaded.
|
||||
display_app = self.display_applications[display_app.id]
|
||||
|
||||
self.display_applications[display_app.id] = display_app
|
||||
self.datatypes_by_extension[extension].add_display_application(display_app)
|
||||
if (
|
||||
inherit
|
||||
and (self.datatypes_by_extension[extension], display_app)
|
||||
not in self.inherit_display_application_by_class
|
||||
):
|
||||
self.inherit_display_application_by_class.append(
|
||||
(self.datatypes_by_extension[extension], display_app)
|
||||
)
|
||||
self.log.debug(
|
||||
f"Loaded display application '{display_app.id}' for datatype '{extension}', inherit={inherit}."
|
||||
)
|
||||
except Exception:
|
||||
if deactivate:
|
||||
self.log.exception(f"Error deactivating display application ({config_path})")
|
||||
else:
|
||||
self.log.exception(f"Error loading display application ({config_path})")
|
||||
self.log.exception(f"Error loading display application ({config_path})")
|
||||
# Handle display_application subclass inheritance.
|
||||
for extension, d_type1 in self.datatypes_by_extension.items():
|
||||
for d_type2, display_app in self.inherit_display_application_by_class:
|
||||
|
||||
@@ -1,182 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from galaxy.tool_shed.util.basic_util import strip_path
|
||||
from galaxy.tool_shed.util.hg_util import get_config_from_disk
|
||||
from galaxy.tool_shed.util.shed_util_common import DATATYPES_CONFIG_FILENAME
|
||||
from galaxy.util.tool_shed.xml_util import parse_xml
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CustomDatatypeLoader:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
def _alter_config_and_load_proprietary_datatypes(
|
||||
self, datatypes_config, relative_install_dir, deactivate=False, override=True
|
||||
):
|
||||
"""
|
||||
Parse a custom datatypes config (a datatypes_conf.xml file included in an installed
|
||||
tool shed repository) and add information to appropriate element attributes that will
|
||||
enable custom datatype class modules, datatypes converters and display applications
|
||||
to be discovered and properly imported by the datatypes registry. The value of override
|
||||
will be False when a tool shed repository is being installed. Since installation is
|
||||
occurring after the datatypes registry has been initialized, the registry's contents
|
||||
cannot be overridden by conflicting data types.
|
||||
"""
|
||||
tree, error_message = parse_xml(datatypes_config)
|
||||
if tree is None:
|
||||
return None, None
|
||||
datatypes_config_root = tree.getroot()
|
||||
registration = datatypes_config_root.find("registration")
|
||||
if registration is None:
|
||||
# We have valid XML, but not a valid custom datatypes definition.
|
||||
return None, None
|
||||
converter_path, display_path = self._get_converter_and_display_paths(registration, relative_install_dir)
|
||||
if converter_path:
|
||||
# Path to datatype converters
|
||||
registration.attrib["proprietary_converter_path"] = converter_path
|
||||
if display_path:
|
||||
# Path to datatype display applications
|
||||
registration.attrib["proprietary_display_path"] = display_path
|
||||
relative_path_to_datatype_file_name = None
|
||||
datatype_files = datatypes_config_root.find("datatype_files")
|
||||
datatype_class_modules = []
|
||||
if datatype_files is not None:
|
||||
# The <datatype_files> tag set contains any number of <datatype_file> tags.
|
||||
# <datatype_files>
|
||||
# <datatype_file name="gmap.py"/>
|
||||
# <datatype_file name="metagenomics.py"/>
|
||||
# </datatype_files>
|
||||
# We'll add attributes to the datatype tag sets so that the modules can be properly imported
|
||||
# by the datatypes registry.
|
||||
for elem in datatype_files.findall("datatype_file"):
|
||||
datatype_file_name = elem.get("name", None)
|
||||
if datatype_file_name:
|
||||
# Find the file in the installed repository.
|
||||
for root, _dirs, files in os.walk(relative_install_dir):
|
||||
if root.find(".hg") < 0:
|
||||
for name in files:
|
||||
if name == datatype_file_name:
|
||||
datatype_class_modules.append(os.path.join(root, name))
|
||||
break
|
||||
break
|
||||
if datatype_class_modules:
|
||||
for relative_path_to_datatype_file_name in datatype_class_modules:
|
||||
datatype_file_name_path, datatype_file_name = os.path.split(relative_path_to_datatype_file_name)
|
||||
for elem in registration.findall("datatype"):
|
||||
# Handle 'type' attribute which should be something like one of the following:
|
||||
# type="gmap:GmapDB"
|
||||
# type="galaxy.datatypes.gmap:GmapDB"
|
||||
dtype = elem.get("type", None)
|
||||
if dtype:
|
||||
fields = dtype.split(":")
|
||||
proprietary_datatype_module = fields[0]
|
||||
if proprietary_datatype_module.find(".") >= 0:
|
||||
# Handle the case where datatype_module is "galaxy.datatypes.gmap".
|
||||
proprietary_datatype_module = proprietary_datatype_module.split(".")[-1]
|
||||
# The value of proprietary_path must be an absolute path due to job_working_directory.
|
||||
elem.attrib["proprietary_path"] = os.path.abspath(datatype_file_name_path)
|
||||
elem.attrib["proprietary_datatype_module"] = proprietary_datatype_module
|
||||
# Load custom datatypes
|
||||
self.app.datatypes_registry.load_datatypes(
|
||||
root_dir=self.app.config.root, config=datatypes_config_root, deactivate=deactivate, override=override
|
||||
)
|
||||
return converter_path, display_path
|
||||
|
||||
def _create_repository_dict_for_proprietary_datatypes(
|
||||
self, tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None
|
||||
):
|
||||
return dict(
|
||||
tool_shed=tool_shed,
|
||||
repository_name=name,
|
||||
repository_owner=owner,
|
||||
installed_changeset_revision=installed_changeset_revision,
|
||||
tool_dicts=tool_dicts,
|
||||
converter_path=converter_path,
|
||||
display_path=display_path,
|
||||
)
|
||||
|
||||
def _get_converter_and_display_paths(self, registration_elem, relative_install_dir):
|
||||
"""
|
||||
Find the relative path to data type converters and display applications included
|
||||
in installed tool shed repositories.
|
||||
"""
|
||||
converter_path = None
|
||||
display_path = None
|
||||
for elem in registration_elem.findall("datatype"):
|
||||
if not converter_path:
|
||||
# If any of the <datatype> tag sets contain <converter> tags, set the converter_path
|
||||
# if it is not already set. This requires developers to place all converters in the
|
||||
# same subdirectory within the repository hierarchy.
|
||||
for converter in elem.findall("converter"):
|
||||
converter_config = converter.get("file", None)
|
||||
if converter_config:
|
||||
converter_config_file_name = strip_path(converter_config)
|
||||
for root, _dirs, files in os.walk(relative_install_dir):
|
||||
if root.find(".hg") < 0:
|
||||
for name in files:
|
||||
if name == converter_config_file_name:
|
||||
# The value of converter_path must be absolute due to job_working_directory.
|
||||
converter_path = os.path.abspath(root)
|
||||
break
|
||||
if converter_path:
|
||||
break
|
||||
if not display_path:
|
||||
# If any of the <datatype> tag sets contain <display> tags, set the display_path
|
||||
# if it is not already set. This requires developers to place all display acpplications
|
||||
# in the same subdirectory within the repository hierarchy.
|
||||
for display_app in elem.findall("display"):
|
||||
display_config = display_app.get("file", None)
|
||||
if display_config:
|
||||
display_config_file_name = strip_path(display_config)
|
||||
for root, _dirs, files in os.walk(relative_install_dir):
|
||||
if root.find(".hg") < 0:
|
||||
for name in files:
|
||||
if name == display_config_file_name:
|
||||
# The value of display_path must be absolute due to job_working_directory.
|
||||
display_path = os.path.abspath(root)
|
||||
break
|
||||
if display_path:
|
||||
break
|
||||
if converter_path and display_path:
|
||||
break
|
||||
return converter_path, display_path
|
||||
|
||||
def load_installed_datatype_converters(self, installed_repository_dict, deactivate=False):
|
||||
"""Load or deactivate proprietary datatype converters."""
|
||||
self.app.datatypes_registry.load_datatype_converters(
|
||||
self.app.toolbox, installed_repository_dict=installed_repository_dict, deactivate=deactivate
|
||||
)
|
||||
|
||||
def load_installed_datatypes(self, repository, relative_install_dir, deactivate=False):
|
||||
"""
|
||||
Load proprietary datatypes and return information needed for loading custom
|
||||
datatypes converters and display applications later.
|
||||
"""
|
||||
metadata = repository.metadata_
|
||||
repository_dict = None
|
||||
datatypes_config = get_config_from_disk(DATATYPES_CONFIG_FILENAME, relative_install_dir)
|
||||
if datatypes_config:
|
||||
converter_path, display_path = self._alter_config_and_load_proprietary_datatypes(
|
||||
datatypes_config, relative_install_dir, deactivate=deactivate
|
||||
)
|
||||
if converter_path or display_path:
|
||||
# Create a dictionary of tool shed repository related information.
|
||||
repository_dict = self._create_repository_dict_for_proprietary_datatypes(
|
||||
tool_shed=repository.tool_shed,
|
||||
name=repository.name,
|
||||
owner=repository.owner,
|
||||
installed_changeset_revision=repository.installed_changeset_revision,
|
||||
tool_dicts=metadata.get("tools", []),
|
||||
converter_path=converter_path,
|
||||
display_path=display_path,
|
||||
)
|
||||
return repository_dict
|
||||
|
||||
def load_installed_display_applications(self, installed_repository_dict, deactivate=False):
|
||||
"""Load or deactivate custom datatype display applications."""
|
||||
self.app.datatypes_registry.load_display_applications(
|
||||
self.app, installed_repository_dict=installed_repository_dict, deactivate=deactivate
|
||||
)
|
||||
@@ -6,14 +6,7 @@ import logging
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from sqlalchemy import (
|
||||
and_,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
|
||||
from galaxy import util
|
||||
from galaxy.tool_shed.galaxy_install.datatypes import custom_datatype_manager
|
||||
from galaxy.tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import (
|
||||
InstalledRepositoryMetadataManager,
|
||||
)
|
||||
@@ -619,32 +612,6 @@ class InstalledRepositoryManager:
|
||||
deleted_tool_dependency_names.append(original_dependency_val_dict["name"])
|
||||
return updated_tool_dependency_names, deleted_tool_dependency_names
|
||||
|
||||
def load_proprietary_datatypes(self):
|
||||
cdl = custom_datatype_manager.CustomDatatypeLoader(self.app)
|
||||
for tool_shed_repository in (
|
||||
self.context.query(self.install_model.ToolShedRepository)
|
||||
.filter(
|
||||
and_(
|
||||
self.install_model.ToolShedRepository.table.c.includes_datatypes == true(),
|
||||
self.install_model.ToolShedRepository.table.c.deleted == false(),
|
||||
)
|
||||
)
|
||||
.order_by(self.install_model.ToolShedRepository.table.c.id)
|
||||
):
|
||||
relative_install_dir = self.get_repository_install_dir(tool_shed_repository)
|
||||
if relative_install_dir:
|
||||
installed_repository_dict = cdl.load_installed_datatypes(tool_shed_repository, relative_install_dir)
|
||||
if installed_repository_dict:
|
||||
self.installed_repository_dicts.append(installed_repository_dict)
|
||||
|
||||
def load_proprietary_converters_and_display_applications(self, deactivate=False):
|
||||
cdl = custom_datatype_manager.CustomDatatypeLoader(self.app)
|
||||
for installed_repository_dict in self.installed_repository_dicts:
|
||||
if installed_repository_dict["converter_path"]:
|
||||
cdl.load_installed_datatype_converters(installed_repository_dict, deactivate=deactivate)
|
||||
if installed_repository_dict["display_path"]:
|
||||
cdl.load_installed_display_applications(installed_repository_dict, deactivate=deactivate)
|
||||
|
||||
def uninstall_repository(self, repository, remove_from_disk=True):
|
||||
errors = ""
|
||||
shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir(
|
||||
|
||||
Reference in New Issue
Block a user