diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index f3d208869d3..ba065d78c59 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -327,19 +327,9 @@ class ConfiguresGalaxyMixin: if exc.errno != errno.ENOENT or self.config.is_set("shed_tool_data_table_config"): raise - def _configure_datatypes_registry( - self, installed_repository_manager=None, use_display_applications=True, use_converters=True - ): + def _configure_datatypes_registry(self, use_display_applications=True, use_converters=True): # Create an empty datatypes registry. self.datatypes_registry = Registry(self.config) - if installed_repository_manager and self.config.load_tool_shed_datatypes: - # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories. We - # load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some - # generic sniffers (eg text,xml) which catch anything, so it's impossible for proprietary sniffers to be used. - # However, if there is a conflict (2 datatypes with the same extension) between a proprietary datatype and a datatype - # in the Galaxy distribution, the datatype in the Galaxy distribution will take precedence. If there is a conflict - # between 2 proprietary datatypes, the datatype from the repository that was installed earliest will take precedence. - installed_repository_manager.load_proprietary_datatypes() # Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config. datatypes_configs = self.config.datatypes_config for datatypes_config in listify(datatypes_configs): @@ -583,7 +573,6 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication): ) self.dynamic_tool_manager = self._register_singleton(DynamicToolManager) self._configure_datatypes_registry( - self.installed_repository_manager, use_converters=use_converters, use_display_applications=use_display_applications, ) @@ -659,8 +648,6 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): self.update_repository_manager = self._register_singleton( UpdateRepositoryManager, UpdateRepositoryManager(self) ) - # Load proprietary datatype converters and display applications. - self.installed_repository_manager.load_proprietary_converters_and_display_applications() # Load datatype display applications defined in local datatypes_conf.xml self.datatypes_registry.load_display_applications(self) # Load datatype converters defined in local datatypes_conf.xml diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 7e78c567d4f..d1ed27adfc1 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -466,15 +466,6 @@ galaxy: # . #tool_sheds_config_file: tool_sheds_conf.xml - # This option controls whether legacy datatypes are loaded from - # installed tool shed repositories. We're are in the process of - # disabling Tool Shed datatypes. This option with a default of true - # will be added in 22.01, we will disable the datatypes on the big - # public servers during that release. This option will be switched to - # False by default in 22.05 and this broken functionality will be - # removed all together during some future release. - #load_tool_shed_datatypes: true - # Monitor the tools and tool directories listed in any tool config # file specified in tool_config_file option. If changes are found, # tools are automatically reloaded. Watchdog ( diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index 841cbe61a8b..d73f00840e0 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -443,17 +443,6 @@ mapping: File containing the Galaxy Tool Sheds that should be made available to install from in the admin interface (.sample used if default does not exist). - load_tool_shed_datatypes: - type: bool - default: true - required: false - desc: | - This option controls whether legacy datatypes are loaded from installed tool shed repositories. - We're are in the process of disabling Tool Shed datatypes. This option with a default of true - will be added in 22.01, we will disable the datatypes on the big public servers during that - release. This option will be switched to False by default in 22.05 and this broken functionality - will be removed all together during some future release. - watch_tools: type: str default: 'false' diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index c7431d846d2..746761d9a74 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -68,8 +68,6 @@ class Registry: # Converters defined in local datatypes_conf.xml self.converters = [] self.converter_tools = set() - # Converters defined in datatypes_conf.xml included in installed tool shed repositories. - self.proprietary_converters = [] self.converter_deps = {} self.available_tracks = [] self.set_external_metadata_tool = None @@ -77,9 +75,6 @@ class Registry: self.upload_file_formats = [] # Datatype elements defined in local datatypes_conf.xml that contain display applications. self.display_app_containers = [] - # Datatype elements in datatypes_conf.xml included in installed - # tool shed repositories that contain display applications. - self.proprietary_display_app_containers = [] # Map a display application id to a display application self.display_applications: Dict[str, DisplayApplication] = {} # The following 2 attributes are used in the to_xml_file() @@ -91,8 +86,6 @@ class Registry: # The 'default' display_path defined in local datatypes_conf.xml self.display_applications_path = None self.inherit_display_application_by_class = [] - # Keep a list of imported proprietary datatype class modules. - self.imported_modules = [] self.datatype_elems = [] self.datatype_info_dicts = [] self.sniffer_elems = [] @@ -109,7 +102,6 @@ class Registry: self, root_dir=None, config=None, - deactivate=False, override=True, use_converters=True, use_display_applications=True, @@ -117,10 +109,7 @@ class Registry: ): """ Parse a datatypes XML file located at root_dir/config (if processing the Galaxy distributed config) or contained within - an installed Tool Shed repository. If deactivate is True, an installed Tool Shed repository that includes custom datatypes - is being deactivated or uninstalled, so appropriate loaded datatypes will be removed from the registry. The value of - override will be False when a Tool Shed repository is being installed. Since installation is occurring after the datatypes - registry has been initialized at server startup, its contents cannot be overridden by newly introduced conflicting data types. + an installed Tool Shed repository. """ def __import_module(full_path: str, datatype_module: str): @@ -134,24 +123,13 @@ class Registry: return module if root_dir and config: - # If handling_proprietary_datatypes is determined as True below, we'll have an elem that looks something like this: - # compressed_sniffers = {} - handling_proprietary_datatypes = False if isinstance(config, (str, Path)): # Parse datatypes_conf.xml tree = galaxy.util.parse_xml(config) root = tree.getroot() # Load datatypes and converters from config - if deactivate: - self.log.debug(f"Deactivating datatypes from {config}") - else: - self.log.debug(f"Loading datatypes from {config}") + self.log.debug(f"Loading datatypes from {config}") else: root = config registration = root.find("registration") @@ -182,15 +160,7 @@ class Registry: os.path.join(os.path.dirname(__file__), "display_applications", "configs") ) self.display_applications_path = self.display_path_attr - # Proprietary datatype's tag may have special attributes, proprietary_converter_path and proprietary_display_path. - proprietary_converter_path = registration.get("proprietary_converter_path", None) - proprietary_display_path = registration.get("proprietary_display_path", None) - if ( - proprietary_converter_path is not None - or proprietary_display_path is not None - and not handling_proprietary_datatypes - ): - handling_proprietary_datatypes = True + for elem in registration.findall("datatype"): # Keep a status of the process steps to enable stopping the process of handling the datatype if necessary. ok = True @@ -217,263 +187,192 @@ class Registry: if edam_data and not make_subclass: self.log.warning("Cannot specify edam_data without setting subclass to True, skipping datatype.") continue - # Proprietary datatypes included in installed tool shed repositories will include two special attributes - # (proprietary_path and proprietary_datatype_module) if they depend on proprietary datatypes classes. - # The value of proprietary_path is the path to the cloned location of the tool shed repository's contained - # datatypes_conf.xml file. - proprietary_path = elem.get("proprietary_path", None) - proprietary_datatype_module = elem.get("proprietary_datatype_module", None) - if ( - proprietary_path is not None - or proprietary_datatype_module is not None - and not handling_proprietary_datatypes - ): - handling_proprietary_datatypes = True - if deactivate: - # We are deactivating or uninstalling an installed tool shed repository, so eliminate the datatype - # elem from the in-memory list of datatype elems. - for in_memory_elem in self.datatype_elems: - in_memory_extension = in_memory_elem.get("extension", None) - if in_memory_extension == extension: - in_memory_dtype = elem.get("type", None) - in_memory_type_extension = elem.get("type_extension", None) - in_memory_mimetype = elem.get("mimetype", None) - in_memory_display_in_upload = galaxy.util.string_as_bool( - elem.get("display_in_upload", False) - ) - in_memory_make_subclass = galaxy.util.string_as_bool(elem.get("subclass", False)) - if ( - in_memory_dtype == dtype - and in_memory_type_extension == type_extension - and in_memory_mimetype == mimetype - and in_memory_display_in_upload == display_in_upload - and in_memory_make_subclass == make_subclass - ): - self.datatype_elems.remove(in_memory_elem) - if extension is not None and extension in self.datatypes_by_extension: - # We are deactivating or uninstalling an installed tool shed repository, so eliminate the datatype - # from the registry. TODO: Handle deactivating datatype converters, etc before removing from - # self.datatypes_by_extension. - del self.datatypes_by_extension[extension] - if extension in self.upload_file_formats: - self.upload_file_formats.remove(extension) - self.log.debug(f"Removed datatype with extension '{extension}' from the registry.") - else: - # We are loading new datatype, so we'll make sure it is correctly defined before proceeding. - can_process_datatype = False - if extension is not None: - if dtype is not None or type_extension is not None: - if override or extension not in self.datatypes_by_extension: - can_process_datatype = True - if can_process_datatype: - if dtype is not None: - try: - fields = dtype.split(":") - datatype_module = fields[0] - datatype_class_name = fields[1] - except Exception: - self.log.exception("Error parsing datatype definition for dtype %s", str(dtype)) - ok = False - if ok: - datatype_class = None - if proprietary_path and proprietary_datatype_module and datatype_class_name: - # TODO: previously comments suggested this needs to be locked because it modifies - # the sys.path, probably true but the previous lock wasn't doing that. - try: - imported_module = __import_module(proprietary_path, proprietary_datatype_module) - if imported_module not in self.imported_modules: - self.imported_modules.append(imported_module) - if hasattr(imported_module, datatype_class_name): - datatype_class = getattr(imported_module, datatype_class_name) - except Exception as e: - full_path = os.path.join(proprietary_path, proprietary_datatype_module) - self.log.debug( - "Exception importing proprietary code file %s: %s", - full_path, - e, - ) - # Either the above exception was thrown because the proprietary_datatype_module is not derived from a class - # in the repository, or we are loading Galaxy's datatypes. In either case we'll look in the registry. - if datatype_class is None: - try: - # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution. - fields = datatype_module.split(".")[1:] - module = __import__(datatype_module) - for mod in fields: - module = getattr(module, mod) - datatype_class = getattr(module, datatype_class_name) - self.log.debug( - f"Retrieved datatype module {str(datatype_module)}:{datatype_class_name} from the datatype registry for extension {extension}." - ) - except Exception: - self.log.exception("Error importing datatype module %s", str(datatype_module)) - ok = False - elif type_extension is not None: - try: - datatype_class = self.datatypes_by_extension[type_extension].__class__ - self.log.debug( - f"Retrieved datatype module {str(datatype_class.__name__)} from type_extension {type_extension} for extension {extension}." - ) - except Exception: - self.log.exception( - "Error determining datatype_class for type_extension %s", str(type_extension) - ) - ok = False + + # We are loading new datatype, so we'll make sure it is correctly defined before proceeding. + can_process_datatype = False + if extension is not None: + if dtype is not None or type_extension is not None: + if override or extension not in self.datatypes_by_extension: + can_process_datatype = True + if can_process_datatype: + if dtype is not None: + try: + fields = dtype.split(":") + datatype_module = fields[0] + datatype_class_name = fields[1] + except Exception: + self.log.exception("Error parsing datatype definition for dtype %s", str(dtype)) + ok = False if ok: - if not deactivate: - # A new tool shed repository that contains custom datatypes is being installed, and since installation is - # occurring after the datatypes registry has been initialized at server startup, its contents cannot be - # overridden by new introduced conflicting data types unless the value of override is True. - if extension in self.datatypes_by_extension: - # Because of the way that the value of can_process_datatype was set above, we know that the value of - # override is True. + datatype_class = None + if datatype_class is None: + try: + # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution. + fields = datatype_module.split(".")[1:] + module = __import__(datatype_module) + for mod in fields: + module = getattr(module, mod) + datatype_class = getattr(module, datatype_class_name) self.log.debug( - "Overriding conflicting datatype with extension '%s', using datatype from %s." - % (str(extension), str(config)) + f"Retrieved datatype module {str(datatype_module)}:{datatype_class_name} from the datatype registry for extension {extension}." ) - if make_subclass: - datatype_class = type(datatype_class_name, (datatype_class,), {}) - if edam_format: - datatype_class.edam_format = edam_format - if edam_data: - datatype_class.edam_data = edam_data - datatype_class.is_subclass = make_subclass - description = elem.get("description", None) - description_url = elem.get("description_url", None) - datatype_instance = datatype_class() - self.datatypes_by_extension[extension] = datatype_instance - if mimetype is None: - # Use default mimetype per datatype specification. - mimetype = self.datatypes_by_extension[extension].get_mime() - self.mimetypes_by_extension[extension] = mimetype - if datatype_class.track_type: - self.available_tracks.append(extension) - if display_in_upload and extension not in self.upload_file_formats: - self.upload_file_formats.append(extension) - # Max file size cut off for setting optional metadata. - self.datatypes_by_extension[extension].max_optional_metadata_filesize = elem.get( - "max_optional_metadata_filesize", None - ) - infer_from_suffixes = [] - # read from element instead of attribute so we can customize references to - # compressed files in the future (e.g. maybe some day faz will be a compressed fasta - # or something along those lines) - for infer_from in elem.findall("infer_from"): - suffix = infer_from.get("suffix", None) - if suffix is None: - raise Exception("Failed to parse infer_from datatype element") - infer_from_suffixes.append(suffix) - self.datatypes_by_suffix_inferences[suffix] = datatype_instance - for converter in elem.findall("converter"): - # Build the list of datatype converters which will later be loaded into the calling app's toolbox. - converter_config = converter.get("file", None) - target_datatype = converter.get("target_datatype", None) - depends_on = converter.get("depends_on", None) - if depends_on is not None and target_datatype is not None: - if extension not in self.converter_deps: - self.converter_deps[extension] = {} - self.converter_deps[extension][target_datatype] = depends_on.split(",") - if converter_config and target_datatype: - if proprietary_converter_path: - self.proprietary_converters.append( - (converter_config, extension, target_datatype) - ) - else: - self.converters.append((converter_config, extension, target_datatype)) - # Add composite files. - for composite_file in elem.findall("composite_file"): - name = composite_file.get("name", None) - if name is None: - self.log.warning( - f"You must provide a name for your composite_file ({composite_file})." - ) - optional = composite_file.get("optional", False) - mimetype = composite_file.get("mimetype", None) - self.datatypes_by_extension[extension].add_composite_file( - name, optional=optional, mimetype=mimetype - ) - for _display_app in elem.findall("display"): - if proprietary_display_path: - if elem not in self.proprietary_display_app_containers: - self.proprietary_display_app_containers.append(elem) - else: - if elem not in self.display_app_containers: - self.display_app_containers.append(elem) - datatype_info_dict = { + except Exception: + self.log.exception("Error importing datatype module %s", str(datatype_module)) + ok = False + elif type_extension is not None: + try: + datatype_class = self.datatypes_by_extension[type_extension].__class__ + self.log.debug( + f"Retrieved datatype module {str(datatype_class.__name__)} from type_extension {type_extension} for extension {extension}." + ) + except Exception: + self.log.exception( + "Error determining datatype_class for type_extension %s", str(type_extension) + ) + ok = False + if ok: + # A new tool shed repository that contains custom datatypes is being installed, and since installation is + # occurring after the datatypes registry has been initialized at server startup, its contents cannot be + # overridden by new introduced conflicting data types unless the value of override is True. + if extension in self.datatypes_by_extension: + # Because of the way that the value of can_process_datatype was set above, we know that the value of + # override is True. + self.log.debug( + "Overriding conflicting datatype with extension '%s', using datatype from %s." + % (str(extension), str(config)) + ) + if make_subclass: + datatype_class = type(datatype_class_name, (datatype_class,), {}) + if edam_format: + datatype_class.edam_format = edam_format + if edam_data: + datatype_class.edam_data = edam_data + datatype_class.is_subclass = make_subclass + description = elem.get("description", None) + description_url = elem.get("description_url", None) + datatype_instance = datatype_class() + self.datatypes_by_extension[extension] = datatype_instance + if mimetype is None: + # Use default mimetype per datatype specification. + mimetype = self.datatypes_by_extension[extension].get_mime() + self.mimetypes_by_extension[extension] = mimetype + if datatype_class.track_type: + self.available_tracks.append(extension) + if display_in_upload and extension not in self.upload_file_formats: + self.upload_file_formats.append(extension) + # Max file size cut off for setting optional metadata. + self.datatypes_by_extension[extension].max_optional_metadata_filesize = elem.get( + "max_optional_metadata_filesize", None + ) + infer_from_suffixes = [] + # read from element instead of attribute so we can customize references to + # compressed files in the future (e.g. maybe some day faz will be a compressed fasta + # or something along those lines) + for infer_from in elem.findall("infer_from"): + suffix = infer_from.get("suffix", None) + if suffix is None: + raise Exception("Failed to parse infer_from datatype element") + infer_from_suffixes.append(suffix) + self.datatypes_by_suffix_inferences[suffix] = datatype_instance + for converter in elem.findall("converter"): + # Build the list of datatype converters which will later be loaded into the calling app's toolbox. + converter_config = converter.get("file", None) + target_datatype = converter.get("target_datatype", None) + depends_on = converter.get("depends_on", None) + if depends_on is not None and target_datatype is not None: + if extension not in self.converter_deps: + self.converter_deps[extension] = {} + self.converter_deps[extension][target_datatype] = depends_on.split(",") + if converter_config and target_datatype: + self.converters.append((converter_config, extension, target_datatype)) + # Add composite files. + for composite_file in elem.findall("composite_file"): + name = composite_file.get("name", None) + if name is None: + self.log.warning(f"You must provide a name for your composite_file ({composite_file}).") + optional = composite_file.get("optional", False) + mimetype = composite_file.get("mimetype", None) + self.datatypes_by_extension[extension].add_composite_file( + name, optional=optional, mimetype=mimetype + ) + for _display_app in elem.findall("display"): + if elem not in self.display_app_containers: + self.display_app_containers.append(elem) + datatype_info_dict = { + "display_in_upload": display_in_upload, + "extension": extension, + "description": description, + "description_url": description_url, + } + composite_files = datatype_instance.get_composite_files() + if composite_files: + _composite_files = [] + for name, composite_file in composite_files.items(): + _composite_file = composite_file.dict() + _composite_file["name"] = name + _composite_files.append(_composite_file) + datatype_info_dict["composite_files"] = _composite_files + self.datatype_info_dicts.append(datatype_info_dict) + + for auto_compressed_type in auto_compressed_types: + compressed_extension = f"{extension}.{auto_compressed_type}" + upper_compressed_type = auto_compressed_type[0].upper() + auto_compressed_type[1:] + auto_compressed_type_name = datatype_class_name + upper_compressed_type + attributes = {} + if auto_compressed_type == "gz": + dynamic_parent = binary.GzDynamicCompressedArchive + elif auto_compressed_type == "bz2": + dynamic_parent = binary.Bz2DynamicCompressedArchive + else: + raise Exception(f"Unknown auto compression type [{auto_compressed_type}]") + attributes["file_ext"] = compressed_extension + attributes["uncompressed_datatype_instance"] = datatype_instance + compressed_datatype_class = type( + auto_compressed_type_name, + ( + datatype_class, + dynamic_parent, + ), + attributes, + ) + if edam_format: + compressed_datatype_class.edam_format = edam_format + if edam_data: + compressed_datatype_class.edam_data = edam_data + compressed_datatype_instance = compressed_datatype_class() + self.datatypes_by_extension[compressed_extension] = compressed_datatype_instance + for suffix in infer_from_suffixes: + self.datatypes_by_suffix_inferences[ + f"{suffix}.{auto_compressed_type}" + ] = compressed_datatype_instance + if display_in_upload and compressed_extension not in self.upload_file_formats: + self.upload_file_formats.append(compressed_extension) + self.datatype_info_dicts.append( + { "display_in_upload": display_in_upload, - "extension": extension, + "extension": compressed_extension, "description": description, "description_url": description_url, } - composite_files = datatype_instance.get_composite_files() - if composite_files: - _composite_files = [] - for name, composite_file in composite_files.items(): - _composite_file = composite_file.dict() - _composite_file["name"] = name - _composite_files.append(_composite_file) - datatype_info_dict["composite_files"] = _composite_files - self.datatype_info_dicts.append(datatype_info_dict) - - for auto_compressed_type in auto_compressed_types: - compressed_extension = f"{extension}.{auto_compressed_type}" - upper_compressed_type = auto_compressed_type[0].upper() + auto_compressed_type[1:] - auto_compressed_type_name = datatype_class_name + upper_compressed_type - attributes = {} - if auto_compressed_type == "gz": - dynamic_parent = binary.GzDynamicCompressedArchive - elif auto_compressed_type == "bz2": - dynamic_parent = binary.Bz2DynamicCompressedArchive - else: - raise Exception(f"Unknown auto compression type [{auto_compressed_type}]") - attributes["file_ext"] = compressed_extension - attributes["uncompressed_datatype_instance"] = datatype_instance - compressed_datatype_class = type( - auto_compressed_type_name, - ( - datatype_class, - dynamic_parent, - ), - attributes, + ) + if auto_compressed_type == "gz": + self.converters.append( + ( + f"uncompressed_to_{auto_compressed_type}.xml", + extension, + compressed_extension, ) - if edam_format: - compressed_datatype_class.edam_format = edam_format - if edam_data: - compressed_datatype_class.edam_data = edam_data - compressed_datatype_instance = compressed_datatype_class() - self.datatypes_by_extension[compressed_extension] = compressed_datatype_instance - for suffix in infer_from_suffixes: - self.datatypes_by_suffix_inferences[ - f"{suffix}.{auto_compressed_type}" - ] = compressed_datatype_instance - if display_in_upload and compressed_extension not in self.upload_file_formats: - self.upload_file_formats.append(compressed_extension) - self.datatype_info_dicts.append( - { - "display_in_upload": display_in_upload, - "extension": compressed_extension, - "description": description, - "description_url": description_url, - } - ) - if auto_compressed_type == "gz": - self.converters.append( - ( - f"uncompressed_to_{auto_compressed_type}.xml", - extension, - compressed_extension, - ) - ) - self.converters.append( - (f"{auto_compressed_type}_to_uncompressed.xml", compressed_extension, extension) - ) - if datatype_class not in compressed_sniffers: - compressed_sniffers[datatype_class] = [] - if sniff_compressed_types: - compressed_sniffers[datatype_class].append(compressed_datatype_instance) - # Processing the new datatype elem is now complete, so make sure the element defining it is retained by appending - # the new datatype to the in-memory list of datatype elems to enable persistence. - self.datatype_elems.append(elem) + ) + self.converters.append( + (f"{auto_compressed_type}_to_uncompressed.xml", compressed_extension, extension) + ) + if datatype_class not in compressed_sniffers: + compressed_sniffers[datatype_class] = [] + if sniff_compressed_types: + compressed_sniffers[datatype_class].append(compressed_datatype_instance) + # Processing the new datatype elem is now complete, so make sure the element defining it is retained by appending + # the new datatype to the in-memory list of datatype elems to enable persistence. + self.datatype_elems.append(elem) else: if extension is not None: if dtype is not None or type_extension is not None: @@ -488,8 +387,6 @@ class Registry: # since sniffers are not tightly coupled with datatypes. self.load_datatype_sniffers( root, - deactivate=deactivate, - handling_proprietary_datatypes=handling_proprietary_datatypes, override=override, compressed_sniffers=compressed_sniffers, ) @@ -570,16 +467,10 @@ class Registry: def get_display_sites(self, site_type): return self.display_sites.get(site_type, []) - def load_datatype_sniffers( - self, root, deactivate=False, handling_proprietary_datatypes=False, override=False, compressed_sniffers=None - ): + def load_datatype_sniffers(self, root, override=False, compressed_sniffers=None): """ Process the sniffers element from a parsed a datatypes XML file located at root_dir/config (if processing the Galaxy - distributed config) or contained within an installed Tool Shed repository. If deactivate is True, an installed Tool - Shed repository that includes custom sniffers is being deactivated or uninstalled, so appropriate loaded sniffers will - be removed from the registry. The value of override will be False when a Tool Shed repository is being installed. - Since installation is occurring after the datatypes registry has been initialized at server startup, its contents - cannot be overridden by newly introduced conflicting sniffers. + distributed config) or contained within an installed Tool Shed repository. """ sniffer_elem_classes = [e.attrib["type"] for e in self.sniffer_elems] sniffers = root.find("sniffers") @@ -598,12 +489,6 @@ class Registry: self.log.exception("Error determining datatype class or module for dtype %s", str(dtype)) ok = False if ok: - if handling_proprietary_datatypes: - # See if one of the imported modules contains the datatype class name. - for imported_module in self.imported_modules: - if hasattr(imported_module, datatype_class_name): - module = imported_module - break if module is None: try: # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution. @@ -622,47 +507,27 @@ class Registry: ) ok = False if ok: - if deactivate: - # We are deactivating or uninstalling an installed Tool Shed repository, so eliminate the appropriate sniffers. - sniffer_class = elem.get("type", None) - if sniffer_class is not None: - for index, s_e_c in enumerate(sniffer_elem_classes): - if sniffer_class == s_e_c: - del self.sniffer_elems[index] - sniffer_elem_classes = [ - elem.attrib["type"] for elem in self.sniffer_elems - ] - self.log.debug(f"Removed sniffer element for datatype '{str(dtype)}'") - break - for sniffer_class in self.sniff_order: - if sniffer_class.__class__ == aclass.__class__: - self.sniff_order.remove(sniffer_class) - self.log.debug( - f"Removed sniffer class for datatype '{str(dtype)}' from sniff order" - ) - break - else: - # We are loading new sniffer, so see if we have a conflicting sniffer already loaded. - conflict = False - for conflict_loc, sniffer_class in enumerate(self.sniff_order): - if sniffer_class.__class__ == aclass.__class__: - # We have a conflicting sniffer, so replace the one previously loaded. - conflict = True - if override: - del self.sniff_order[conflict_loc] - self.log.debug(f"Removed conflicting sniffer for datatype '{dtype}'") - break - if not conflict or override: - if compressed_sniffers and aclass.__class__ in compressed_sniffers: - for compressed_sniffer in compressed_sniffers[aclass.__class__]: - self.sniff_order.append(compressed_sniffer) - self.sniff_order.append(aclass) - self.log.debug(f"Loaded sniffer for datatype '{dtype}'") - # Processing the new sniffer elem is now complete, so make sure the element defining it is loaded if necessary. - sniffer_class = elem.get("type", None) - if sniffer_class is not None: - if sniffer_class not in sniffer_elem_classes: - self.sniffer_elems.append(elem) + # We are loading new sniffer, so see if we have a conflicting sniffer already loaded. + conflict = False + for conflict_loc, sniffer_class in enumerate(self.sniff_order): + if sniffer_class.__class__ == aclass.__class__: + # We have a conflicting sniffer, so replace the one previously loaded. + conflict = True + if override: + del self.sniff_order[conflict_loc] + self.log.debug(f"Removed conflicting sniffer for datatype '{dtype}'") + break + if not conflict or override: + if compressed_sniffers and aclass.__class__ in compressed_sniffers: + for compressed_sniffer in compressed_sniffers[aclass.__class__]: + self.sniff_order.append(compressed_sniffer) + self.sniff_order.append(aclass) + self.log.debug(f"Loaded sniffer for datatype '{dtype}'") + # Processing the new sniffer elem is now complete, so make sure the element defining it is loaded if necessary. + sniffer_class = elem.get("type", None) + if sniffer_class is not None: + if sniffer_class not in sniffer_elem_classes: + self.sniffer_elems.append(elem) def get_datatype_from_filename(self, name): max_extension_parts = 3 @@ -734,149 +599,64 @@ class Registry: data.init_meta(copy_from=data) return data - def load_datatype_converters(self, toolbox, installed_repository_dict=None, deactivate=False, use_cached=False): + def load_datatype_converters(self, toolbox, use_cached=False): """ - If deactivate is False, add datatype converters from self.converters or self.proprietary_converters - to the calling app's toolbox. If deactivate is True, eliminates relevant converters from the calling - app's toolbox. + Add datatype converters from self.converters to the calling app's toolbox. """ - if installed_repository_dict: - # Load converters defined by datatypes_conf.xml included in installed tool shed repository. - converters = self.proprietary_converters - else: - # Load converters defined by local datatypes_conf.xml. - converters = self.converters + # Load converters defined by local datatypes_conf.xml. + converters = self.converters for elem in converters: tool_config = elem[0] source_datatype = elem[1] target_datatype = elem[2] - if installed_repository_dict: - converter_path = installed_repository_dict["converter_path"] - else: - converter_path = self.converters_path + converter_path = self.converters_path try: config_path = os.path.join(converter_path, tool_config) converter = toolbox.load_tool(config_path, use_cached=use_cached) self.converter_tools.add(converter) - if installed_repository_dict: - # If the converter is included in an installed tool shed repository, set the tool - # shed related tool attributes. - converter.tool_shed = installed_repository_dict["tool_shed"] - converter.repository_name = installed_repository_dict["repository_name"] - converter.repository_owner = installed_repository_dict["repository_owner"] - converter.installed_changeset_revision = installed_repository_dict["installed_changeset_revision"] - converter.old_id = converter.id - # The converter should be included in the list of tools defined in tool_dicts. - tool_dicts = installed_repository_dict["tool_dicts"] - for tool_dict in tool_dicts: - if tool_dict["id"] == converter.id: - converter.guid = tool_dict["guid"] - converter.id = tool_dict["guid"] - break - if deactivate: - toolbox.remove_tool_by_id(converter.id, remove_from_panel=False) - if source_datatype in self.datatype_converters: - if target_datatype in self.datatype_converters[source_datatype]: - del self.datatype_converters[source_datatype][target_datatype] - self.log.debug("Deactivated converter: %s", converter.id) - else: - toolbox.register_tool(converter) - if source_datatype not in self.datatype_converters: - self.datatype_converters[source_datatype] = {} - self.datatype_converters[source_datatype][target_datatype] = converter - if not hasattr(toolbox.app, "tool_cache") or converter.id in toolbox.app.tool_cache._new_tool_ids: - self.log.debug("Loaded converter: %s", converter.id) + toolbox.register_tool(converter) + if source_datatype not in self.datatype_converters: + self.datatype_converters[source_datatype] = {} + self.datatype_converters[source_datatype][target_datatype] = converter + if not hasattr(toolbox.app, "tool_cache") or converter.id in toolbox.app.tool_cache._new_tool_ids: + self.log.debug("Loaded converter: %s", converter.id) except Exception: - if deactivate: - self.log.exception(f"Error deactivating converter from ({converter_path})") - else: - self.log.exception(f"Error loading converter ({converter_path})") + self.log.exception(f"Error loading converter ({converter_path})") - def load_display_applications(self, app, installed_repository_dict=None, deactivate=False): + def load_display_applications(self, app): """ - If deactivate is False, add display applications from self.display_app_containers or - self.proprietary_display_app_containers to appropriate datatypes. If deactivate is - True, eliminates relevant display applications from appropriate datatypes. + Add display applications from self.display_app_containers or to appropriate datatypes. """ - if installed_repository_dict: - # Load display applications defined by datatypes_conf.xml included in installed tool shed repository. - datatype_elems = self.proprietary_display_app_containers - else: - # Load display applications defined by local datatypes_conf.xml. - datatype_elems = self.display_app_containers + # Load display applications defined by local datatypes_conf.xml. + datatype_elems = self.display_app_containers for elem in datatype_elems: extension = self.get_extension(elem) for display_app in elem.findall("display"): display_file = display_app.get("file", None) - if installed_repository_dict: - display_path = installed_repository_dict["display_path"] - display_file_head, display_file_tail = os.path.split(display_file) - config_path = os.path.join(display_path, display_file_tail) - else: - config_path = os.path.join(self.display_applications_path, display_file) + config_path = os.path.join(self.display_applications_path, display_file) try: inherit = galaxy.util.string_as_bool(display_app.get("inherit", "False")) display_app = DisplayApplication.from_file(config_path, app) if display_app: if display_app.id in self.display_applications: - if deactivate: - del self.display_applications[display_app.id] - else: - # If we already loaded this display application, we'll use the first one loaded. - display_app = self.display_applications[display_app.id] - elif installed_repository_dict: - # If the display application is included in an installed tool shed repository, - # set the tool shed related tool attributes. - display_app.tool_shed = installed_repository_dict["tool_shed"] - display_app.repository_name = installed_repository_dict["repository_name"] - display_app.repository_owner = installed_repository_dict["repository_owner"] - display_app.installed_changeset_revision = installed_repository_dict[ - "installed_changeset_revision" - ] - display_app.old_id = display_app.id - # The display application should be included in the list of tools defined in tool_dicts. - tool_dicts = installed_repository_dict["tool_dicts"] - for tool_dict in tool_dicts: - if tool_dict["id"] == display_app.id: - display_app.guid = tool_dict["guid"] - display_app.id = tool_dict["guid"] - break - if deactivate: - if display_app.id in self.display_applications: - del self.display_applications[display_app.id] - if extension in self.datatypes_by_extension: - if display_app.id in self.datatypes_by_extension[extension].display_applications: - del self.datatypes_by_extension[extension].display_applications[display_app.id] - if ( - inherit - and (self.datatypes_by_extension[extension], display_app) - in self.inherit_display_application_by_class - ): - self.inherit_display_application_by_class.remove( - (self.datatypes_by_extension[extension], display_app) - ) - self.log.debug( - f"Deactivated display application '{display_app.id}' for datatype '{extension}'." - ) - else: - self.display_applications[display_app.id] = display_app - self.datatypes_by_extension[extension].add_display_application(display_app) - if ( - inherit - and (self.datatypes_by_extension[extension], display_app) - not in self.inherit_display_application_by_class - ): - self.inherit_display_application_by_class.append( - (self.datatypes_by_extension[extension], display_app) - ) - self.log.debug( - f"Loaded display application '{display_app.id}' for datatype '{extension}', inherit={inherit}." + # If we already loaded this display application, we'll use the first one loaded. + display_app = self.display_applications[display_app.id] + + self.display_applications[display_app.id] = display_app + self.datatypes_by_extension[extension].add_display_application(display_app) + if ( + inherit + and (self.datatypes_by_extension[extension], display_app) + not in self.inherit_display_application_by_class + ): + self.inherit_display_application_by_class.append( + (self.datatypes_by_extension[extension], display_app) ) + self.log.debug( + f"Loaded display application '{display_app.id}' for datatype '{extension}', inherit={inherit}." + ) except Exception: - if deactivate: - self.log.exception(f"Error deactivating display application ({config_path})") - else: - self.log.exception(f"Error loading display application ({config_path})") + self.log.exception(f"Error loading display application ({config_path})") # Handle display_application subclass inheritance. for extension, d_type1 in self.datatypes_by_extension.items(): for d_type2, display_app in self.inherit_display_application_by_class: diff --git a/lib/galaxy/tool_shed/galaxy_install/datatypes/__init__.py b/lib/galaxy/tool_shed/galaxy_install/datatypes/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/lib/galaxy/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py b/lib/galaxy/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py deleted file mode 100644 index 8edc6bb7250..00000000000 --- a/lib/galaxy/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py +++ /dev/null @@ -1,182 +0,0 @@ -import logging -import os - -from galaxy.tool_shed.util.basic_util import strip_path -from galaxy.tool_shed.util.hg_util import get_config_from_disk -from galaxy.tool_shed.util.shed_util_common import DATATYPES_CONFIG_FILENAME -from galaxy.util.tool_shed.xml_util import parse_xml - -log = logging.getLogger(__name__) - - -class CustomDatatypeLoader: - def __init__(self, app): - self.app = app - - def _alter_config_and_load_proprietary_datatypes( - self, datatypes_config, relative_install_dir, deactivate=False, override=True - ): - """ - Parse a custom datatypes config (a datatypes_conf.xml file included in an installed - tool shed repository) and add information to appropriate element attributes that will - enable custom datatype class modules, datatypes converters and display applications - to be discovered and properly imported by the datatypes registry. The value of override - will be False when a tool shed repository is being installed. Since installation is - occurring after the datatypes registry has been initialized, the registry's contents - cannot be overridden by conflicting data types. - """ - tree, error_message = parse_xml(datatypes_config) - if tree is None: - return None, None - datatypes_config_root = tree.getroot() - registration = datatypes_config_root.find("registration") - if registration is None: - # We have valid XML, but not a valid custom datatypes definition. - return None, None - converter_path, display_path = self._get_converter_and_display_paths(registration, relative_install_dir) - if converter_path: - # Path to datatype converters - registration.attrib["proprietary_converter_path"] = converter_path - if display_path: - # Path to datatype display applications - registration.attrib["proprietary_display_path"] = display_path - relative_path_to_datatype_file_name = None - datatype_files = datatypes_config_root.find("datatype_files") - datatype_class_modules = [] - if datatype_files is not None: - # The tag set contains any number of tags. - # - # - # - # - # We'll add attributes to the datatype tag sets so that the modules can be properly imported - # by the datatypes registry. - for elem in datatype_files.findall("datatype_file"): - datatype_file_name = elem.get("name", None) - if datatype_file_name: - # Find the file in the installed repository. - for root, _dirs, files in os.walk(relative_install_dir): - if root.find(".hg") < 0: - for name in files: - if name == datatype_file_name: - datatype_class_modules.append(os.path.join(root, name)) - break - break - if datatype_class_modules: - for relative_path_to_datatype_file_name in datatype_class_modules: - datatype_file_name_path, datatype_file_name = os.path.split(relative_path_to_datatype_file_name) - for elem in registration.findall("datatype"): - # Handle 'type' attribute which should be something like one of the following: - # type="gmap:GmapDB" - # type="galaxy.datatypes.gmap:GmapDB" - dtype = elem.get("type", None) - if dtype: - fields = dtype.split(":") - proprietary_datatype_module = fields[0] - if proprietary_datatype_module.find(".") >= 0: - # Handle the case where datatype_module is "galaxy.datatypes.gmap". - proprietary_datatype_module = proprietary_datatype_module.split(".")[-1] - # The value of proprietary_path must be an absolute path due to job_working_directory. - elem.attrib["proprietary_path"] = os.path.abspath(datatype_file_name_path) - elem.attrib["proprietary_datatype_module"] = proprietary_datatype_module - # Load custom datatypes - self.app.datatypes_registry.load_datatypes( - root_dir=self.app.config.root, config=datatypes_config_root, deactivate=deactivate, override=override - ) - return converter_path, display_path - - def _create_repository_dict_for_proprietary_datatypes( - self, tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None - ): - return dict( - tool_shed=tool_shed, - repository_name=name, - repository_owner=owner, - installed_changeset_revision=installed_changeset_revision, - tool_dicts=tool_dicts, - converter_path=converter_path, - display_path=display_path, - ) - - def _get_converter_and_display_paths(self, registration_elem, relative_install_dir): - """ - Find the relative path to data type converters and display applications included - in installed tool shed repositories. - """ - converter_path = None - display_path = None - for elem in registration_elem.findall("datatype"): - if not converter_path: - # If any of the tag sets contain tags, set the converter_path - # if it is not already set. This requires developers to place all converters in the - # same subdirectory within the repository hierarchy. - for converter in elem.findall("converter"): - converter_config = converter.get("file", None) - if converter_config: - converter_config_file_name = strip_path(converter_config) - for root, _dirs, files in os.walk(relative_install_dir): - if root.find(".hg") < 0: - for name in files: - if name == converter_config_file_name: - # The value of converter_path must be absolute due to job_working_directory. - converter_path = os.path.abspath(root) - break - if converter_path: - break - if not display_path: - # If any of the tag sets contain tags, set the display_path - # if it is not already set. This requires developers to place all display acpplications - # in the same subdirectory within the repository hierarchy. - for display_app in elem.findall("display"): - display_config = display_app.get("file", None) - if display_config: - display_config_file_name = strip_path(display_config) - for root, _dirs, files in os.walk(relative_install_dir): - if root.find(".hg") < 0: - for name in files: - if name == display_config_file_name: - # The value of display_path must be absolute due to job_working_directory. - display_path = os.path.abspath(root) - break - if display_path: - break - if converter_path and display_path: - break - return converter_path, display_path - - def load_installed_datatype_converters(self, installed_repository_dict, deactivate=False): - """Load or deactivate proprietary datatype converters.""" - self.app.datatypes_registry.load_datatype_converters( - self.app.toolbox, installed_repository_dict=installed_repository_dict, deactivate=deactivate - ) - - def load_installed_datatypes(self, repository, relative_install_dir, deactivate=False): - """ - Load proprietary datatypes and return information needed for loading custom - datatypes converters and display applications later. - """ - metadata = repository.metadata_ - repository_dict = None - datatypes_config = get_config_from_disk(DATATYPES_CONFIG_FILENAME, relative_install_dir) - if datatypes_config: - converter_path, display_path = self._alter_config_and_load_proprietary_datatypes( - datatypes_config, relative_install_dir, deactivate=deactivate - ) - if converter_path or display_path: - # Create a dictionary of tool shed repository related information. - repository_dict = self._create_repository_dict_for_proprietary_datatypes( - tool_shed=repository.tool_shed, - name=repository.name, - owner=repository.owner, - installed_changeset_revision=repository.installed_changeset_revision, - tool_dicts=metadata.get("tools", []), - converter_path=converter_path, - display_path=display_path, - ) - return repository_dict - - def load_installed_display_applications(self, installed_repository_dict, deactivate=False): - """Load or deactivate custom datatype display applications.""" - self.app.datatypes_registry.load_display_applications( - self.app, installed_repository_dict=installed_repository_dict, deactivate=deactivate - ) diff --git a/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py b/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py index 9587a418deb..fd425b681a4 100644 --- a/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py @@ -6,14 +6,7 @@ import logging import os import shutil -from sqlalchemy import ( - and_, - false, - true, -) - from galaxy import util -from galaxy.tool_shed.galaxy_install.datatypes import custom_datatype_manager from galaxy.tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import ( InstalledRepositoryMetadataManager, ) @@ -619,32 +612,6 @@ class InstalledRepositoryManager: deleted_tool_dependency_names.append(original_dependency_val_dict["name"]) return updated_tool_dependency_names, deleted_tool_dependency_names - def load_proprietary_datatypes(self): - cdl = custom_datatype_manager.CustomDatatypeLoader(self.app) - for tool_shed_repository in ( - self.context.query(self.install_model.ToolShedRepository) - .filter( - and_( - self.install_model.ToolShedRepository.table.c.includes_datatypes == true(), - self.install_model.ToolShedRepository.table.c.deleted == false(), - ) - ) - .order_by(self.install_model.ToolShedRepository.table.c.id) - ): - relative_install_dir = self.get_repository_install_dir(tool_shed_repository) - if relative_install_dir: - installed_repository_dict = cdl.load_installed_datatypes(tool_shed_repository, relative_install_dir) - if installed_repository_dict: - self.installed_repository_dicts.append(installed_repository_dict) - - def load_proprietary_converters_and_display_applications(self, deactivate=False): - cdl = custom_datatype_manager.CustomDatatypeLoader(self.app) - for installed_repository_dict in self.installed_repository_dicts: - if installed_repository_dict["converter_path"]: - cdl.load_installed_datatype_converters(installed_repository_dict, deactivate=deactivate) - if installed_repository_dict["display_path"]: - cdl.load_installed_display_applications(installed_repository_dict, deactivate=deactivate) - def uninstall_repository(self, repository, remove_from_disk=True): errors = "" shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir(