From 62959eec39daf2d390da520054ed2ec674506427 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 27 Feb 2022 14:21:52 +0100 Subject: [PATCH 01/27] Move app-related functions and classes to galaxy.app --- lib/galaxy/app.py | 261 +++++++++++++++++- lib/galaxy/config/__init__.py | 358 +++++-------------------- lib/galaxy/web/formatting.py | 32 --- lib/galaxy/web_stack/__init__.py | 4 - lib/tool_shed/webapp/config.py | 2 +- test/unit/config/test_config_values.py | 2 +- 6 files changed, 322 insertions(+), 337 deletions(-) delete mode 100644 lib/galaxy/web/formatting.py diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 0b131b4b3dc..b2b9090385c 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -1,20 +1,28 @@ +import collections +import errno import logging +import os import signal import sys +import threading import time from typing import Any, Callable, List, Tuple +from beaker.cache import CacheManager +from beaker.util import parse_cache_config_options + import galaxy.model import galaxy.model.security import galaxy.queues import galaxy.security -from galaxy import auth, config, jobs +from galaxy import auth, config, jobs, tools from galaxy.config_watchers import ConfigWatchers from galaxy.containers import build_container_interfaces from galaxy.datatypes.registry import Registry from galaxy.files import ConfiguredFileSources from galaxy.job_metrics import JobMetrics from galaxy.managers.api_keys import ApiKeyManager +from galaxy.managers.citations import CitationsManager from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.folders import FolderManager from galaxy.managers.hdas import HDAManager @@ -31,19 +39,23 @@ from galaxy.managers.workflows import ( WorkflowContentsManager, WorkflowsManager, ) +from galaxy.model import mapping from galaxy.model.base import SharedModelMapping from galaxy.model.database_heartbeat import DatabaseHeartbeat +from galaxy.model.database_utils import database_exists from galaxy.model.mapping import GalaxyModelMapping from galaxy.model.scoped_session import ( galaxy_scoped_session, install_model_scoped_session, ) from galaxy.model.tags import GalaxyTagHandler +from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database from galaxy.queue_worker import ( GalaxyQueueWorker, send_local_control_task, ) from galaxy.quota import get_quota_agent, QuotaAgent +from galaxy.schema.fields import BaseDatabaseIdField from galaxy.security.idencoding import IdEncodingHelper from galaxy.security.vault import ( Vault, @@ -51,21 +63,27 @@ from galaxy.security.vault import ( ) from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager +from galaxy.tool_util.deps import containers +from galaxy.tool_util.deps.dependencies import AppInfo from galaxy.tool_util.deps.views import DependencyResolversView from galaxy.tool_util.verify.test_data import TestDataResolver +from galaxy.tools.biotools import get_galaxy_biotools_metadata_source from galaxy.tools.cache import ( ToolCache, ToolShedRepositoryCache ) from galaxy.tools.data_manager.manager import DataManagers from galaxy.tools.error_reports import ErrorReports +from galaxy.tools.search import ToolBoxSearch from galaxy.tools.special_tools import load_lib_tools from galaxy.tours import build_tours_registry, ToursRegistry from galaxy.util import ( ExecutionTimer, heartbeat, + listify, StructuredExecutionTimer, ) +from galaxy.util.dbkeys import GenomeBuilds from galaxy.util.task import IntervalTask from galaxy.visualization.data_providers.registry import DataProviderRegistry from galaxy.visualization.genomes import Genomes @@ -125,10 +143,241 @@ class SentryClientMixin: self.application_stack.register_postfork_function(postfork_sentry_client) -class MinimalGalaxyApplication(BasicSharedApp, config.ConfiguresGalaxyMixin, HaltableContainer, SentryClientMixin): +def get_database_url(config): + db_url = config.database_connection + return db_url + + +def init_models_from_config(config, map_install_models=False, object_store=None, trace_logger=None): + db_url = get_database_url(config) + model = mapping.init( + config.file_path, + db_url, + config.database_engine_options, + map_install_models=map_install_models, + database_query_profiling_proxy=config.database_query_profiling_proxy, + object_store=object_store, + trace_logger=trace_logger, + use_pbkdf2=config.get_bool('use_pbkdf2', True), + slow_query_log_threshold=config.slow_query_log_threshold, + thread_local_log=config.thread_local_log, + log_query_counts=config.database_log_query_counts, + ) + return model + + +class ConfiguresGalaxyMixin: + """Shared code for configuring Galaxy-like app objects.""" + + config: config.GalaxyAppConfiguration + tool_cache: ToolCache + job_config: jobs.JobConfiguration + toolbox: tools.ToolBox + toolbox_search: ToolBoxSearch + container_finder: containers.ContainerFinder + + def _configure_genome_builds(self, data_table_name="__dbkeys__", load_old_style=True): + self.genome_builds = GenomeBuilds(self, data_table_name=data_table_name, load_old_style=load_old_style) + + def wait_for_toolbox_reload(self, old_toolbox): + timer = ExecutionTimer() + log.debug('Waiting for toolbox reload') + # Wait till toolbox reload has been triggered (or more than 60 seconds have passed) + while timer.elapsed < 60: + if self.toolbox.has_reloaded(old_toolbox): + log.debug('Finished waiting for toolbox reload %s', timer) + break + time.sleep(0.1) + else: + log.warning('Waiting for toolbox reload timed out after 60 seconds') + + def _configure_tool_config_files(self): + if self.config.shed_tool_config_file not in self.config.tool_configs: + self.config.tool_configs.append(self.config.shed_tool_config_file) + # The value of migrated_tools_config is the file reserved for containing only those tools that have been + # eliminated from the distribution and moved to the tool shed. If migration checking is disabled, only add it if + # it exists (since this may be an existing deployment where migrations were previously run). + if (os.path.exists(self.config.migrated_tools_config) + and self.config.migrated_tools_config not in self.config.tool_configs): + self.config.tool_configs.append(self.config.migrated_tools_config) + + def _configure_toolbox(self): + if not isinstance(self, BasicSharedApp): + raise Exception("Must inherit from BasicSharedApp") + + self.citations_manager = CitationsManager(self) + self.biotools_metadata_source = get_galaxy_biotools_metadata_source(self.config) + + from galaxy.managers.tools import DynamicToolManager + + self.dynamic_tools_manager = DynamicToolManager(self) + self._toolbox_lock = threading.RLock() + self.toolbox = tools.ToolBox(self.config.tool_configs, self.config.tool_path, self) + galaxy_root_dir = os.path.abspath(self.config.root) + file_path = os.path.abspath(self.config.file_path) + app_info = AppInfo( + galaxy_root_dir=galaxy_root_dir, + default_file_path=file_path, + tool_data_path=self.config.tool_data_path, + shed_tool_data_path=self.config.shed_tool_data_path, + outputs_to_working_directory=self.config.outputs_to_working_directory, + container_image_cache_path=self.config.container_image_cache_path, + library_import_dir=self.config.library_import_dir, + enable_mulled_containers=self.config.enable_mulled_containers, + container_resolvers_config_file=self.config.container_resolvers_config_file, + container_resolvers_config_dict=self.config.container_resolvers, + involucro_path=self.config.involucro_path, + involucro_auto_init=self.config.involucro_auto_init, + mulled_channels=self.config.mulled_channels, + ) + mulled_resolution_cache = None + if self.config.mulled_resolution_cache_type: + cache_opts = { + 'cache.type': self.config.mulled_resolution_cache_type, + 'cache.data_dir': self.config.mulled_resolution_cache_data_dir, + 'cache.lock_dir': self.config.mulled_resolution_cache_lock_dir, + } + mulled_resolution_cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache('mulled_resolution') + self.container_finder = containers.ContainerFinder(app_info, mulled_resolution_cache=mulled_resolution_cache) + self._set_enabled_container_types() + index_help = getattr(self.config, "index_tool_help", True) + self.toolbox_search = ToolBoxSearch(self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help) + + def reindex_tool_search(self): + # Call this when tools are added or removed. + self.toolbox_search.build_index(tool_cache=self.tool_cache) + self.tool_cache.reset_status() + + def _set_enabled_container_types(self): + container_types_to_destinations = collections.defaultdict(list) + for destinations in self.job_config.destinations.values(): + for destination in destinations: + for enabled_container_type in self.container_finder._enabled_container_types(destination.params): + container_types_to_destinations[enabled_container_type].append(destination) + self.toolbox.dependency_manager.set_enabled_container_types(container_types_to_destinations) + self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.default_container_registry.resolver_classes) + self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.default_container_registry.container_resolvers) + + def _configure_tool_data_tables(self, from_shed_config): + from galaxy.tools.data import ToolDataTableManager + + # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path. + self.tool_data_tables = ToolDataTableManager(tool_data_path=self.config.tool_data_path, + config_filename=self.config.tool_data_table_config_path, + other_config_dict=self.config) + # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables. + try: + self.tool_data_tables.load_from_config_file(config_filename=self.config.shed_tool_data_table_config, + tool_data_path=self.tool_data_tables.tool_data_path, + from_shed_config=from_shed_config) + except OSError as exc: + # Missing shed_tool_data_table_config is okay if it's the default + if exc.errno != errno.ENOENT or self.config.is_set('shed_tool_data_table_config'): + raise + + def _configure_datatypes_registry(self, installed_repository_manager=None): + from galaxy.datatypes import registry + # Create an empty datatypes registry. + self.datatypes_registry = registry.Registry(self.config) + if installed_repository_manager and self.config.load_tool_shed_datatypes: + # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories. We + # load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some + # generic sniffers (eg text,xml) which catch anything, so it's impossible for proprietary sniffers to be used. + # However, if there is a conflict (2 datatypes with the same extension) between a proprietary datatype and a datatype + # in the Galaxy distribution, the datatype in the Galaxy distribution will take precedence. If there is a conflict + # between 2 proprietary datatypes, the datatype from the repository that was installed earliest will take precedence. + installed_repository_manager.load_proprietary_datatypes() + # Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config. + datatypes_configs = self.config.datatypes_config + for datatypes_config in listify(datatypes_configs): + # Setting override=False would make earlier files would take + # precedence - but then they wouldn't override tool shed + # datatypes. + self.datatypes_registry.load_datatypes(self.config.root, datatypes_config, override=True) + + def _configure_object_store(self, **kwds): + from galaxy.objectstore import build_object_store_from_config + self.object_store = build_object_store_from_config(self.config, **kwds) + + def _configure_security(self): + from galaxy.security import idencoding + self.security = idencoding.IdEncodingHelper(id_secret=self.config.id_secret) + BaseDatabaseIdField.security = self.security + + def _configure_tool_shed_registry(self): + import galaxy.tool_shed.tool_shed_registry + + # Set up the tool sheds registry + if os.path.isfile(self.config.tool_sheds_config_file): + self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry(self.config.tool_sheds_config_file) + else: + self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry() + + def _configure_models(self, check_migrate_databases=False, config_file=None): + """Preconditions: object_store must be set on self.""" + db_url = get_database_url(self.config) + install_db_url = self.config.install_database_connection + # TODO: Consider more aggressive check here that this is not the same + # database file under the hood. + combined_install_database = not(install_db_url and install_db_url != db_url) + install_db_url = install_db_url or db_url + install_database_options = self.config.database_engine_options if combined_install_database else self.config.install_database_engine_options + + if self.config.database_wait: + self._wait_for_database(db_url) + + if getattr(self.config, "max_metadata_value_size", None): + from galaxy.model import custom_types + custom_types.MAX_METADATA_VALUE_SIZE = self.config.max_metadata_value_size + + if check_migrate_databases: + # Initialize database / check for appropriate schema version. # If this + # is a new installation, we'll restrict the tool migration messaging. + from galaxy.model.migrate.check import create_or_verify_database + create_or_verify_database(db_url, config_file, self.config.database_engine_options, app=self, map_install_models=combined_install_database) + if not combined_install_database: + tsi_create_or_verify_database(install_db_url, install_database_options, app=self) + + self.model = init_models_from_config( + self.config, + map_install_models=combined_install_database, + object_store=self.object_store, + trace_logger=getattr(self, "trace_logger", None) + ) + if combined_install_database: + log.info("Install database targetting Galaxy's database configuration.") + self.install_model = self.model + else: + from galaxy.model.tool_shed_install import mapping as install_mapping + install_db_url = self.config.install_database_connection + log.info(f"Install database using its own connection {install_db_url}") + self.install_model = install_mapping.init(install_db_url, + install_database_options) + + def _configure_signal_handlers(self, handlers): + for sig, handler in handlers.items(): + signal.signal(sig, handler) + + def _wait_for_database(self, url): + attempts = self.config.database_wait_attempts + pause = self.config.database_wait_sleep + for i in range(1, attempts): + try: + database_exists(url) + break + except Exception: + log.info("Waiting for database: attempt %d of %d" % (i, attempts)) + time.sleep(pause) + + @property + def tool_dependency_dir(self): + return self.toolbox.dependency_manager.default_base_path + + +class MinimalGalaxyApplication(BasicSharedApp, ConfiguresGalaxyMixin, HaltableContainer, SentryClientMixin): """Encapsulates the state of a minimal Galaxy application""" - def __init__(self, fsmon=False, configure_logging=True, **kwargs) -> None: + def __init__(self, fsmon=False, **kwargs) -> None: super().__init__() self.haltables = [ ("object store", self._shutdown_object_store), @@ -147,8 +396,6 @@ class MinimalGalaxyApplication(BasicSharedApp, config.ConfiguresGalaxyMixin, Hal # Read config file and check for errors self.config: Any = self._register_singleton(config.Configuration, config.Configuration(**kwargs)) self.config.check() - if configure_logging: - config.configure_logging(self.config) self._configure_object_store(fsmon=True) config_file = kwargs.get('global_conf', {}).get('__file__', None) if config_file: @@ -179,12 +426,14 @@ class MinimalGalaxyApplication(BasicSharedApp, config.ConfiguresGalaxyMixin, Hal class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication): """Extends the MinimalGalaxyApplication with most managers that are not tied to a web or job handling context.""" - def __init__(self, **kwargs): + def __init__(self, configure_logging=True, **kwargs): super().__init__(**kwargs) self._register_singleton(MinimalManagerApp, self) self.execution_timer_factory = self._register_singleton(ExecutionTimerFactory, ExecutionTimerFactory(self.config)) self.configure_fluent_log() self.application_stack = self._register_singleton(ApplicationStack, application_stack_instance(app=self)) + if configure_logging: + config.configure_logging(self.config, self.application_stack.facts) # Initialize job metrics manager, needs to be in place before # config so per-destination modifications can be made. self.job_metrics = self._register_singleton(JobMetrics, JobMetrics(self.config.job_metrics_config_file, app=self)) diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 49073dfd3fa..50806e4177d 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -3,21 +3,18 @@ Universe configuration builder. """ # absolute_import needed for tool_shed package. -import collections import configparser -import errno import ipaddress +import locale import logging import logging.config import os import re -import signal import socket import string import sys import tempfile import threading -import time from datetime import timedelta from typing import ( Any, @@ -28,39 +25,28 @@ from typing import ( Optional, Set, SupportsInt, - TYPE_CHECKING, TypeVar, Union, ) import yaml -from beaker.cache import CacheManager -from beaker.util import parse_cache_config_options from galaxy.config.schema import AppSchema from galaxy.containers import parse_containers_config from galaxy.exceptions import ConfigurationError -from galaxy.model import mapping -from galaxy.model.database_utils import database_exists -from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database -from galaxy.schema.fields import BaseDatabaseIdField -from galaxy.structured_app import BasicSharedApp from galaxy.util import ( - ExecutionTimer, listify, string_as_bool, unicodify, ) from galaxy.util.custom_logging import LOGLV_TRACE -from galaxy.util.dbkeys import GenomeBuilds from galaxy.util.dynamic import HasDynamicProperties +from galaxy.util.facts import get_facts from galaxy.util.properties import ( find_config_file, read_properties_from_file, running_from_source, ) -from galaxy.web.formatting import expand_pretty_datetime_format -from galaxy.web_stack import get_stack_facts from ..version import VERSION_MAJOR, VERSION_MINOR try: @@ -69,15 +55,11 @@ except ImportError: # Python < 3.9 from importlib_resources import files # type: ignore[no-redef] -if TYPE_CHECKING: - from galaxy.jobs import JobConfiguration - from galaxy.tool_util.deps.containers import ContainerFinder - from galaxy.tools import ToolBox - from galaxy.tools.cache import ToolCache - from galaxy.tools.search import ToolBoxSearch - log = logging.getLogger(__name__) +DEFAULT_LOCALE_FORMAT = '%a %b %e %H:%M:%S %Y' +ISO_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" + GALAXY_APP_NAME = 'galaxy' GALAXY_SCHEMAS_PATH = files('galaxy.config') / 'schemas' GALAXY_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / 'config_schema.yml' @@ -138,10 +120,70 @@ LOGGING_CONFIG_DEFAULT: Dict[str, Any] = { """Default value for logging configuration, passed to :func:`logging.config.dictConfig`""" +def configure_logging(config, facts=None): + """Allow some basic logging configuration to be read from ini file. + + This should be able to consume either a galaxy.config.Configuration object + or a simple dictionary of configuration variables. + """ + facts = facts or get_facts(config=config) + # Get root logger + logging.addLevelName(LOGLV_TRACE, "TRACE") + # PasteScript will have already configured the logger if the + # 'loggers' section was found in the config file, otherwise we do + # some simple setup using the 'log_*' values from the config. + parser = getattr(config, "global_conf_parser", None) + if parser: + paste_configures_logging = config.global_conf_parser.has_section("loggers") + else: + paste_configures_logging = False + auto_configure_logging = not paste_configures_logging and string_as_bool(config.get("auto_configure_logging", "True")) + if auto_configure_logging: + logging_conf = config.get('logging', None) + if logging_conf is None: + # if using the default logging config, honor the log_level setting + logging_conf = LOGGING_CONFIG_DEFAULT + if config.get('log_level', 'DEBUG') != 'DEBUG': + logging_conf['handlers']['console']['level'] = config.get('log_level', 'DEBUG') + # configure logging with logging dict in config, template *FileHandler handler filenames with the `filename_template` option + for name, conf in logging_conf.get('handlers', {}).items(): + if conf['class'].startswith('logging.') and conf['class'].endswith('FileHandler') and 'filename_template' in conf: + conf['filename'] = conf.pop('filename_template').format(**get_facts(config=config)) + logging_conf['handlers'][name] = conf + logging.config.dictConfig(logging_conf) + + def find_root(kwargs): return os.path.abspath(kwargs.get('root_dir', '.')) +def expand_pretty_datetime_format(value): + """ + + >>> expand_pretty_datetime_format("%H:%M:%S %Z") + '%H:%M:%S %Z' + >>> locale_format = expand_pretty_datetime_format("$locale (UTC)") + >>> import locale + >>> expected_format = '%s (UTC)' % locale.nl_langinfo(locale.D_T_FMT) + >>> locale_format == expected_format + True + >>> expand_pretty_datetime_format("$iso8601") + '%Y-%m-%d %H:%M:%S' + """ + locale_format = None + try: + locale_format = locale.nl_langinfo(locale.D_T_FMT) + except AttributeError: # nl_langinfo not available + pass + if not locale_format: + locale_format = DEFAULT_LOCALE_FORMAT + stock_formats = dict( + locale=locale_format, + iso8601=ISO_DATETIME_FORMAT, + ) + return string.Template(value).safe_substitute(**stock_formats) + + OptStr = TypeVar("OptStr", None, str) @@ -1182,273 +1224,3 @@ def get_database_engine_options(kwargs, model_prefix=''): value = conversions[key](value) rval[key] = value return rval - - -def get_database_url(config): - db_url = config.database_connection - return db_url - - -def init_models_from_config(config, map_install_models=False, object_store=None, trace_logger=None): - db_url = get_database_url(config) - model = mapping.init( - config.file_path, - db_url, - config.database_engine_options, - map_install_models=map_install_models, - database_query_profiling_proxy=config.database_query_profiling_proxy, - object_store=object_store, - trace_logger=trace_logger, - use_pbkdf2=config.get_bool('use_pbkdf2', True), - slow_query_log_threshold=config.slow_query_log_threshold, - thread_local_log=config.thread_local_log, - log_query_counts=config.database_log_query_counts, - ) - return model - - -def configure_logging(config): - """Allow some basic logging configuration to be read from ini file. - - This should be able to consume either a galaxy.config.Configuration object - or a simple dictionary of configuration variables. - """ - # Get root logger - logging.addLevelName(LOGLV_TRACE, "TRACE") - # PasteScript will have already configured the logger if the - # 'loggers' section was found in the config file, otherwise we do - # some simple setup using the 'log_*' values from the config. - parser = getattr(config, "global_conf_parser", None) - if parser: - paste_configures_logging = config.global_conf_parser.has_section("loggers") - else: - paste_configures_logging = False - auto_configure_logging = not paste_configures_logging and string_as_bool(config.get("auto_configure_logging", "True")) - if auto_configure_logging: - logging_conf = config.get('logging', None) - if logging_conf is None: - # if using the default logging config, honor the log_level setting - logging_conf = LOGGING_CONFIG_DEFAULT - if config.get('log_level', 'DEBUG') != 'DEBUG': - logging_conf['handlers']['console']['level'] = config.get('log_level', 'DEBUG') - # configure logging with logging dict in config, template *FileHandler handler filenames with the `filename_template` option - for name, conf in logging_conf.get('handlers', {}).items(): - if conf['class'].startswith('logging.') and conf['class'].endswith('FileHandler') and 'filename_template' in conf: - conf['filename'] = conf.pop('filename_template').format(**get_stack_facts(config=config)) - logging_conf['handlers'][name] = conf - logging.config.dictConfig(logging_conf) - - -class ConfiguresGalaxyMixin: - """Shared code for configuring Galaxy-like app objects.""" - - config: GalaxyAppConfiguration - tool_cache: "ToolCache" - job_config: "JobConfiguration" - toolbox: "ToolBox" - toolbox_search: "ToolBoxSearch" - container_finder: "ContainerFinder" - - def _configure_genome_builds(self, data_table_name="__dbkeys__", load_old_style=True): - self.genome_builds = GenomeBuilds(self, data_table_name=data_table_name, load_old_style=load_old_style) - - def wait_for_toolbox_reload(self, old_toolbox): - timer = ExecutionTimer() - log.debug('Waiting for toolbox reload') - # Wait till toolbox reload has been triggered (or more than 60 seconds have passed) - while timer.elapsed < 60: - if self.toolbox.has_reloaded(old_toolbox): - log.debug('Finished waiting for toolbox reload %s', timer) - break - time.sleep(0.1) - else: - log.warning('Waiting for toolbox reload timed out after 60 seconds') - - def _configure_tool_config_files(self): - if self.config.shed_tool_config_file not in self.config.tool_configs: - self.config.tool_configs.append(self.config.shed_tool_config_file) - # The value of migrated_tools_config is the file reserved for containing only those tools that have been - # eliminated from the distribution and moved to the tool shed. If migration checking is disabled, only add it if - # it exists (since this may be an existing deployment where migrations were previously run). - if (os.path.exists(self.config.migrated_tools_config) - and self.config.migrated_tools_config not in self.config.tool_configs): - self.config.tool_configs.append(self.config.migrated_tools_config) - - def _configure_toolbox(self): - from galaxy import tools - from galaxy.tools.biotools import get_galaxy_biotools_metadata_source - from galaxy.managers.citations import CitationsManager - from galaxy.tool_util.deps import containers - from galaxy.tool_util.deps.dependencies import AppInfo - import galaxy.tools.search - - if not isinstance(self, BasicSharedApp): - raise Exception("Must inherit from BasicSharedApp") - - self.citations_manager = CitationsManager(self) - self.biotools_metadata_source = get_galaxy_biotools_metadata_source(self.config) - - from galaxy.managers.tools import DynamicToolManager - - self.dynamic_tools_manager = DynamicToolManager(self) - self._toolbox_lock = threading.RLock() - self.toolbox = tools.ToolBox(self.config.tool_configs, self.config.tool_path, self) - galaxy_root_dir = os.path.abspath(self.config.root) - file_path = os.path.abspath(self.config.file_path) - app_info = AppInfo( - galaxy_root_dir=galaxy_root_dir, - default_file_path=file_path, - tool_data_path=self.config.tool_data_path, - shed_tool_data_path=self.config.shed_tool_data_path, - outputs_to_working_directory=self.config.outputs_to_working_directory, - container_image_cache_path=self.config.container_image_cache_path, - library_import_dir=self.config.library_import_dir, - enable_mulled_containers=self.config.enable_mulled_containers, - container_resolvers_config_file=self.config.container_resolvers_config_file, - container_resolvers_config_dict=self.config.container_resolvers, - involucro_path=self.config.involucro_path, - involucro_auto_init=self.config.involucro_auto_init, - mulled_channels=self.config.mulled_channels, - ) - mulled_resolution_cache = None - if self.config.mulled_resolution_cache_type: - cache_opts = { - 'cache.type': self.config.mulled_resolution_cache_type, - 'cache.data_dir': self.config.mulled_resolution_cache_data_dir, - 'cache.lock_dir': self.config.mulled_resolution_cache_lock_dir, - } - mulled_resolution_cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache('mulled_resolution') - self.container_finder = containers.ContainerFinder(app_info, mulled_resolution_cache=mulled_resolution_cache) - self._set_enabled_container_types() - index_help = getattr(self.config, "index_tool_help", True) - self.toolbox_search = galaxy.tools.search.ToolBoxSearch(self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help) - - def reindex_tool_search(self): - # Call this when tools are added or removed. - self.toolbox_search.build_index(tool_cache=self.tool_cache) - self.tool_cache.reset_status() - - def _set_enabled_container_types(self): - container_types_to_destinations = collections.defaultdict(list) - for destinations in self.job_config.destinations.values(): - for destination in destinations: - for enabled_container_type in self.container_finder._enabled_container_types(destination.params): - container_types_to_destinations[enabled_container_type].append(destination) - self.toolbox.dependency_manager.set_enabled_container_types(container_types_to_destinations) - self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.default_container_registry.resolver_classes) - self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.default_container_registry.container_resolvers) - - def _configure_tool_data_tables(self, from_shed_config): - from galaxy.tools.data import ToolDataTableManager - - # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path. - self.tool_data_tables = ToolDataTableManager(tool_data_path=self.config.tool_data_path, - config_filename=self.config.tool_data_table_config_path, - other_config_dict=self.config) - # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables. - try: - self.tool_data_tables.load_from_config_file(config_filename=self.config.shed_tool_data_table_config, - tool_data_path=self.tool_data_tables.tool_data_path, - from_shed_config=from_shed_config) - except OSError as exc: - # Missing shed_tool_data_table_config is okay if it's the default - if exc.errno != errno.ENOENT or self.config.is_set('shed_tool_data_table_config'): - raise - - def _configure_datatypes_registry(self, installed_repository_manager=None): - from galaxy.datatypes import registry - # Create an empty datatypes registry. - self.datatypes_registry = registry.Registry(self.config) - if installed_repository_manager and self.config.load_tool_shed_datatypes: - # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories. We - # load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some - # generic sniffers (eg text,xml) which catch anything, so it's impossible for proprietary sniffers to be used. - # However, if there is a conflict (2 datatypes with the same extension) between a proprietary datatype and a datatype - # in the Galaxy distribution, the datatype in the Galaxy distribution will take precedence. If there is a conflict - # between 2 proprietary datatypes, the datatype from the repository that was installed earliest will take precedence. - installed_repository_manager.load_proprietary_datatypes() - # Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config. - datatypes_configs = self.config.datatypes_config - for datatypes_config in listify(datatypes_configs): - # Setting override=False would make earlier files would take - # precedence - but then they wouldn't override tool shed - # datatypes. - self.datatypes_registry.load_datatypes(self.config.root, datatypes_config, override=True) - - def _configure_object_store(self, **kwds): - from galaxy.objectstore import build_object_store_from_config - self.object_store = build_object_store_from_config(self.config, **kwds) - - def _configure_security(self): - from galaxy.security import idencoding - self.security = idencoding.IdEncodingHelper(id_secret=self.config.id_secret) - BaseDatabaseIdField.security = self.security - - def _configure_tool_shed_registry(self): - import galaxy.tool_shed.tool_shed_registry - - # Set up the tool sheds registry - if os.path.isfile(self.config.tool_sheds_config_file): - self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry(self.config.tool_sheds_config_file) - else: - self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry() - - def _configure_models(self, check_migrate_databases=False, config_file=None): - """Preconditions: object_store must be set on self.""" - db_url = get_database_url(self.config) - install_db_url = self.config.install_database_connection - # TODO: Consider more aggressive check here that this is not the same - # database file under the hood. - combined_install_database = not(install_db_url and install_db_url != db_url) - install_db_url = install_db_url or db_url - install_database_options = self.config.database_engine_options if combined_install_database else self.config.install_database_engine_options - - if self.config.database_wait: - self._wait_for_database(db_url) - - if getattr(self.config, "max_metadata_value_size", None): - from galaxy.model import custom_types - custom_types.MAX_METADATA_VALUE_SIZE = self.config.max_metadata_value_size - - if check_migrate_databases: - # Initialize database / check for appropriate schema version. # If this - # is a new installation, we'll restrict the tool migration messaging. - from galaxy.model.migrate.check import create_or_verify_database - create_or_verify_database(db_url, config_file, self.config.database_engine_options, app=self, map_install_models=combined_install_database) - if not combined_install_database: - tsi_create_or_verify_database(install_db_url, install_database_options, app=self) - - self.model = init_models_from_config( - self.config, - map_install_models=combined_install_database, - object_store=self.object_store, - trace_logger=getattr(self, "trace_logger", None) - ) - if combined_install_database: - log.info("Install database targetting Galaxy's database configuration.") - self.install_model = self.model - else: - from galaxy.model.tool_shed_install import mapping as install_mapping - install_db_url = self.config.install_database_connection - log.info(f"Install database using its own connection {install_db_url}") - self.install_model = install_mapping.init(install_db_url, - install_database_options) - - def _configure_signal_handlers(self, handlers): - for sig, handler in handlers.items(): - signal.signal(sig, handler) - - def _wait_for_database(self, url): - attempts = self.config.database_wait_attempts - pause = self.config.database_wait_sleep - for i in range(1, attempts): - try: - database_exists(url) - break - except Exception: - log.info("Waiting for database: attempt %d of %d" % (i, attempts)) - time.sleep(pause) - - @property - def tool_dependency_dir(self): - return self.toolbox.dependency_manager.default_base_path diff --git a/lib/galaxy/web/formatting.py b/lib/galaxy/web/formatting.py deleted file mode 100644 index d7b2151b566..00000000000 --- a/lib/galaxy/web/formatting.py +++ /dev/null @@ -1,32 +0,0 @@ -import locale -from string import Template - -DEFAULT_LOCALE_FORMAT = '%a %b %e %H:%M:%S %Y' -ISO_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" - - -def expand_pretty_datetime_format(value): - """ - - >>> expand_pretty_datetime_format("%H:%M:%S %Z") - '%H:%M:%S %Z' - >>> locale_format = expand_pretty_datetime_format("$locale (UTC)") - >>> import locale - >>> expected_format = '%s (UTC)' % locale.nl_langinfo(locale.D_T_FMT) - >>> locale_format == expected_format - True - >>> expand_pretty_datetime_format("$iso8601") - '%Y-%m-%d %H:%M:%S' - """ - locale_format = None - try: - locale_format = locale.nl_langinfo(locale.D_T_FMT) - except AttributeError: # nl_langinfo not available - pass - if not locale_format: - locale_format = DEFAULT_LOCALE_FORMAT - stock_formats = dict( - locale=locale_format, - iso8601=ISO_DATETIME_FORMAT, - ) - return Template(value).safe_substitute(**stock_formats) diff --git a/lib/galaxy/web_stack/__init__.py b/lib/galaxy/web_stack/__init__.py index 2310a8ff63b..66684aeef8c 100644 --- a/lib/galaxy/web_stack/__init__.py +++ b/lib/galaxy/web_stack/__init__.py @@ -633,10 +633,6 @@ def get_app_kwds(config_section, app_name=None): return application_stack_class().get_app_kwds(config_section, app_name=app_name) -def get_stack_facts(config=None): - return application_stack_instance(config=config).facts - - def _uwsgi_configured_mules(): mules = uwsgi.opt.get('mule', []) return [mules] if isinstance(mules, str) or mules is True else mules diff --git a/lib/tool_shed/webapp/config.py b/lib/tool_shed/webapp/config.py index 094186b431b..e738c86f4c5 100644 --- a/lib/tool_shed/webapp/config.py +++ b/lib/tool_shed/webapp/config.py @@ -12,11 +12,11 @@ from galaxy.config import ( CommonConfigurationMixin, get_database_engine_options, ) +from galaxy.config import expand_pretty_datetime_format from galaxy.config.schema import AppSchema from galaxy.exceptions import ConfigurationError from galaxy.util import string_as_bool from galaxy.version import VERSION, VERSION_MAJOR, VERSION_MINOR -from galaxy.web.formatting import expand_pretty_datetime_format try: from importlib.resources import files # type: ignore[attr-defined] diff --git a/test/unit/config/test_config_values.py b/test/unit/config/test_config_values.py index 6a4306be3c9..8a2aeae55f4 100644 --- a/test/unit/config/test_config_values.py +++ b/test/unit/config/test_config_values.py @@ -5,8 +5,8 @@ from datetime import timedelta import pytest from galaxy import config +from galaxy.config import expand_pretty_datetime_format from galaxy.util import listify -from galaxy.web.formatting import expand_pretty_datetime_format TestData = namedtuple('TestData', ('key', 'expected', 'loaded')) From 2fab58cac4d496cb15fac98690cfc009bf1f1641 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 28 Feb 2022 15:44:21 +0100 Subject: [PATCH 02/27] Create config package --- packages/app/MANIFEST.in | 2 - packages/app/requirements.txt | 1 + packages/app/setup.py | 6 -- packages/config/HISTORY.rst | 11 +++ packages/config/LICENSE | 1 + packages/config/MANIFEST.in | 3 + packages/config/Makefile | 1 + packages/config/README.rst | 14 +++ packages/config/dev-requirements.txt | 1 + packages/config/galaxy/__init__.py | 1 + packages/{app => config}/galaxy/config | 0 packages/config/galaxy/project_galaxy_util.py | 11 +++ packages/config/requirements.txt | 2 + packages/config/scripts | 1 + packages/config/setup.cfg | 1 + packages/config/setup.py | 99 +++++++++++++++++++ packages/config/test-data | 1 + packages/config/test-requirements.txt | 2 + packages/config/tests/__init__.py | 0 packages/config/tests/config | 1 + packages/config/tests/util | 1 + packages/test.sh | 1 + 22 files changed, 153 insertions(+), 8 deletions(-) create mode 100644 packages/config/HISTORY.rst create mode 120000 packages/config/LICENSE create mode 100644 packages/config/MANIFEST.in create mode 120000 packages/config/Makefile create mode 100644 packages/config/README.rst create mode 120000 packages/config/dev-requirements.txt create mode 100644 packages/config/galaxy/__init__.py rename packages/{app => config}/galaxy/config (100%) create mode 100644 packages/config/galaxy/project_galaxy_util.py create mode 100644 packages/config/requirements.txt create mode 120000 packages/config/scripts create mode 120000 packages/config/setup.cfg create mode 100644 packages/config/setup.py create mode 120000 packages/config/test-data create mode 100644 packages/config/test-requirements.txt create mode 100644 packages/config/tests/__init__.py create mode 120000 packages/config/tests/config create mode 120000 packages/config/tests/util diff --git a/packages/app/MANIFEST.in b/packages/app/MANIFEST.in index a4acb5c99e4..d47c145d0e9 100644 --- a/packages/app/MANIFEST.in +++ b/packages/app/MANIFEST.in @@ -1,5 +1,3 @@ include *.rst *.txt LICENSE -include galaxy/config/schemas/*.yml -include galaxy/config/sample/*.sample* include galaxy/jobs/runners/util/job_script/*.sh include galaxy/tools/*tsv diff --git a/packages/app/requirements.txt b/packages/app/requirements.txt index 4d77125b4d6..c0561b95e6e 100644 --- a/packages/app/requirements.txt +++ b/packages/app/requirements.txt @@ -1,4 +1,5 @@ galaxy-auth +galaxy-config galaxy-data galaxy-job-execution galaxy-job-metrics diff --git a/packages/app/setup.py b/packages/app/setup.py index b07f8886ee7..f92bc4fe976 100644 --- a/packages/app/setup.py +++ b/packages/app/setup.py @@ -34,7 +34,6 @@ PACKAGES = [ 'galaxy.actions', 'galaxy.app_unittest_utils', 'galaxy.authnz', - 'galaxy.config', 'galaxy.dependencies', 'galaxy.forms', 'galaxy.jobs', @@ -95,14 +94,9 @@ PACKAGES = [ ENTRY_POINTS = ''' [console_scripts] galaxy-main=galaxy.main:main - galaxy-config=galaxy.config.script:main ''' PACKAGE_DATA = { # Be sure to update MANIFEST.in for source dist. - 'galaxy': [ - 'config/schemas/*.yml', - 'config/sample/*', - ], 'tool_shed': [ 'scripts/bootstrap_tool_shed/user_info.xml', ], diff --git a/packages/config/HISTORY.rst b/packages/config/HISTORY.rst new file mode 100644 index 00000000000..e5abec134b3 --- /dev/null +++ b/packages/config/HISTORY.rst @@ -0,0 +1,11 @@ +.. :changelog: + +History +------- + +.. to_doc + +--------------------- +22.1.0.dev0 +--------------------- +Initial release diff --git a/packages/config/LICENSE b/packages/config/LICENSE new file mode 120000 index 00000000000..fb4eead94ae --- /dev/null +++ b/packages/config/LICENSE @@ -0,0 +1 @@ +../MIT_LICENSE.txt \ No newline at end of file diff --git a/packages/config/MANIFEST.in b/packages/config/MANIFEST.in new file mode 100644 index 00000000000..ef1f736e7c8 --- /dev/null +++ b/packages/config/MANIFEST.in @@ -0,0 +1,3 @@ +include *.rst *.txt LICENSE +include galaxy/config/schemas/*.yml +include galaxy/config/sample/*.sample* diff --git a/packages/config/Makefile b/packages/config/Makefile new file mode 120000 index 00000000000..37af8bae5ba --- /dev/null +++ b/packages/config/Makefile @@ -0,0 +1 @@ +../package.Makefile \ No newline at end of file diff --git a/packages/config/README.rst b/packages/config/README.rst new file mode 100644 index 00000000000..fde16673c89 --- /dev/null +++ b/packages/config/README.rst @@ -0,0 +1,14 @@ + +.. image:: https://badge.fury.io/py/galaxy-config.svg + :target: https://pypi.org/project/galaxy-config/ + + +Overview +-------- + +The Galaxy_ config module. + +* Free software: Academic Free License version 3.0 +* Code: https://github.com/galaxyproject/galaxy + +.. _Galaxy: http://galaxyproject.org/ diff --git a/packages/config/dev-requirements.txt b/packages/config/dev-requirements.txt new file mode 120000 index 00000000000..467b90d7a23 --- /dev/null +++ b/packages/config/dev-requirements.txt @@ -0,0 +1 @@ +../package-dev-requirements.txt \ No newline at end of file diff --git a/packages/config/galaxy/__init__.py b/packages/config/galaxy/__init__.py new file mode 100644 index 00000000000..8e13dacd253 --- /dev/null +++ b/packages/config/galaxy/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore[has-type] diff --git a/packages/app/galaxy/config b/packages/config/galaxy/config similarity index 100% rename from packages/app/galaxy/config rename to packages/config/galaxy/config diff --git a/packages/config/galaxy/project_galaxy_util.py b/packages/config/galaxy/project_galaxy_util.py new file mode 100644 index 00000000000..48b69d86178 --- /dev/null +++ b/packages/config/galaxy/project_galaxy_util.py @@ -0,0 +1,11 @@ +__version__ = "22.1.0rc1" + +PROJECT_NAME = "galaxy-config" +PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" +PROJECT_URL = "https://github.com/galaxyproject/galaxy" +PROJECT_AUTHOR = 'Galaxy Project and Community' +PROJECT_DESCRIPTION = 'Galaxy Generic Utilities' +PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( + PROJECT_USERAME, PROJECT_NAME +) diff --git a/packages/config/requirements.txt b/packages/config/requirements.txt new file mode 100644 index 00000000000..e663ba0d20f --- /dev/null +++ b/packages/config/requirements.txt @@ -0,0 +1,2 @@ +galaxy-containers +pykwalify diff --git a/packages/config/scripts b/packages/config/scripts new file mode 120000 index 00000000000..9aec9dc5a06 --- /dev/null +++ b/packages/config/scripts @@ -0,0 +1 @@ +../build_scripts \ No newline at end of file diff --git a/packages/config/setup.cfg b/packages/config/setup.cfg new file mode 120000 index 00000000000..eb7cf09393f --- /dev/null +++ b/packages/config/setup.cfg @@ -0,0 +1 @@ +../../setup.cfg \ No newline at end of file diff --git a/packages/config/setup.py b/packages/config/setup.py new file mode 100644 index 00000000000..068ac97847e --- /dev/null +++ b/packages/config/setup.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +import ast +import os +import re +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +SOURCE_DIR = "galaxy" + +_version_re = re.compile(r'__version__\s+=\s+(.*)') + +with open('%s/project_galaxy_util.py' % SOURCE_DIR, 'rb') as f: + init_contents = f.read().decode('utf-8') + + def get_var(var_name): + pattern = re.compile(r'%s\s+=\s+(.*)' % var_name) + match = pattern.search(init_contents).group(1) + return str(ast.literal_eval(match)) + + version = get_var("__version__") + PROJECT_NAME = get_var("PROJECT_NAME") + PROJECT_URL = get_var("PROJECT_URL") + PROJECT_AUTHOR = get_var("PROJECT_AUTHOR") + PROJECT_EMAIL = get_var("PROJECT_EMAIL") + PROJECT_DESCRIPTION = get_var("PROJECT_DESCRIPTION") + +TEST_DIR = 'tests' +PACKAGES = [ + 'galaxy', + 'galaxy.config', +] +ENTRY_POINTS = ''' + [console_scripts] + galaxy-config=galaxy.config.script:main +''' +PACKAGE_DATA = { + # Be sure to update MANIFEST.in for source dist. + 'galaxy': [ + 'config/schemas/*.yml', + 'config/sample/*', + ], +} +PACKAGE_DIR = { + SOURCE_DIR: SOURCE_DIR, +} + +readme = open('README.rst').read() +history = open('HISTORY.rst').read().replace('.. :changelog:', '') + +if os.path.exists("requirements.txt"): + requirements = open("requirements.txt").read().split("\n") +else: + # In tox, it will cover them anyway. + requirements = [] + + +test_requirements = open("test-requirements.txt").read().split("\n") + + +setup( + name=PROJECT_NAME, + version=version, + description=PROJECT_DESCRIPTION, + long_description=readme + '\n\n' + history, + long_description_content_type='text/x-rst', + author=PROJECT_AUTHOR, + author_email=PROJECT_EMAIL, + url=PROJECT_URL, + packages=PACKAGES, + entry_points=ENTRY_POINTS, + package_data=PACKAGE_DATA, + package_dir=PACKAGE_DIR, + include_package_data=True, + install_requires=requirements, + license="MIT", + zip_safe=False, + keywords='galaxy', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Environment :: Console', + 'License :: OSI Approved :: MIT License', + 'Operating System :: POSIX', + 'Topic :: Software Development', + 'Topic :: Software Development :: Code Generators', + 'Topic :: Software Development :: Testing', + 'Natural Language :: English', + "Programming Language :: Python :: 3", + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + ], + test_suite=TEST_DIR, + tests_require=test_requirements +) diff --git a/packages/config/test-data b/packages/config/test-data new file mode 120000 index 00000000000..e50575ea576 --- /dev/null +++ b/packages/config/test-data @@ -0,0 +1 @@ +../../test-data \ No newline at end of file diff --git a/packages/config/test-requirements.txt b/packages/config/test-requirements.txt new file mode 100644 index 00000000000..ca5a0dac0ee --- /dev/null +++ b/packages/config/test-requirements.txt @@ -0,0 +1,2 @@ +pytest<6.1 +responses diff --git a/packages/config/tests/__init__.py b/packages/config/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/config/tests/config b/packages/config/tests/config new file mode 120000 index 00000000000..329fc647500 --- /dev/null +++ b/packages/config/tests/config @@ -0,0 +1 @@ +../../../test/unit/config \ No newline at end of file diff --git a/packages/config/tests/util b/packages/config/tests/util new file mode 120000 index 00000000000..d42557d85de --- /dev/null +++ b/packages/config/tests/util @@ -0,0 +1 @@ +../../../test/unit/util \ No newline at end of file diff --git a/packages/test.sh b/packages/test.sh index f323203da1b..2a9e022611b 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -23,6 +23,7 @@ PACKAGE_DIRS=( objectstore job_metrics containers + config files tool_util data From 98eb857040b87b35ca67e15f61eb5d091de4cbfb Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 11:46:13 +0100 Subject: [PATCH 03/27] Move unit tests that requre TS schema --- .../config_manage/1607_root_filters/config/galaxy.ini | 0 .../config_manage/1607_root_filters/config/tool_shed.ini | 0 .../config_manage/1607_root_samples/config/galaxy.ini | 0 .../config_manage/1607_root_samples/config/reports.ini | 0 .../config_manage/1607_root_samples/config/tool_shed.ini | 0 .../config_manage/1705_allow_path_paste/config/galaxy.ini | 0 test/unit/{config => webapps}/config_manage/__init__.py | 0 .../{config => webapps}/config_manage/embedded/config/galaxy.yml | 0 .../{config => webapps}/config_manage/simple/config/galaxy.yml | 0 test/unit/{config => webapps}/config_manage/test_config_manage.py | 0 test/unit/{config => webapps}/test_config_values.py | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename test/unit/{config => webapps}/config_manage/1607_root_filters/config/galaxy.ini (100%) rename test/unit/{config => webapps}/config_manage/1607_root_filters/config/tool_shed.ini (100%) rename test/unit/{config => webapps}/config_manage/1607_root_samples/config/galaxy.ini (100%) rename test/unit/{config => webapps}/config_manage/1607_root_samples/config/reports.ini (100%) rename test/unit/{config => webapps}/config_manage/1607_root_samples/config/tool_shed.ini (100%) rename test/unit/{config => webapps}/config_manage/1705_allow_path_paste/config/galaxy.ini (100%) rename test/unit/{config => webapps}/config_manage/__init__.py (100%) rename test/unit/{config => webapps}/config_manage/embedded/config/galaxy.yml (100%) rename test/unit/{config => webapps}/config_manage/simple/config/galaxy.yml (100%) rename test/unit/{config => webapps}/config_manage/test_config_manage.py (100%) rename test/unit/{config => webapps}/test_config_values.py (100%) diff --git a/test/unit/config/config_manage/1607_root_filters/config/galaxy.ini b/test/unit/webapps/config_manage/1607_root_filters/config/galaxy.ini similarity index 100% rename from test/unit/config/config_manage/1607_root_filters/config/galaxy.ini rename to test/unit/webapps/config_manage/1607_root_filters/config/galaxy.ini diff --git a/test/unit/config/config_manage/1607_root_filters/config/tool_shed.ini b/test/unit/webapps/config_manage/1607_root_filters/config/tool_shed.ini similarity index 100% rename from test/unit/config/config_manage/1607_root_filters/config/tool_shed.ini rename to test/unit/webapps/config_manage/1607_root_filters/config/tool_shed.ini diff --git a/test/unit/config/config_manage/1607_root_samples/config/galaxy.ini b/test/unit/webapps/config_manage/1607_root_samples/config/galaxy.ini similarity index 100% rename from test/unit/config/config_manage/1607_root_samples/config/galaxy.ini rename to test/unit/webapps/config_manage/1607_root_samples/config/galaxy.ini diff --git a/test/unit/config/config_manage/1607_root_samples/config/reports.ini b/test/unit/webapps/config_manage/1607_root_samples/config/reports.ini similarity index 100% rename from test/unit/config/config_manage/1607_root_samples/config/reports.ini rename to test/unit/webapps/config_manage/1607_root_samples/config/reports.ini diff --git a/test/unit/config/config_manage/1607_root_samples/config/tool_shed.ini b/test/unit/webapps/config_manage/1607_root_samples/config/tool_shed.ini similarity index 100% rename from test/unit/config/config_manage/1607_root_samples/config/tool_shed.ini rename to test/unit/webapps/config_manage/1607_root_samples/config/tool_shed.ini diff --git a/test/unit/config/config_manage/1705_allow_path_paste/config/galaxy.ini b/test/unit/webapps/config_manage/1705_allow_path_paste/config/galaxy.ini similarity index 100% rename from test/unit/config/config_manage/1705_allow_path_paste/config/galaxy.ini rename to test/unit/webapps/config_manage/1705_allow_path_paste/config/galaxy.ini diff --git a/test/unit/config/config_manage/__init__.py b/test/unit/webapps/config_manage/__init__.py similarity index 100% rename from test/unit/config/config_manage/__init__.py rename to test/unit/webapps/config_manage/__init__.py diff --git a/test/unit/config/config_manage/embedded/config/galaxy.yml b/test/unit/webapps/config_manage/embedded/config/galaxy.yml similarity index 100% rename from test/unit/config/config_manage/embedded/config/galaxy.yml rename to test/unit/webapps/config_manage/embedded/config/galaxy.yml diff --git a/test/unit/config/config_manage/simple/config/galaxy.yml b/test/unit/webapps/config_manage/simple/config/galaxy.yml similarity index 100% rename from test/unit/config/config_manage/simple/config/galaxy.yml rename to test/unit/webapps/config_manage/simple/config/galaxy.yml diff --git a/test/unit/config/config_manage/test_config_manage.py b/test/unit/webapps/config_manage/test_config_manage.py similarity index 100% rename from test/unit/config/config_manage/test_config_manage.py rename to test/unit/webapps/config_manage/test_config_manage.py diff --git a/test/unit/config/test_config_values.py b/test/unit/webapps/test_config_values.py similarity index 100% rename from test/unit/config/test_config_values.py rename to test/unit/webapps/test_config_values.py From e252b7371696b3fd03bfecc229483b067b3af021 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 12:00:09 +0100 Subject: [PATCH 04/27] Fix up tests for non-UTC clocks --- .../data/model/mapping/test_model_mapping.py | 141 +++++++++--------- 1 file changed, 71 insertions(+), 70 deletions(-) diff --git a/test/unit/data/model/mapping/test_model_mapping.py b/test/unit/data/model/mapping/test_model_mapping.py index bbe66d80d1c..2ffb44e040b 100644 --- a/test/unit/data/model/mapping/test_model_mapping.py +++ b/test/unit/data/model/mapping/test_model_mapping.py @@ -60,13 +60,14 @@ class TestPlanet(BaseTest): # BaseTest is a base class; we need it to get the t See other model tests in this module for examples of more complex setups. """ -from datetime import datetime, timedelta +from datetime import timedelta from uuid import UUID, uuid4 import pytest from sqlalchemy import func, select from galaxy import model +from galaxy.model.orm.now import now from .common import ( AbstractBaseTest, collection_consists_of_objects, @@ -92,7 +93,7 @@ class TestAPIKeys(BaseTest): assert cls_.__tablename__ == "api_keys" def test_columns(self, session, cls_, user): - create_time, user_id, key = datetime.now(), user.id, get_unique_value() + create_time, user_id, key = now(), user.id, get_unique_value() obj = cls_(user_id=user_id, key=key, create_time=create_time) with dbcleanup(session, obj) as obj_id: @@ -115,7 +116,7 @@ class TestCleanupEvent(BaseTest): assert cls_.__tablename__ == "cleanup_event" def test_columns(self, session, cls_): - create_time, message = datetime.now(), "a" + create_time, message = now(), "a" obj = cls_(create_time=create_time, message=message) with dbcleanup(session, obj) as obj_id: @@ -130,7 +131,7 @@ class TestCleanupEventDatasetAssociation(BaseTest): assert cls_.__tablename__ == "cleanup_event_dataset_association" def test_columns(self, session, cls_, cleanup_event, dataset): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -150,7 +151,7 @@ class TestCleanupEventHistoryAssociation(BaseTest): assert cls_.__tablename__ == "cleanup_event_history_association" def test_columns(self, session, cls_, cleanup_event, history): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -170,7 +171,7 @@ class TestCleanupEventHistoryDatasetAssociationAssociation(BaseTest): assert cls_.__tablename__ == "cleanup_event_hda_association" def test_columns(self, session, cls_, cleanup_event, history_dataset_association): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -192,7 +193,7 @@ class TestCleanupEventImplicitlyConvertedDatasetAssociationAssociation(BaseTest) def test_columns( self, session, cls_, cleanup_event, implicitly_converted_dataset_association ): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -212,7 +213,7 @@ class TestCleanupEventLibraryAssociation(BaseTest): assert cls_.__tablename__ == "cleanup_event_library_association" def test_columns(self, session, cls_, cleanup_event, library): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -232,7 +233,7 @@ class TestCleanupEventLibraryDatasetAssociation(BaseTest): assert cls_.__tablename__ == "cleanup_event_library_dataset_association" def test_columns(self, session, cls_, cleanup_event, library_dataset): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -254,7 +255,7 @@ class TestCleanupEventLibraryDatasetDatasetAssociationAssociation(BaseTest): def test_columns( self, session, cls_, cleanup_event, library_dataset_dataset_association ): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -274,7 +275,7 @@ class TestCleanupEventLibraryFolderAssociation(BaseTest): assert cls_.__tablename__ == "cleanup_event_library_folder_association" def test_columns(self, session, cls_, cleanup_event, library_folder): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -294,7 +295,7 @@ class TestCleanupEventMetadataFileAssociation(BaseTest): assert cls_.__tablename__ == "cleanup_event_metadata_file_association" def test_columns(self, session, cls_, cleanup_event, metadata_file): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -322,7 +323,7 @@ class TestCustosAuthnzToken(BaseTest): access_token = "c" id_token = "d" refresh_token = "e" - expiration_time = datetime.now() + expiration_time = now() refresh_expiration_time = expiration_time + timedelta(hours=1) obj = cls_() @@ -391,7 +392,7 @@ class TestDataManagerHistoryAssociation(BaseTest): assert cls_.__tablename__ == "data_manager_history_association" def test_columns(self, session, cls_, history, user): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.create_time = create_time @@ -424,7 +425,7 @@ class TestDataManagerJobAssociation(BaseTest): assert has_index(cls_.__table__, ("data_manager_id",)) def test_columns(self, session, cls_, job): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) data_manager_id = "a" obj = cls_() @@ -457,7 +458,7 @@ class TestDataset(BaseTest): assert cls_.table.name == "dataset" def test_columns(self, session, cls_, job): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) state = "a" deleted = True @@ -561,7 +562,7 @@ class TestDatasetCollection(BaseTest): populated_state = "b" populated_state_message = "c" element_count = 1 - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() @@ -704,7 +705,7 @@ class TestDatasetPermissions(BaseTest): def test_columns(self, session, cls_, dataset, role): action = "a" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(action, dataset, role) obj.create_time = create_time @@ -817,7 +818,7 @@ class TestDefaultQuotaAssociation(BaseTest): def test_columns(self, session, cls_, quota): type_ = cls_.types.REGISTERED - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(type_, quota) obj.create_time = create_time @@ -877,7 +878,7 @@ class TestDynamicTool(BaseTest): active = True hidden = True value = "f" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() @@ -930,7 +931,7 @@ class TestEvent(BaseTest): def test_columns(self, session, cls_, history, galaxy_session, user): message, tool_id = "a", "b" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.message = message @@ -1018,7 +1019,7 @@ class TestFormDefinition(BaseTest): def test_columns(self, session, cls_, form_definition_current): name, desc, fields, type, layout = "a", "b", "c", "d", "e" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.name = name @@ -1055,7 +1056,7 @@ class TestFormDefinitionCurrent(BaseTest): assert cls_.__tablename__ == "form_definition_current" def test_columns(self, session, cls_, form_definition): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) deleted = True obj = cls_() @@ -1088,7 +1089,7 @@ class TestFormValues(BaseTest): def test_columns(self, session, cls_, form_definition): content = "a" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.form_definition = form_definition @@ -1118,7 +1119,7 @@ class TestGalaxySession(BaseTest): def test_columns(self, session, cls_, user, history, galaxy_session): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) remote_host = "a" remote_addr = "b" @@ -1179,7 +1180,7 @@ class TestGalaxySessionToHistoryAssociation(BaseTest): assert cls_.__tablename__ == "galaxy_session_to_history" def test_columns(self, session, cls_, galaxy_session, history): - create_time = datetime.now() + create_time = now() obj = cls_(galaxy_session, history) obj.create_time = create_time @@ -1205,7 +1206,7 @@ class TestGenomeIndexToolData(BaseTest): def test_columns(self, session, cls_, job, dataset, user): fasta_path = "a" - created_time = datetime.now() + created_time = now() modified_time = created_time + timedelta(hours=1) indexer = "b" obj = cls_() @@ -1246,7 +1247,7 @@ class TestGroup(BaseTest): assert cls_.__tablename__ == "galaxy_group" def test_columns(self, session, cls_): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) name = get_unique_value() deleted = True @@ -1297,7 +1298,7 @@ class TestGroupQuotaAssociation(BaseTest): assert cls_.__tablename__ == "group_quota_association" def test_columns(self, session, cls_, group, quota): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(group, quota) obj.create_time = create_time @@ -1326,7 +1327,7 @@ class TestGroupRoleAssociation(BaseTest): def test_columns(self, session, cls_, group, role): obj = cls_(group, role) - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj.create_time = create_time obj.update_time = update_time @@ -1354,7 +1355,7 @@ class TestHistory(BaseTest): assert has_index(cls_.__table__, ("slug",)) def test_columns(self, session, cls_, user): - create_time = datetime.now() + create_time = now() name = "a" hid_counter = 2 deleted = True @@ -1529,7 +1530,7 @@ class TestHistoryDatasetAssociation(BaseTest): copied_from_hda = history_dataset_association_factory() parent = history_dataset_association_factory() - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) state = "a" name = "b" @@ -1760,7 +1761,7 @@ class TestHistoryDatasetAssociationDisplayAtAuthorization(BaseTest): ) def test_columns(self, session, cls_, history_dataset_association, user): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) site = "a" obj = cls_(history_dataset_association, user, site) @@ -1798,7 +1799,7 @@ class TestHistoryDatasetAssociationHistory(BaseTest): ): name, update_time, version, extension, metadata = ( "a", - datetime.now(), + now(), 2, "b", {"key": "value"}, @@ -1965,7 +1966,7 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): visible = True deleted = True implicit_output_name = "b" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() @@ -2339,7 +2340,7 @@ class TestImplicitlyConvertedDatasetAssociation(BaseTest): history_dataset_association, library_dataset_dataset_association, ): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) deleted = True metadata_safe = False @@ -2453,7 +2454,7 @@ class TestInteractiveToolEntryPoint(BaseTest): info = "f" configured = True deleted = True - created_time = datetime.now() + created_time = now() modified_time = created_time + timedelta(hours=1) obj = cls_() @@ -2513,7 +2514,7 @@ class TestJob(BaseTest): galaxy_session, user, ): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) tool_id = "a" tool_version = "b" @@ -2756,7 +2757,7 @@ class TestJobContainerAssociation(BaseTest): container_type = "a" container_name = "b" container_info = "c" - created_time = datetime.now() + created_time = now() modified_time = created_time + timedelta(hours=1) session.add(job) # must be bound to a session for lazy load of attributes @@ -2988,7 +2989,7 @@ class TestJobStateHistory(BaseTest): def test_columns(self, session, cls_, job): state, info = job.state, job.info - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(job) obj.create_time = create_time @@ -3246,7 +3247,7 @@ class TestLibrary(BaseTest): def test_columns(self, session, cls_, library_folder): name, deleted, purged, description, synopsis = "a", True, True, "b", "c" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(name, description, synopsis, library_folder) obj.create_time = create_time @@ -3290,7 +3291,7 @@ class TestLibraryDataset(BaseTest): def test_columns(self, session, cls_, library_folder): order_id = 9 - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) name = "a" info = "b" @@ -3533,7 +3534,7 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): extended_metadata, user, ): - create_time = datetime.now() + create_time = now() state = "a" name = "b" info = "c" @@ -3715,7 +3716,7 @@ class TestLibraryDatasetDatasetAssociationPermissions(BaseTest): def test_columns(self, session, cls_, library_dataset_dataset_association, role): action = "a" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(action, library_dataset_dataset_association, role) obj.create_time = create_time @@ -3843,7 +3844,7 @@ class TestLibraryDatasetPermissions(BaseTest): def test_columns(self, session, cls_, library_dataset, role): action = "a" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(action, library_dataset, role) obj.create_time = create_time @@ -3874,7 +3875,7 @@ class TestLibraryFolder(BaseTest): def test_columns(self, session, cls_, library_folder): parent = library_folder - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) name = "a" description = "b" @@ -3984,7 +3985,7 @@ class TestLibraryFolderPermissions(BaseTest): def test_columns(self, session, cls_, library_folder, role): action = "a" obj = cls_(action, library_folder, role) - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj.create_time = create_time obj.update_time = update_time @@ -4041,7 +4042,7 @@ class TestLibraryPermissions(BaseTest): def test_columns(self, session, cls_, library, role): action = "a" obj = cls_(action, library, role) - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(action, library, role) obj.create_time = create_time @@ -4077,7 +4078,7 @@ class TestMetadataFile(BaseTest): library_dataset_dataset_association, ): name = "a" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) object_store_id = "b" uuid = uuid4() @@ -4215,7 +4216,7 @@ class TestPage(BaseTest): def test_columns(self, session, cls_, user, page_revision): title, deleted, importable, slug, published = "a", True, True, "b", True - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.user = user @@ -4345,7 +4346,7 @@ class TestPageRevision(BaseTest): def test_columns(self, session, cls_, page): title, content = "a", "b" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.page = page @@ -4438,7 +4439,7 @@ class TestPasswordResetToken(BaseTest): def test_columns_and_relationships(self, session, cls_, user): token = get_unique_value() - expiration_time = datetime.now() + expiration_time = now() obj = cls_(user, token) obj.expiration_time = expiration_time @@ -4508,7 +4509,7 @@ class TestQuota(BaseTest): def test_columns(self, session, cls_): name, description, amount, operation = get_unique_value(), "b", 42, "+" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(name, description, amount, operation) obj.create_time = create_time @@ -4567,7 +4568,7 @@ class TestRole(BaseTest): cls_.types.SYSTEM, True, ) - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(name, description, type_, deleted) obj.create_time = create_time @@ -4616,7 +4617,7 @@ class TestStoredWorkflow(BaseTest): assert has_index(cls_.__table__, ("slug",)) def test_columns(self, session, cls_, workflow, user): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) name = "a" deleted = True @@ -4905,7 +4906,7 @@ class TestTask(BaseTest): assert cls_.__tablename__ == "task" def test_columns(self, session, cls_, job): - create_time = datetime.now() + create_time = now() execution_time = create_time + timedelta(hours=1) update_time = execution_time + timedelta(hours=1) state = "p" @@ -5051,7 +5052,7 @@ class TestUser(BaseTest): assert cls_.__tablename__ == "galaxy_user" def test_columns(self, session, cls_, form_values): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) email = get_unique_value() username = get_unique_value() @@ -5225,7 +5226,7 @@ class TestUserAction(BaseTest): def test_columns(self, session, cls_, user, galaxy_session): action, params, context = "a", "b", "c" - create_time = datetime.now() + create_time = now() obj = cls_() obj.user = user obj.session_id = galaxy_session.id @@ -5271,7 +5272,7 @@ class TestUserAddress(BaseTest): deleted, purged, ) = ("a", "b", "c", "d", "e", "f", "g", "h", "i", True, False) - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.user = user @@ -5349,7 +5350,7 @@ class TestUserGroupAssociation(BaseTest): assert cls_.__tablename__ == "user_group_association" def test_columns(self, session, cls_, user, group): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(user, group) obj.create_time = create_time @@ -5396,7 +5397,7 @@ class TestUserQuotaAssociation(BaseTest): assert cls_.__tablename__ == "user_quota_association" def test_columns(self, session, cls_, user, quota): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(user, quota) obj.create_time = create_time @@ -5424,7 +5425,7 @@ class TestUserRoleAssociation(BaseTest): assert cls_.__tablename__ == "user_role_association" def test_columns(self, session, cls_, user, role): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(user, role) obj.create_time = create_time @@ -5452,7 +5453,7 @@ class TestVault(BaseTest): assert cls_.__tablename__ == "vault" def test_columns(self, session, cls_): - create_time = update_time = datetime.now() + create_time = update_time = now() key = '/some/path' parent_key = '/some' value = 'helloworld' @@ -5474,7 +5475,7 @@ class TestVisualization(BaseTest): assert has_index(cls_.__table__, ("slug",)) def test_columns(self, session, cls_, user, visualization_revision): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) title = "a" type = "b" @@ -5627,7 +5628,7 @@ class TestVisualizationRevision(BaseTest): def test_columns(self, session, cls_, visualization): visualization, title, dbkey, config = visualization, "a", "b", "c" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.visualization = visualization @@ -5720,7 +5721,7 @@ class TestWorkerProcess(BaseTest): def test_columns(self, session, cls_): server_name, hostname = get_unique_value(), "a" - update_time = datetime.now() + update_time = now() obj = cls_() obj.server_name = server_name obj.hostname = hostname @@ -5740,7 +5741,7 @@ class TestWorkflow(BaseTest): assert cls_.__tablename__ == "workflow" def test_columns(self, session, cls_, stored_workflow, workflow): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) name = "a" has_cycles = True @@ -5805,7 +5806,7 @@ class TestWorkflowInvocation(BaseTest): assert cls_.__tablename__ == "workflow_invocation" def test_columns(self, session, cls_, workflow, history): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) state = "a" scheduler = "b" @@ -6095,7 +6096,7 @@ class TestWorkflowInvocationStep(BaseTest): job, implicit_collection_jobs, ): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) state, action = "a", "b" @@ -6581,7 +6582,7 @@ class TestWorkflowStep(BaseTest): assert cls_.__tablename__ == "workflow_step" def test_columns(self, session, cls_, workflow, dynamic_tool, workflow_factory): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) type = "a" tool_id = "b" From 56b4ee844afeba3b8806bcb66a175ad389fc6e05 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 12:24:35 +0100 Subject: [PATCH 05/27] Fix mocking of GalaxyAppConfiguration --- test/unit/webapps/test_config_values.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit/webapps/test_config_values.py b/test/unit/webapps/test_config_values.py index 8a2aeae55f4..96f2c065603 100644 --- a/test/unit/webapps/test_config_values.py +++ b/test/unit/webapps/test_config_values.py @@ -1,6 +1,7 @@ import os from collections import namedtuple from datetime import timedelta +from unittest.mock import patch import pytest @@ -225,8 +226,8 @@ class ExpectedValues: return 'uuid' +@patch('galaxy.config.GalaxyAppConfiguration._override_tempdir', lambda a, b: None) def get_config_data(): - config.GalaxyAppConfiguration._override_tempdir = lambda a, b: None # method must be mocked configuration = config.GalaxyAppConfiguration() ev = ExpectedValues(configuration) items = ((k, v) for k, v in configuration.schema.app_schema.items()) From 6b2045203ef11b5d138a5cec120fed1654afa339 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 12:32:30 +0100 Subject: [PATCH 06/27] Package version.py --- packages/util/galaxy/version.py | 1 + 1 file changed, 1 insertion(+) create mode 120000 packages/util/galaxy/version.py diff --git a/packages/util/galaxy/version.py b/packages/util/galaxy/version.py new file mode 120000 index 00000000000..42d972d9994 --- /dev/null +++ b/packages/util/galaxy/version.py @@ -0,0 +1 @@ +../../../lib/galaxy/version.py \ No newline at end of file From 699e7c31566d68fb0b840019b1e8014c25c8df59 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 13:26:32 +0100 Subject: [PATCH 07/27] Drop _override_tempdir patch and just set override_tempdir to False. --- .../config/test_config_dataset_storage.py | 20 +++++-------------- test/unit/webapps/test_config_values.py | 4 +--- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/test/unit/config/test_config_dataset_storage.py b/test/unit/config/test_config_dataset_storage.py index 8c455c83126..3b66482f11e 100644 --- a/test/unit/config/test_config_dataset_storage.py +++ b/test/unit/config/test_config_dataset_storage.py @@ -1,26 +1,16 @@ import os.path -import pytest - from galaxy import config from galaxy.config import BaseAppConfiguration -@pytest.fixture -def mock_config_file(monkeypatch): - # Patch this; otherwise tempfile.tempdir will be set, which is a global variable that - # defines the value of the default `dir` argument to the functions in Python's - # tempfile module - which breaks multiple tests. - monkeypatch.setattr(config.GalaxyAppConfiguration, '_override_tempdir', lambda a, b: None) - - -def test_object_store_store_by_set(mock_config_file, monkeypatch): +def test_object_store_store_by_set(): # object_store_store_by set by admin appconfig = config.GalaxyAppConfiguration(object_store_store_by='id') assert appconfig.object_store_store_by == 'id' -def test_uuid_1(mock_config_file, monkeypatch): +def test_uuid_1(): # object_store_store_by not set # file_path set by admin to `objects` (no need for the dir to exist) appconfig = config.GalaxyAppConfiguration(file_path='objects') @@ -28,7 +18,7 @@ def test_uuid_1(mock_config_file, monkeypatch): assert appconfig.object_store_store_by == 'uuid' -def test_uuid_2(mock_config_file, monkeypatch): +def test_uuid_2(monkeypatch): # object_store_store_by not set # file_path not set, `files` dir doesn't exist monkeypatch.setattr(BaseAppConfiguration, '_path_exists', lambda self, path: False) @@ -37,7 +27,7 @@ def test_uuid_2(mock_config_file, monkeypatch): assert appconfig.object_store_store_by == 'uuid' -def test_id_1(mock_config_file, monkeypatch): +def test_id_1(): # object_store_store_by not set # file_path set by admin to `not_objects` (no need for the dir to exist) appconfig = config.GalaxyAppConfiguration(file_path='not_objects') @@ -45,7 +35,7 @@ def test_id_1(mock_config_file, monkeypatch): assert appconfig.object_store_store_by == 'id' -def test_id_2(mock_config_file, monkeypatch): +def test_id_2(monkeypatch): # object_store_store_by not set # file_path not set, `files` dir exists monkeypatch.setattr(BaseAppConfiguration, '_path_exists', lambda self, path: True if os.path.basename(path) == 'files' else False) diff --git a/test/unit/webapps/test_config_values.py b/test/unit/webapps/test_config_values.py index 96f2c065603..bd2e1fe0631 100644 --- a/test/unit/webapps/test_config_values.py +++ b/test/unit/webapps/test_config_values.py @@ -1,7 +1,6 @@ import os from collections import namedtuple from datetime import timedelta -from unittest.mock import patch import pytest @@ -226,9 +225,8 @@ class ExpectedValues: return 'uuid' -@patch('galaxy.config.GalaxyAppConfiguration._override_tempdir', lambda a, b: None) def get_config_data(): - configuration = config.GalaxyAppConfiguration() + configuration = config.GalaxyAppConfiguration(override_tempdir=False) ev = ExpectedValues(configuration) items = ((k, v) for k, v in configuration.schema.app_schema.items()) for key, data in items: From 0a2025954f62f00a5438e52a13b0cc94bb62fd7f Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Tue, 1 Mar 2022 15:01:45 +0100 Subject: [PATCH 08/27] Drop local import We're already importing at the top Co-authored-by: Nicola Soranzo --- lib/galaxy/app.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index b2b9090385c..8319669bf48 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -208,8 +208,6 @@ class ConfiguresGalaxyMixin: self.citations_manager = CitationsManager(self) self.biotools_metadata_source = get_galaxy_biotools_metadata_source(self.config) - from galaxy.managers.tools import DynamicToolManager - self.dynamic_tools_manager = DynamicToolManager(self) self._toolbox_lock = threading.RLock() self.toolbox = tools.ToolBox(self.config.tool_configs, self.config.tool_path, self) From 3acb7312148d812d2f43c9c11cd0b6d46f94aac4 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Tue, 1 Mar 2022 15:02:24 +0100 Subject: [PATCH 09/27] Use facts directly Co-authored-by: Nicola Soranzo --- lib/galaxy/config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 50806e4177d..96c2a63023e 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -148,7 +148,7 @@ def configure_logging(config, facts=None): # configure logging with logging dict in config, template *FileHandler handler filenames with the `filename_template` option for name, conf in logging_conf.get('handlers', {}).items(): if conf['class'].startswith('logging.') and conf['class'].endswith('FileHandler') and 'filename_template' in conf: - conf['filename'] = conf.pop('filename_template').format(**get_facts(config=config)) + conf['filename'] = conf.pop('filename_template').format(**facts) logging_conf['handlers'][name] = conf logging.config.dictConfig(logging_conf) From 9ec7e878bca9067a589282a6582cab6482ed7516 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 15:10:12 +0100 Subject: [PATCH 10/27] Debug unit tests --- lib/galaxy/config/__init__.py | 1 + test/unit/webapps/test_config_values.py | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 96c2a63023e..8a8d0fc9f5a 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -707,6 +707,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): def _override_tempdir(self, kwargs): if string_as_bool(kwargs.get("override_tempdir", "True")): + raise Exception("How did we get here ?") tempfile.tempdir = self.new_file_path def config_value_for_host(self, config_option, host): diff --git a/test/unit/webapps/test_config_values.py b/test/unit/webapps/test_config_values.py index bd2e1fe0631..24f922cee46 100644 --- a/test/unit/webapps/test_config_values.py +++ b/test/unit/webapps/test_config_values.py @@ -18,10 +18,6 @@ def appconfig(): @pytest.fixture def mock_config_file(monkeypatch): - # Patch this; otherwise tempfile.tempdir will be set, which is a global variable that - # defines the value of the default `dir` argument to the functions in Python's - # tempfile module - which breaks multiple tests. - monkeypatch.setattr(config.GalaxyAppConfiguration, '_override_tempdir', lambda a, b: None) # Set this to return None to force the creation of base config directories # in _set_config_directories(). Used to test the values of these directories only. monkeypatch.setattr(config, 'find_config_file', lambda x: None) From a3094f7dbeb61bd9af18dcac76002143c0925449 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 13:26:32 +0100 Subject: [PATCH 11/27] Drop _override_tempdir patch and just set override_tempdir to False. --- lib/galaxy/config/__init__.py | 1 - test/unit/config/test_config_dataset_storage.py | 10 +++++----- test/unit/webapps/test_config_values.py | 6 +++--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 8a8d0fc9f5a..96c2a63023e 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -707,7 +707,6 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): def _override_tempdir(self, kwargs): if string_as_bool(kwargs.get("override_tempdir", "True")): - raise Exception("How did we get here ?") tempfile.tempdir = self.new_file_path def config_value_for_host(self, config_option, host): diff --git a/test/unit/config/test_config_dataset_storage.py b/test/unit/config/test_config_dataset_storage.py index 3b66482f11e..428e4968f0d 100644 --- a/test/unit/config/test_config_dataset_storage.py +++ b/test/unit/config/test_config_dataset_storage.py @@ -6,14 +6,14 @@ from galaxy.config import BaseAppConfiguration def test_object_store_store_by_set(): # object_store_store_by set by admin - appconfig = config.GalaxyAppConfiguration(object_store_store_by='id') + appconfig = config.GalaxyAppConfiguration(object_store_store_by='id', override_tempdir=False) assert appconfig.object_store_store_by == 'id' def test_uuid_1(): # object_store_store_by not set # file_path set by admin to `objects` (no need for the dir to exist) - appconfig = config.GalaxyAppConfiguration(file_path='objects') + appconfig = config.GalaxyAppConfiguration(file_path='objects', override_tempdir=False) assert appconfig.object_store_store_by == 'uuid' @@ -22,7 +22,7 @@ def test_uuid_2(monkeypatch): # object_store_store_by not set # file_path not set, `files` dir doesn't exist monkeypatch.setattr(BaseAppConfiguration, '_path_exists', lambda self, path: False) - appconfig = config.GalaxyAppConfiguration() + appconfig = config.GalaxyAppConfiguration(override_tempdir=False) assert appconfig.object_store_store_by == 'uuid' @@ -30,7 +30,7 @@ def test_uuid_2(monkeypatch): def test_id_1(): # object_store_store_by not set # file_path set by admin to `not_objects` (no need for the dir to exist) - appconfig = config.GalaxyAppConfiguration(file_path='not_objects') + appconfig = config.GalaxyAppConfiguration(file_path='not_objects', override_tempdir=False) assert appconfig.object_store_store_by == 'id' @@ -39,6 +39,6 @@ def test_id_2(monkeypatch): # object_store_store_by not set # file_path not set, `files` dir exists monkeypatch.setattr(BaseAppConfiguration, '_path_exists', lambda self, path: True if os.path.basename(path) == 'files' else False) - appconfig = config.GalaxyAppConfiguration() + appconfig = config.GalaxyAppConfiguration(override_tempdir=False) assert appconfig.object_store_store_by == 'id' diff --git a/test/unit/webapps/test_config_values.py b/test/unit/webapps/test_config_values.py index 24f922cee46..661752c9837 100644 --- a/test/unit/webapps/test_config_values.py +++ b/test/unit/webapps/test_config_values.py @@ -13,7 +13,7 @@ TestData = namedtuple('TestData', ('key', 'expected', 'loaded')) @pytest.fixture(scope='module') def appconfig(): - return config.GalaxyAppConfiguration() + return config.GalaxyAppConfiguration(override_tempdir=False) @pytest.fixture @@ -35,7 +35,7 @@ def test_common_base_config(appconfig): def test_base_config_if_running_from_source(monkeypatch, mock_config_file): # Simulated condition: running from source, config_file is None. monkeypatch.setattr(config, 'running_from_source', True) - appconfig = config.GalaxyAppConfiguration() + appconfig = config.GalaxyAppConfiguration(override_tempdir=False) assert not appconfig.config_file assert appconfig.config_dir == os.path.join(appconfig.root, 'config') assert appconfig.data_dir == os.path.join(appconfig.root, 'database') @@ -45,7 +45,7 @@ def test_base_config_if_running_from_source(monkeypatch, mock_config_file): def test_base_config_if_running_not_from_source(monkeypatch, mock_config_file): # Simulated condition: running not from source, config_file is None. monkeypatch.setattr(config, 'running_from_source', False) - appconfig = config.GalaxyAppConfiguration() + appconfig = config.GalaxyAppConfiguration(override_tempdir=False) assert not appconfig.config_file assert appconfig.config_dir == os.getcwd() assert appconfig.data_dir == os.path.join(appconfig.config_dir, 'data') From c08a12f5533631139e24bfff7eadea827293ae54 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 15:30:15 +0100 Subject: [PATCH 12/27] Drop get_database_url helper I don't see an advantage over accessing the config attribute instead. --- lib/galaxy/app.py | 10 ++-------- scripts/cleanup_datasets/pgcleanup.py | 2 +- scripts/runtime_stats.py | 2 +- scripts/set_user_disk_usage.py | 2 +- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 8319669bf48..58b218a3c2b 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -143,16 +143,10 @@ class SentryClientMixin: self.application_stack.register_postfork_function(postfork_sentry_client) -def get_database_url(config): - db_url = config.database_connection - return db_url - - def init_models_from_config(config, map_install_models=False, object_store=None, trace_logger=None): - db_url = get_database_url(config) model = mapping.init( config.file_path, - db_url, + config.database_connection, config.database_engine_options, map_install_models=map_install_models, database_query_profiling_proxy=config.database_query_profiling_proxy, @@ -313,7 +307,7 @@ class ConfiguresGalaxyMixin: def _configure_models(self, check_migrate_databases=False, config_file=None): """Preconditions: object_store must be set on self.""" - db_url = get_database_url(self.config) + db_url = self.config.database_connection install_db_url = self.config.install_database_connection # TODO: Consider more aggressive check here that this is not the same # database file under the hood. diff --git a/scripts/cleanup_datasets/pgcleanup.py b/scripts/cleanup_datasets/pgcleanup.py index b20ca097a46..48721ac3202 100755 --- a/scripts/cleanup_datasets/pgcleanup.py +++ b/scripts/cleanup_datasets/pgcleanup.py @@ -1112,7 +1112,7 @@ class Cleanup: @property def conn(self): if self.__conn is None: - url = make_url(galaxy.config.get_database_url(self.config)) + url = make_url(self.config.database_connection) log.info(f'Connecting to database with URL: {url}') args = url.translate_connect_args(username='user') args.update(url.query) diff --git a/scripts/runtime_stats.py b/scripts/runtime_stats.py index 4e358d278dd..f4206874b16 100755 --- a/scripts/runtime_stats.py +++ b/scripts/runtime_stats.py @@ -117,7 +117,7 @@ def parse_arguments(): app_properties = app_properties_from_args(args) config = galaxy.config.Configuration(**app_properties) - uri = args.config.get_database_url(config) + uri = config.database_connection names = {'database': 'dbname', 'username': 'user'} args.connect_args = url.make_url(uri).translate_connect_args(**names) diff --git a/scripts/set_user_disk_usage.py b/scripts/set_user_disk_usage.py index bfd1f3d394e..684f6a7b1b7 100755 --- a/scripts/set_user_disk_usage.py +++ b/scripts/set_user_disk_usage.py @@ -31,7 +31,7 @@ def init(): app_properties = app_properties_from_args(args) config = galaxy.config.Configuration(**app_properties) object_store = build_object_store_from_config(config) - engine = galaxy.config.get_database_url(config).split(":")[0] + engine = config.database_connection.split(":")[0] return galaxy.config.init_models_from_config(config, object_store=object_store), object_store, engine From 034f518e43888791ff61feef31c9dcea519556fc Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 15:36:40 +0100 Subject: [PATCH 13/27] Move init_models_from_config to galaxy.model.mapping --- lib/galaxy/app.py | 23 ++++---------------- lib/galaxy/model/mapping.py | 18 +++++++++++++++ scripts/cleanup_datasets/cleanup_datasets.py | 3 ++- scripts/cleanup_datasets/populate_uuid.py | 3 ++- scripts/grt/export.py | 3 ++- scripts/helper.py | 3 ++- scripts/pages_identifier_conversion.py | 3 ++- scripts/set_dataset_sizes.py | 3 ++- scripts/set_user_disk_usage.py | 3 ++- 9 files changed, 36 insertions(+), 26 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 58b218a3c2b..f0903c9bc66 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -39,11 +39,13 @@ from galaxy.managers.workflows import ( WorkflowContentsManager, WorkflowsManager, ) -from galaxy.model import mapping from galaxy.model.base import SharedModelMapping from galaxy.model.database_heartbeat import DatabaseHeartbeat from galaxy.model.database_utils import database_exists -from galaxy.model.mapping import GalaxyModelMapping +from galaxy.model.mapping import ( + GalaxyModelMapping, + init_models_from_config, +) from galaxy.model.scoped_session import ( galaxy_scoped_session, install_model_scoped_session, @@ -143,23 +145,6 @@ class SentryClientMixin: self.application_stack.register_postfork_function(postfork_sentry_client) -def init_models_from_config(config, map_install_models=False, object_store=None, trace_logger=None): - model = mapping.init( - config.file_path, - config.database_connection, - config.database_engine_options, - map_install_models=map_install_models, - database_query_profiling_proxy=config.database_query_profiling_proxy, - object_store=object_store, - trace_logger=trace_logger, - use_pbkdf2=config.get_bool('use_pbkdf2', True), - slow_query_log_threshold=config.slow_query_log_threshold, - thread_local_log=config.thread_local_log, - log_query_counts=config.database_log_query_counts, - ) - return model - - class ConfiguresGalaxyMixin: """Shared code for configuring Galaxy-like app objects.""" diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index c41e6f6efdd..a17f3c06003 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -9,6 +9,7 @@ from threading import local from typing import Optional, Type from galaxy import model +from galaxy.config import GalaxyAppConfiguration from galaxy.model import mapper_registry from galaxy.model.base import SharedModelMapping from galaxy.model.migrate.triggers.update_audit_table import install as install_timestamp_triggers @@ -64,3 +65,20 @@ def init(file_path, url, engine_options=None, create_tables=False, map_install_m result.security_agent = GalaxyRBACAgent(result) result.thread_local_log = thread_local_log return result + + +def init_models_from_config(config: GalaxyAppConfiguration, map_install_models=False, object_store=None, trace_logger=None): + model = init( + config.file_path, + config.database_connection, + config.database_engine_options, + map_install_models=map_install_models, + database_query_profiling_proxy=config.database_query_profiling_proxy, + object_store=object_store, + trace_logger=trace_logger, + use_pbkdf2=config.get_bool('use_pbkdf2', True), + slow_query_log_threshold=config.slow_query_log_threshold, + thread_local_log=config.thread_local_log, + log_query_counts=config.database_log_query_counts, + ) + return model diff --git a/scripts/cleanup_datasets/cleanup_datasets.py b/scripts/cleanup_datasets/cleanup_datasets.py index ac858881d44..965fd44e2e2 100755 --- a/scripts/cleanup_datasets/cleanup_datasets.py +++ b/scripts/cleanup_datasets/cleanup_datasets.py @@ -18,6 +18,7 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa import galaxy.config from galaxy.datatypes.registry import Registry from galaxy.exceptions import ObjectNotFound +from galaxy.model.mapping import init_models_from_config from galaxy.objectstore import build_object_store_from_config from galaxy.util import unicodify from galaxy.util.script import app_properties_from_args, populate_config_args @@ -519,7 +520,7 @@ class CleanupDatasetsApplication: def __init__(self, config): self.object_store = build_object_store_from_config(config) # Setup the database engine and ORM - self.model = galaxy.config.init_models_from_config(config, object_store=self.object_store) + self.model = init_models_from_config(config, object_store=self.object_store) registry = Registry() registry.load_datatypes() galaxy.model.set_datatypes_registry(registry) diff --git a/scripts/cleanup_datasets/populate_uuid.py b/scripts/cleanup_datasets/populate_uuid.py index da262653a41..b5f2ea624c5 100755 --- a/scripts/cleanup_datasets/populate_uuid.py +++ b/scripts/cleanup_datasets/populate_uuid.py @@ -15,6 +15,7 @@ import uuid sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) import galaxy.config +from galaxy.model.mapping import init_models_from_config from galaxy.util.script import app_properties_from_args, populate_config_args DESCRIPTION = """ @@ -32,7 +33,7 @@ def main(): app_properties = app_properties_from_args(args) config = galaxy.config.Configuration(**app_properties) - model = galaxy.config.init_models_from_config(config) + model = init_models_from_config(config) for row in model.context.query(model.Dataset): if row.uuid is None: diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 3b586d2da07..b1b12ffec9c 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -18,6 +18,7 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa import galaxy import galaxy.app import galaxy.config +from galaxy.model.mapping import init_models_from_config from galaxy.objectstore import build_object_store_from_config from galaxy.util import ( hash_util, @@ -36,7 +37,7 @@ def _init(args): if not config.database_connection: logging.warning("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml") - model = galaxy.config.init_models_from_config(config, object_store=object_store) + model = init_models_from_config(config, object_store=object_store) return ( model, object_store, diff --git a/scripts/helper.py b/scripts/helper.py index 97b3e394f1d..e52c11ba9ed 100644 --- a/scripts/helper.py +++ b/scripts/helper.py @@ -12,6 +12,7 @@ import sys sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, 'lib')) import galaxy.config +from galaxy.model.mapping import init_models_from_config from galaxy.security import idencoding from galaxy.util.script import app_properties_from_args, populate_config_args @@ -26,7 +27,7 @@ args = parser.parse_args() app_properties = app_properties_from_args(args) config = galaxy.config.Configuration(**app_properties) helper = idencoding.IdEncodingHelper(id_secret=app_properties.get('id_secret')) -model = galaxy.config.init_models_from_config(config) +model = init_models_from_config(config) if args.encode_id: print(f'Encoded "{args.encode_id}": {helper.encode_id(args.encode_id)}') diff --git a/scripts/pages_identifier_conversion.py b/scripts/pages_identifier_conversion.py index b9cec401071..d91c252aac0 100644 --- a/scripts/pages_identifier_conversion.py +++ b/scripts/pages_identifier_conversion.py @@ -10,6 +10,7 @@ import galaxy import galaxy.app import galaxy.config from galaxy.managers.pages import PageContentProcessor, placeholderRenderForSave +from galaxy.model.mapping import init_models_from_config from galaxy.objectstore import build_object_store_from_config from galaxy.security.idencoding import IdEncodingHelper from galaxy.util import unicodify @@ -31,7 +32,7 @@ def main(argv): if not config.database_connection: print("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml") - model = galaxy.config.init_models_from_config(config, object_store=object_store) + model = init_models_from_config(config, object_store=object_store) session = model.context.current pagerevs = session.query(model.PageRevision).all() mock_trans = Bunch(app=Bunch(security=security_helper), model=model, user_is_admin=lambda: True, sa_session=session) diff --git a/scripts/set_dataset_sizes.py b/scripts/set_dataset_sizes.py index d3a6e355478..c980d755a8f 100644 --- a/scripts/set_dataset_sizes.py +++ b/scripts/set_dataset_sizes.py @@ -7,6 +7,7 @@ import sys sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) import galaxy.config +from galaxy.model.mapping import init_models_from_config from galaxy.objectstore import build_object_store_from_config from galaxy.util.script import app_properties_from_args, populate_config_args @@ -20,7 +21,7 @@ def init(): config = galaxy.config.Configuration(**app_properties) object_store = build_object_store_from_config(config) - model = galaxy.config.init_models_from_config(config, object_store=object_store) + model = init_models_from_config(config, object_store=object_store) return model, object_store diff --git a/scripts/set_user_disk_usage.py b/scripts/set_user_disk_usage.py index 684f6a7b1b7..ca8b936a2ee 100755 --- a/scripts/set_user_disk_usage.py +++ b/scripts/set_user_disk_usage.py @@ -7,6 +7,7 @@ import sys sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) import galaxy.config +from galaxy.model.mapping import init_models_from_config from galaxy.objectstore import build_object_store_from_config from galaxy.util import nice_size from galaxy.util.script import app_properties_from_args, populate_config_args @@ -32,7 +33,7 @@ def init(): config = galaxy.config.Configuration(**app_properties) object_store = build_object_store_from_config(config) engine = config.database_connection.split(":")[0] - return galaxy.config.init_models_from_config(config, object_store=object_store), object_store, engine + return init_models_from_config(config, object_store=object_store), object_store, engine def quotacheck(sa_session, users, engine): From cb24832617feab67ecc24d25078a49f5f1b22161 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Tue, 1 Mar 2022 16:12:47 +0100 Subject: [PATCH 14/27] Adapt project boilerplate Co-authored-by: Nicola Soranzo --- packages/config/galaxy/project_galaxy_util.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/config/galaxy/project_galaxy_util.py b/packages/config/galaxy/project_galaxy_util.py index 48b69d86178..b7a5fd77095 100644 --- a/packages/config/galaxy/project_galaxy_util.py +++ b/packages/config/galaxy/project_galaxy_util.py @@ -3,9 +3,7 @@ __version__ = "22.1.0rc1" PROJECT_NAME = "galaxy-config" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Generic Utilities' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Configuration" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) From 13e6f341991d51ac2f1bb80d45fbc4cf2976238b Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Tue, 1 Mar 2022 16:13:34 +0100 Subject: [PATCH 15/27] Drop responses test requirement from config package Co-authored-by: Nicola Soranzo --- packages/config/test-requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/config/test-requirements.txt b/packages/config/test-requirements.txt index ca5a0dac0ee..99440d27fbc 100644 --- a/packages/config/test-requirements.txt +++ b/packages/config/test-requirements.txt @@ -1,2 +1 @@ pytest<6.1 -responses From e3b7fe0805a8b2ebaac2b96c9c82fa13f5480665 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 16:15:09 +0100 Subject: [PATCH 16/27] Fix package symlinks --- packages/app/galaxy/version.py | 1 - packages/config/tests/util | 1 - 2 files changed, 2 deletions(-) delete mode 120000 packages/app/galaxy/version.py delete mode 120000 packages/config/tests/util diff --git a/packages/app/galaxy/version.py b/packages/app/galaxy/version.py deleted file mode 120000 index 42d972d9994..00000000000 --- a/packages/app/galaxy/version.py +++ /dev/null @@ -1 +0,0 @@ -../../../lib/galaxy/version.py \ No newline at end of file diff --git a/packages/config/tests/util b/packages/config/tests/util deleted file mode 120000 index d42557d85de..00000000000 --- a/packages/config/tests/util +++ /dev/null @@ -1 +0,0 @@ -../../../test/unit/util \ No newline at end of file From 25c5a47ef1149c972409e403f58f72a84890e8d0 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Tue, 1 Mar 2022 16:18:26 +0100 Subject: [PATCH 17/27] Use imports from top Co-authored-by: Nicola Soranzo --- lib/galaxy/app.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index f0903c9bc66..f20e7d9bb47 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -52,6 +52,7 @@ from galaxy.model.scoped_session import ( ) from galaxy.model.tags import GalaxyTagHandler from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database +from galaxy.objectstore import build_object_store_from_config from galaxy.queue_worker import ( GalaxyQueueWorker, send_local_control_task, @@ -74,6 +75,7 @@ from galaxy.tools.cache import ( ToolCache, ToolShedRepositoryCache ) +from galaxy.tools.data import ToolDataTableManager from galaxy.tools.data_manager.manager import DataManagers from galaxy.tools.error_reports import ErrorReports from galaxy.tools.search import ToolBoxSearch @@ -94,6 +96,7 @@ from galaxy.web import url_for from galaxy.web.proxy import ProxyManager from galaxy.web_stack import application_stack_instance, ApplicationStack from galaxy.webhooks import WebhooksRegistry +from galaxy.workflow import scheduling_manager from galaxy.workflow.trs_proxy import TrsProxy from .di import Container from .structured_app import BasicSharedApp, MinimalManagerApp, StructuredApp @@ -236,8 +239,6 @@ class ConfiguresGalaxyMixin: self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.default_container_registry.container_resolvers) def _configure_tool_data_tables(self, from_shed_config): - from galaxy.tools.data import ToolDataTableManager - # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path. self.tool_data_tables = ToolDataTableManager(tool_data_path=self.config.tool_data_path, config_filename=self.config.tool_data_table_config_path, @@ -253,9 +254,8 @@ class ConfiguresGalaxyMixin: raise def _configure_datatypes_registry(self, installed_repository_manager=None): - from galaxy.datatypes import registry # Create an empty datatypes registry. - self.datatypes_registry = registry.Registry(self.config) + self.datatypes_registry = Registry(self.config) if installed_repository_manager and self.config.load_tool_shed_datatypes: # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories. We # load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some @@ -273,12 +273,10 @@ class ConfiguresGalaxyMixin: self.datatypes_registry.load_datatypes(self.config.root, datatypes_config, override=True) def _configure_object_store(self, **kwds): - from galaxy.objectstore import build_object_store_from_config self.object_store = build_object_store_from_config(self.config, **kwds) def _configure_security(self): - from galaxy.security import idencoding - self.security = idencoding.IdEncodingHelper(id_secret=self.config.id_secret) + self.security = IdEncodingHelper(id_secret=self.config.id_secret) BaseDatabaseIdField.security = self.security def _configure_tool_shed_registry(self): @@ -580,7 +578,6 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): self.job_manager._check_jobs_at_startup() self.proxy_manager = ProxyManager(self.config) - from galaxy.workflow import scheduling_manager # Must be initialized after job_config. self.workflow_scheduling_manager = scheduling_manager.WorkflowSchedulingManager(self) From 7ddf3ea07560c161cc24ddac91835c8de20d9a73 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Tue, 1 Mar 2022 16:25:57 +0100 Subject: [PATCH 18/27] Add galaxy-objectstore to app package requirement Co-authored-by: Nicola Soranzo --- packages/app/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/app/requirements.txt b/packages/app/requirements.txt index c0561b95e6e..7355cd14454 100644 --- a/packages/app/requirements.txt +++ b/packages/app/requirements.txt @@ -3,6 +3,7 @@ galaxy-config galaxy-data galaxy-job-execution galaxy-job-metrics +galaxy-objectstore galaxy-tool-util[cwl,edam] galaxy-web-framework galaxy-web-stack From 6b08108a88739669134baeaa1a7a0d3eeadf7335 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Tue, 1 Mar 2022 16:26:56 +0100 Subject: [PATCH 19/27] Fix History formatting Co-authored-by: Nicola Soranzo --- packages/config/HISTORY.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/config/HISTORY.rst b/packages/config/HISTORY.rst index e5abec134b3..3988a66c6d3 100644 --- a/packages/config/HISTORY.rst +++ b/packages/config/HISTORY.rst @@ -8,4 +8,5 @@ History --------------------- 22.1.0.dev0 --------------------- -Initial release + +* Initial release From 3c3e60d9b1714ef263fa8303d486a5d035eb54a8 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Tue, 1 Mar 2022 16:29:13 +0100 Subject: [PATCH 20/27] Use black style double quotes Co-authored-by: Nicola Soranzo --- packages/config/galaxy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/config/galaxy/__init__.py b/packages/config/galaxy/__init__.py index 8e13dacd253..2e50d9cce89 100644 --- a/packages/config/galaxy/__init__.py +++ b/packages/config/galaxy/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore[has-type] +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore[has-type] From cfd66f49b5b1fe8e0f87595f97df5a7305be8639 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 16:33:36 +0100 Subject: [PATCH 21/27] Switch license back to AFL --- packages/config/LICENSE | 2 +- packages/config/setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/config/LICENSE b/packages/config/LICENSE index fb4eead94ae..1ef648f64b3 120000 --- a/packages/config/LICENSE +++ b/packages/config/LICENSE @@ -1 +1 @@ -../MIT_LICENSE.txt \ No newline at end of file +../../LICENSE.txt \ No newline at end of file diff --git a/packages/config/setup.py b/packages/config/setup.py index 068ac97847e..d84fe28de4f 100644 --- a/packages/config/setup.py +++ b/packages/config/setup.py @@ -75,14 +75,14 @@ setup( package_dir=PACKAGE_DIR, include_package_data=True, install_requires=requirements, - license="MIT", + license="AFL", zip_safe=False, keywords='galaxy', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Environment :: Console', - 'License :: OSI Approved :: MIT License', + 'License :: OSI Approved :: Academic Free License (AFL)', 'Operating System :: POSIX', 'Topic :: Software Development', 'Topic :: Software Development :: Code Generators', From 1828f4aa816d5cd16b93f09d08955573b61bfbe3 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 17:03:38 +0100 Subject: [PATCH 22/27] Move more things to the top --- lib/galaxy/app.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index f20e7d9bb47..f80c7b131cf 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -21,6 +21,7 @@ from galaxy.containers import build_container_interfaces from galaxy.datatypes.registry import Registry from galaxy.files import ConfiguredFileSources from galaxy.job_metrics import JobMetrics +from galaxy.jobs.manager import JobManager from galaxy.managers.api_keys import ApiKeyManager from galaxy.managers.citations import CitationsManager from galaxy.managers.collections import DatasetCollectionManager @@ -39,6 +40,7 @@ from galaxy.managers.workflows import ( WorkflowContentsManager, WorkflowsManager, ) +from galaxy.model import custom_types from galaxy.model.base import SharedModelMapping from galaxy.model.database_heartbeat import DatabaseHeartbeat from galaxy.model.database_utils import database_exists @@ -46,11 +48,13 @@ from galaxy.model.mapping import ( GalaxyModelMapping, init_models_from_config, ) +from galaxy.model.migrate.check import create_or_verify_database from galaxy.model.scoped_session import ( galaxy_scoped_session, install_model_scoped_session, ) from galaxy.model.tags import GalaxyTagHandler +from galaxy.model.tool_shed_install import mapping as install_mapping from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database from galaxy.objectstore import build_object_store_from_config from galaxy.queue_worker import ( @@ -64,6 +68,7 @@ from galaxy.security.vault import ( Vault, VaultFactory ) +from galaxy.tool_shed import tool_shed_registry from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager from galaxy.tool_util.deps import containers @@ -280,13 +285,11 @@ class ConfiguresGalaxyMixin: BaseDatabaseIdField.security = self.security def _configure_tool_shed_registry(self): - import galaxy.tool_shed.tool_shed_registry - # Set up the tool sheds registry if os.path.isfile(self.config.tool_sheds_config_file): - self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry(self.config.tool_sheds_config_file) + self.tool_shed_registry = tool_shed_registry.Registry(self.config.tool_sheds_config_file) else: - self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry() + self.tool_shed_registry = tool_shed_registry.Registry() def _configure_models(self, check_migrate_databases=False, config_file=None): """Preconditions: object_store must be set on self.""" @@ -302,13 +305,11 @@ class ConfiguresGalaxyMixin: self._wait_for_database(db_url) if getattr(self.config, "max_metadata_value_size", None): - from galaxy.model import custom_types custom_types.MAX_METADATA_VALUE_SIZE = self.config.max_metadata_value_size if check_migrate_databases: # Initialize database / check for appropriate schema version. # If this # is a new installation, we'll restrict the tool migration messaging. - from galaxy.model.migrate.check import create_or_verify_database create_or_verify_database(db_url, config_file, self.config.database_engine_options, app=self, map_install_models=combined_install_database) if not combined_install_database: tsi_create_or_verify_database(install_db_url, install_database_options, app=self) @@ -323,7 +324,6 @@ class ConfiguresGalaxyMixin: log.info("Install database targetting Galaxy's database configuration.") self.install_model = self.model else: - from galaxy.model.tool_shed_install import mapping as install_mapping install_db_url = self.config.install_database_connection log.info(f"Install database using its own connection {install_db_url}") self.install_model = install_mapping.init(install_db_url, @@ -429,7 +429,6 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication): self.library_manager = self._register_singleton(LibraryManager) self.library_datasets_manager = self._register_singleton(LibraryDatasetsManager) self.role_manager = self._register_singleton(RoleManager) - from galaxy.jobs.manager import JobManager self.job_manager = self._register_singleton(JobManager) # ConfiguredFileSources From 48e2ebe20f5932a01d5e883fe548d8ac72ae72a2 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Tue, 1 Mar 2022 19:22:45 +0100 Subject: [PATCH 23/27] List transitive dependencies that are imported directly in config package Co-authored-by: Nicola Soranzo --- packages/config/requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/config/requirements.txt b/packages/config/requirements.txt index e663ba0d20f..a3caa0acfea 100644 --- a/packages/config/requirements.txt +++ b/packages/config/requirements.txt @@ -1,2 +1,5 @@ galaxy-containers +boltons pykwalify +PyYAML +requests From 4cffa788365d87628a0ecccdb59c4918fbf47bdb Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 18:46:12 +0100 Subject: [PATCH 24/27] Drop pykwalify from app requirements Should be redundant as it is handled by the config package requirements. --- packages/app/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/app/requirements.txt b/packages/app/requirements.txt index 7355cd14454..6e5279f1e2f 100644 --- a/packages/app/requirements.txt +++ b/packages/app/requirements.txt @@ -20,7 +20,6 @@ Markdown paramiko!=2.9.0,!=2.9.1 pulsar-galaxy-lib>=0.14.13 pydantic -pykwalify refgenconf>=0.12.0 sqlitedict svgwrite From 022bae06541b1b589ab41a492b81012c6216d8f5 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 19:26:45 +0100 Subject: [PATCH 25/27] Add pykwalify to ``app`` package test requirements --- packages/app/test-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/app/test-requirements.txt b/packages/app/test-requirements.txt index 1fd0bf58c90..f93c7d6a95a 100644 --- a/packages/app/test-requirements.txt +++ b/packages/app/test-requirements.txt @@ -1,3 +1,4 @@ mock-ssh-server +pykwalify pytest testfixtures From 6b32e231019cc4eb80c02c9566c2f8633e3a9f1e Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 1 Mar 2022 19:35:56 +0000 Subject: [PATCH 26/27] Reformat with black and pyupgrade --- packages/config/setup.py | 64 +++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/packages/config/setup.py b/packages/config/setup.py index d84fe28de4f..46f35f268fe 100644 --- a/packages/config/setup.py +++ b/packages/config/setup.py @@ -3,6 +3,7 @@ import ast import os import re + try: from setuptools import setup except ImportError: @@ -10,13 +11,14 @@ except ImportError: SOURCE_DIR = "galaxy" -_version_re = re.compile(r'__version__\s+=\s+(.*)') +_version_re = re.compile(r"__version__\s+=\s+(.*)") -with open('%s/project_galaxy_util.py' % SOURCE_DIR, 'rb') as f: - init_contents = f.read().decode('utf-8') +project_short_name = os.path.basename(os.path.dirname(os.path.realpath(__file__))) +with open(f"{SOURCE_DIR}/project_galaxy_{project_short_name}.py", "rb") as f: + init_contents = f.read().decode("utf-8") def get_var(var_name): - pattern = re.compile(r'%s\s+=\s+(.*)' % var_name) + pattern = re.compile(rf"{var_name}\s+=\s+(.*)") match = pattern.search(init_contents).group(1) return str(ast.literal_eval(match)) @@ -27,28 +29,28 @@ with open('%s/project_galaxy_util.py' % SOURCE_DIR, 'rb') as f: PROJECT_EMAIL = get_var("PROJECT_EMAIL") PROJECT_DESCRIPTION = get_var("PROJECT_DESCRIPTION") -TEST_DIR = 'tests' +TEST_DIR = "tests" PACKAGES = [ - 'galaxy', - 'galaxy.config', + "galaxy", + "galaxy.config", ] -ENTRY_POINTS = ''' +ENTRY_POINTS = """ [console_scripts] galaxy-config=galaxy.config.script:main -''' +""" PACKAGE_DATA = { # Be sure to update MANIFEST.in for source dist. - 'galaxy': [ - 'config/schemas/*.yml', - 'config/sample/*', + "galaxy": [ + "config/schemas/*.yml", + "config/sample/*", ], } PACKAGE_DIR = { SOURCE_DIR: SOURCE_DIR, } -readme = open('README.rst').read() -history = open('HISTORY.rst').read().replace('.. :changelog:', '') +readme = open("README.rst").read() +history = open("HISTORY.rst").read().replace(".. :changelog:", "") if os.path.exists("requirements.txt"): requirements = open("requirements.txt").read().split("\n") @@ -64,8 +66,8 @@ setup( name=PROJECT_NAME, version=version, description=PROJECT_DESCRIPTION, - long_description=readme + '\n\n' + history, - long_description_content_type='text/x-rst', + long_description=readme + "\n\n" + history, + long_description_content_type="text/x-rst", author=PROJECT_AUTHOR, author_email=PROJECT_EMAIL, url=PROJECT_URL, @@ -77,23 +79,23 @@ setup( install_requires=requirements, license="AFL", zip_safe=False, - keywords='galaxy', + keywords="galaxy", classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Environment :: Console', - 'License :: OSI Approved :: Academic Free License (AFL)', - 'Operating System :: POSIX', - 'Topic :: Software Development', - 'Topic :: Software Development :: Code Generators', - 'Topic :: Software Development :: Testing', - 'Natural Language :: English', + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Environment :: Console", + "License :: OSI Approved :: Academic Free License (AFL)", + "Operating System :: POSIX", + "Topic :: Software Development", + "Topic :: Software Development :: Code Generators", + "Topic :: Software Development :: Testing", + "Natural Language :: English", "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], test_suite=TEST_DIR, - tests_require=test_requirements + tests_require=test_requirements, ) From e89b47b7d6668981413875e220e7f148de5515c6 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Mar 2022 20:43:56 +0100 Subject: [PATCH 27/27] Rename project_galaxy_config.py --- .../galaxy/{project_galaxy_util.py => project_galaxy_config.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/config/galaxy/{project_galaxy_util.py => project_galaxy_config.py} (100%) diff --git a/packages/config/galaxy/project_galaxy_util.py b/packages/config/galaxy/project_galaxy_config.py similarity index 100% rename from packages/config/galaxy/project_galaxy_util.py rename to packages/config/galaxy/project_galaxy_config.py