diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 0b131b4b3dc..f80c7b131cf 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -1,20 +1,29 @@ +import collections +import errno import logging +import os import signal import sys +import threading import time from typing import Any, Callable, List, Tuple +from beaker.cache import CacheManager +from beaker.util import parse_cache_config_options + import galaxy.model import galaxy.model.security import galaxy.queues import galaxy.security -from galaxy import auth, config, jobs +from galaxy import auth, config, jobs, tools from galaxy.config_watchers import ConfigWatchers from galaxy.containers import build_container_interfaces from galaxy.datatypes.registry import Registry from galaxy.files import ConfiguredFileSources from galaxy.job_metrics import JobMetrics +from galaxy.jobs.manager import JobManager from galaxy.managers.api_keys import ApiKeyManager +from galaxy.managers.citations import CitationsManager from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.folders import FolderManager from galaxy.managers.hdas import HDAManager @@ -31,41 +40,59 @@ from galaxy.managers.workflows import ( WorkflowContentsManager, WorkflowsManager, ) +from galaxy.model import custom_types from galaxy.model.base import SharedModelMapping from galaxy.model.database_heartbeat import DatabaseHeartbeat -from galaxy.model.mapping import GalaxyModelMapping +from galaxy.model.database_utils import database_exists +from galaxy.model.mapping import ( + GalaxyModelMapping, + init_models_from_config, +) +from galaxy.model.migrate.check import create_or_verify_database from galaxy.model.scoped_session import ( galaxy_scoped_session, install_model_scoped_session, ) from galaxy.model.tags import GalaxyTagHandler +from galaxy.model.tool_shed_install import mapping as install_mapping +from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database +from galaxy.objectstore import build_object_store_from_config from galaxy.queue_worker import ( GalaxyQueueWorker, send_local_control_task, ) from galaxy.quota import get_quota_agent, QuotaAgent +from galaxy.schema.fields import BaseDatabaseIdField from galaxy.security.idencoding import IdEncodingHelper from galaxy.security.vault import ( Vault, VaultFactory ) +from galaxy.tool_shed import tool_shed_registry from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager +from galaxy.tool_util.deps import containers +from galaxy.tool_util.deps.dependencies import AppInfo from galaxy.tool_util.deps.views import DependencyResolversView from galaxy.tool_util.verify.test_data import TestDataResolver +from galaxy.tools.biotools import get_galaxy_biotools_metadata_source from galaxy.tools.cache import ( ToolCache, ToolShedRepositoryCache ) +from galaxy.tools.data import ToolDataTableManager from galaxy.tools.data_manager.manager import DataManagers from galaxy.tools.error_reports import ErrorReports +from galaxy.tools.search import ToolBoxSearch from galaxy.tools.special_tools import load_lib_tools from galaxy.tours import build_tours_registry, ToursRegistry from galaxy.util import ( ExecutionTimer, heartbeat, + listify, StructuredExecutionTimer, ) +from galaxy.util.dbkeys import GenomeBuilds from galaxy.util.task import IntervalTask from galaxy.visualization.data_providers.registry import DataProviderRegistry from galaxy.visualization.genomes import Genomes @@ -74,6 +101,7 @@ from galaxy.web import url_for from galaxy.web.proxy import ProxyManager from galaxy.web_stack import application_stack_instance, ApplicationStack from galaxy.webhooks import WebhooksRegistry +from galaxy.workflow import scheduling_manager from galaxy.workflow.trs_proxy import TrsProxy from .di import Container from .structured_app import BasicSharedApp, MinimalManagerApp, StructuredApp @@ -125,10 +153,206 @@ class SentryClientMixin: self.application_stack.register_postfork_function(postfork_sentry_client) -class MinimalGalaxyApplication(BasicSharedApp, config.ConfiguresGalaxyMixin, HaltableContainer, SentryClientMixin): +class ConfiguresGalaxyMixin: + """Shared code for configuring Galaxy-like app objects.""" + + config: config.GalaxyAppConfiguration + tool_cache: ToolCache + job_config: jobs.JobConfiguration + toolbox: tools.ToolBox + toolbox_search: ToolBoxSearch + container_finder: containers.ContainerFinder + + def _configure_genome_builds(self, data_table_name="__dbkeys__", load_old_style=True): + self.genome_builds = GenomeBuilds(self, data_table_name=data_table_name, load_old_style=load_old_style) + + def wait_for_toolbox_reload(self, old_toolbox): + timer = ExecutionTimer() + log.debug('Waiting for toolbox reload') + # Wait till toolbox reload has been triggered (or more than 60 seconds have passed) + while timer.elapsed < 60: + if self.toolbox.has_reloaded(old_toolbox): + log.debug('Finished waiting for toolbox reload %s', timer) + break + time.sleep(0.1) + else: + log.warning('Waiting for toolbox reload timed out after 60 seconds') + + def _configure_tool_config_files(self): + if self.config.shed_tool_config_file not in self.config.tool_configs: + self.config.tool_configs.append(self.config.shed_tool_config_file) + # The value of migrated_tools_config is the file reserved for containing only those tools that have been + # eliminated from the distribution and moved to the tool shed. If migration checking is disabled, only add it if + # it exists (since this may be an existing deployment where migrations were previously run). + if (os.path.exists(self.config.migrated_tools_config) + and self.config.migrated_tools_config not in self.config.tool_configs): + self.config.tool_configs.append(self.config.migrated_tools_config) + + def _configure_toolbox(self): + if not isinstance(self, BasicSharedApp): + raise Exception("Must inherit from BasicSharedApp") + + self.citations_manager = CitationsManager(self) + self.biotools_metadata_source = get_galaxy_biotools_metadata_source(self.config) + + self.dynamic_tools_manager = DynamicToolManager(self) + self._toolbox_lock = threading.RLock() + self.toolbox = tools.ToolBox(self.config.tool_configs, self.config.tool_path, self) + galaxy_root_dir = os.path.abspath(self.config.root) + file_path = os.path.abspath(self.config.file_path) + app_info = AppInfo( + galaxy_root_dir=galaxy_root_dir, + default_file_path=file_path, + tool_data_path=self.config.tool_data_path, + shed_tool_data_path=self.config.shed_tool_data_path, + outputs_to_working_directory=self.config.outputs_to_working_directory, + container_image_cache_path=self.config.container_image_cache_path, + library_import_dir=self.config.library_import_dir, + enable_mulled_containers=self.config.enable_mulled_containers, + container_resolvers_config_file=self.config.container_resolvers_config_file, + container_resolvers_config_dict=self.config.container_resolvers, + involucro_path=self.config.involucro_path, + involucro_auto_init=self.config.involucro_auto_init, + mulled_channels=self.config.mulled_channels, + ) + mulled_resolution_cache = None + if self.config.mulled_resolution_cache_type: + cache_opts = { + 'cache.type': self.config.mulled_resolution_cache_type, + 'cache.data_dir': self.config.mulled_resolution_cache_data_dir, + 'cache.lock_dir': self.config.mulled_resolution_cache_lock_dir, + } + mulled_resolution_cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache('mulled_resolution') + self.container_finder = containers.ContainerFinder(app_info, mulled_resolution_cache=mulled_resolution_cache) + self._set_enabled_container_types() + index_help = getattr(self.config, "index_tool_help", True) + self.toolbox_search = ToolBoxSearch(self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help) + + def reindex_tool_search(self): + # Call this when tools are added or removed. + self.toolbox_search.build_index(tool_cache=self.tool_cache) + self.tool_cache.reset_status() + + def _set_enabled_container_types(self): + container_types_to_destinations = collections.defaultdict(list) + for destinations in self.job_config.destinations.values(): + for destination in destinations: + for enabled_container_type in self.container_finder._enabled_container_types(destination.params): + container_types_to_destinations[enabled_container_type].append(destination) + self.toolbox.dependency_manager.set_enabled_container_types(container_types_to_destinations) + self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.default_container_registry.resolver_classes) + self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.default_container_registry.container_resolvers) + + def _configure_tool_data_tables(self, from_shed_config): + # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path. + self.tool_data_tables = ToolDataTableManager(tool_data_path=self.config.tool_data_path, + config_filename=self.config.tool_data_table_config_path, + other_config_dict=self.config) + # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables. + try: + self.tool_data_tables.load_from_config_file(config_filename=self.config.shed_tool_data_table_config, + tool_data_path=self.tool_data_tables.tool_data_path, + from_shed_config=from_shed_config) + except OSError as exc: + # Missing shed_tool_data_table_config is okay if it's the default + if exc.errno != errno.ENOENT or self.config.is_set('shed_tool_data_table_config'): + raise + + def _configure_datatypes_registry(self, installed_repository_manager=None): + # Create an empty datatypes registry. + self.datatypes_registry = Registry(self.config) + if installed_repository_manager and self.config.load_tool_shed_datatypes: + # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories. We + # load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some + # generic sniffers (eg text,xml) which catch anything, so it's impossible for proprietary sniffers to be used. + # However, if there is a conflict (2 datatypes with the same extension) between a proprietary datatype and a datatype + # in the Galaxy distribution, the datatype in the Galaxy distribution will take precedence. If there is a conflict + # between 2 proprietary datatypes, the datatype from the repository that was installed earliest will take precedence. + installed_repository_manager.load_proprietary_datatypes() + # Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config. + datatypes_configs = self.config.datatypes_config + for datatypes_config in listify(datatypes_configs): + # Setting override=False would make earlier files would take + # precedence - but then they wouldn't override tool shed + # datatypes. + self.datatypes_registry.load_datatypes(self.config.root, datatypes_config, override=True) + + def _configure_object_store(self, **kwds): + self.object_store = build_object_store_from_config(self.config, **kwds) + + def _configure_security(self): + self.security = IdEncodingHelper(id_secret=self.config.id_secret) + BaseDatabaseIdField.security = self.security + + def _configure_tool_shed_registry(self): + # Set up the tool sheds registry + if os.path.isfile(self.config.tool_sheds_config_file): + self.tool_shed_registry = tool_shed_registry.Registry(self.config.tool_sheds_config_file) + else: + self.tool_shed_registry = tool_shed_registry.Registry() + + def _configure_models(self, check_migrate_databases=False, config_file=None): + """Preconditions: object_store must be set on self.""" + db_url = self.config.database_connection + install_db_url = self.config.install_database_connection + # TODO: Consider more aggressive check here that this is not the same + # database file under the hood. + combined_install_database = not(install_db_url and install_db_url != db_url) + install_db_url = install_db_url or db_url + install_database_options = self.config.database_engine_options if combined_install_database else self.config.install_database_engine_options + + if self.config.database_wait: + self._wait_for_database(db_url) + + if getattr(self.config, "max_metadata_value_size", None): + custom_types.MAX_METADATA_VALUE_SIZE = self.config.max_metadata_value_size + + if check_migrate_databases: + # Initialize database / check for appropriate schema version. # If this + # is a new installation, we'll restrict the tool migration messaging. + create_or_verify_database(db_url, config_file, self.config.database_engine_options, app=self, map_install_models=combined_install_database) + if not combined_install_database: + tsi_create_or_verify_database(install_db_url, install_database_options, app=self) + + self.model = init_models_from_config( + self.config, + map_install_models=combined_install_database, + object_store=self.object_store, + trace_logger=getattr(self, "trace_logger", None) + ) + if combined_install_database: + log.info("Install database targetting Galaxy's database configuration.") + self.install_model = self.model + else: + install_db_url = self.config.install_database_connection + log.info(f"Install database using its own connection {install_db_url}") + self.install_model = install_mapping.init(install_db_url, + install_database_options) + + def _configure_signal_handlers(self, handlers): + for sig, handler in handlers.items(): + signal.signal(sig, handler) + + def _wait_for_database(self, url): + attempts = self.config.database_wait_attempts + pause = self.config.database_wait_sleep + for i in range(1, attempts): + try: + database_exists(url) + break + except Exception: + log.info("Waiting for database: attempt %d of %d" % (i, attempts)) + time.sleep(pause) + + @property + def tool_dependency_dir(self): + return self.toolbox.dependency_manager.default_base_path + + +class MinimalGalaxyApplication(BasicSharedApp, ConfiguresGalaxyMixin, HaltableContainer, SentryClientMixin): """Encapsulates the state of a minimal Galaxy application""" - def __init__(self, fsmon=False, configure_logging=True, **kwargs) -> None: + def __init__(self, fsmon=False, **kwargs) -> None: super().__init__() self.haltables = [ ("object store", self._shutdown_object_store), @@ -147,8 +371,6 @@ class MinimalGalaxyApplication(BasicSharedApp, config.ConfiguresGalaxyMixin, Hal # Read config file and check for errors self.config: Any = self._register_singleton(config.Configuration, config.Configuration(**kwargs)) self.config.check() - if configure_logging: - config.configure_logging(self.config) self._configure_object_store(fsmon=True) config_file = kwargs.get('global_conf', {}).get('__file__', None) if config_file: @@ -179,12 +401,14 @@ class MinimalGalaxyApplication(BasicSharedApp, config.ConfiguresGalaxyMixin, Hal class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication): """Extends the MinimalGalaxyApplication with most managers that are not tied to a web or job handling context.""" - def __init__(self, **kwargs): + def __init__(self, configure_logging=True, **kwargs): super().__init__(**kwargs) self._register_singleton(MinimalManagerApp, self) self.execution_timer_factory = self._register_singleton(ExecutionTimerFactory, ExecutionTimerFactory(self.config)) self.configure_fluent_log() self.application_stack = self._register_singleton(ApplicationStack, application_stack_instance(app=self)) + if configure_logging: + config.configure_logging(self.config, self.application_stack.facts) # Initialize job metrics manager, needs to be in place before # config so per-destination modifications can be made. self.job_metrics = self._register_singleton(JobMetrics, JobMetrics(self.config.job_metrics_config_file, app=self)) @@ -205,7 +429,6 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication): self.library_manager = self._register_singleton(LibraryManager) self.library_datasets_manager = self._register_singleton(LibraryDatasetsManager) self.role_manager = self._register_singleton(RoleManager) - from galaxy.jobs.manager import JobManager self.job_manager = self._register_singleton(JobManager) # ConfiguredFileSources @@ -354,7 +577,6 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): self.job_manager._check_jobs_at_startup() self.proxy_manager = ProxyManager(self.config) - from galaxy.workflow import scheduling_manager # Must be initialized after job_config. self.workflow_scheduling_manager = scheduling_manager.WorkflowSchedulingManager(self) diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 49073dfd3fa..96c2a63023e 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -3,21 +3,18 @@ Universe configuration builder. """ # absolute_import needed for tool_shed package. -import collections import configparser -import errno import ipaddress +import locale import logging import logging.config import os import re -import signal import socket import string import sys import tempfile import threading -import time from datetime import timedelta from typing import ( Any, @@ -28,39 +25,28 @@ from typing import ( Optional, Set, SupportsInt, - TYPE_CHECKING, TypeVar, Union, ) import yaml -from beaker.cache import CacheManager -from beaker.util import parse_cache_config_options from galaxy.config.schema import AppSchema from galaxy.containers import parse_containers_config from galaxy.exceptions import ConfigurationError -from galaxy.model import mapping -from galaxy.model.database_utils import database_exists -from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database -from galaxy.schema.fields import BaseDatabaseIdField -from galaxy.structured_app import BasicSharedApp from galaxy.util import ( - ExecutionTimer, listify, string_as_bool, unicodify, ) from galaxy.util.custom_logging import LOGLV_TRACE -from galaxy.util.dbkeys import GenomeBuilds from galaxy.util.dynamic import HasDynamicProperties +from galaxy.util.facts import get_facts from galaxy.util.properties import ( find_config_file, read_properties_from_file, running_from_source, ) -from galaxy.web.formatting import expand_pretty_datetime_format -from galaxy.web_stack import get_stack_facts from ..version import VERSION_MAJOR, VERSION_MINOR try: @@ -69,15 +55,11 @@ except ImportError: # Python < 3.9 from importlib_resources import files # type: ignore[no-redef] -if TYPE_CHECKING: - from galaxy.jobs import JobConfiguration - from galaxy.tool_util.deps.containers import ContainerFinder - from galaxy.tools import ToolBox - from galaxy.tools.cache import ToolCache - from galaxy.tools.search import ToolBoxSearch - log = logging.getLogger(__name__) +DEFAULT_LOCALE_FORMAT = '%a %b %e %H:%M:%S %Y' +ISO_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" + GALAXY_APP_NAME = 'galaxy' GALAXY_SCHEMAS_PATH = files('galaxy.config') / 'schemas' GALAXY_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / 'config_schema.yml' @@ -138,10 +120,70 @@ LOGGING_CONFIG_DEFAULT: Dict[str, Any] = { """Default value for logging configuration, passed to :func:`logging.config.dictConfig`""" +def configure_logging(config, facts=None): + """Allow some basic logging configuration to be read from ini file. + + This should be able to consume either a galaxy.config.Configuration object + or a simple dictionary of configuration variables. + """ + facts = facts or get_facts(config=config) + # Get root logger + logging.addLevelName(LOGLV_TRACE, "TRACE") + # PasteScript will have already configured the logger if the + # 'loggers' section was found in the config file, otherwise we do + # some simple setup using the 'log_*' values from the config. + parser = getattr(config, "global_conf_parser", None) + if parser: + paste_configures_logging = config.global_conf_parser.has_section("loggers") + else: + paste_configures_logging = False + auto_configure_logging = not paste_configures_logging and string_as_bool(config.get("auto_configure_logging", "True")) + if auto_configure_logging: + logging_conf = config.get('logging', None) + if logging_conf is None: + # if using the default logging config, honor the log_level setting + logging_conf = LOGGING_CONFIG_DEFAULT + if config.get('log_level', 'DEBUG') != 'DEBUG': + logging_conf['handlers']['console']['level'] = config.get('log_level', 'DEBUG') + # configure logging with logging dict in config, template *FileHandler handler filenames with the `filename_template` option + for name, conf in logging_conf.get('handlers', {}).items(): + if conf['class'].startswith('logging.') and conf['class'].endswith('FileHandler') and 'filename_template' in conf: + conf['filename'] = conf.pop('filename_template').format(**facts) + logging_conf['handlers'][name] = conf + logging.config.dictConfig(logging_conf) + + def find_root(kwargs): return os.path.abspath(kwargs.get('root_dir', '.')) +def expand_pretty_datetime_format(value): + """ + + >>> expand_pretty_datetime_format("%H:%M:%S %Z") + '%H:%M:%S %Z' + >>> locale_format = expand_pretty_datetime_format("$locale (UTC)") + >>> import locale + >>> expected_format = '%s (UTC)' % locale.nl_langinfo(locale.D_T_FMT) + >>> locale_format == expected_format + True + >>> expand_pretty_datetime_format("$iso8601") + '%Y-%m-%d %H:%M:%S' + """ + locale_format = None + try: + locale_format = locale.nl_langinfo(locale.D_T_FMT) + except AttributeError: # nl_langinfo not available + pass + if not locale_format: + locale_format = DEFAULT_LOCALE_FORMAT + stock_formats = dict( + locale=locale_format, + iso8601=ISO_DATETIME_FORMAT, + ) + return string.Template(value).safe_substitute(**stock_formats) + + OptStr = TypeVar("OptStr", None, str) @@ -1182,273 +1224,3 @@ def get_database_engine_options(kwargs, model_prefix=''): value = conversions[key](value) rval[key] = value return rval - - -def get_database_url(config): - db_url = config.database_connection - return db_url - - -def init_models_from_config(config, map_install_models=False, object_store=None, trace_logger=None): - db_url = get_database_url(config) - model = mapping.init( - config.file_path, - db_url, - config.database_engine_options, - map_install_models=map_install_models, - database_query_profiling_proxy=config.database_query_profiling_proxy, - object_store=object_store, - trace_logger=trace_logger, - use_pbkdf2=config.get_bool('use_pbkdf2', True), - slow_query_log_threshold=config.slow_query_log_threshold, - thread_local_log=config.thread_local_log, - log_query_counts=config.database_log_query_counts, - ) - return model - - -def configure_logging(config): - """Allow some basic logging configuration to be read from ini file. - - This should be able to consume either a galaxy.config.Configuration object - or a simple dictionary of configuration variables. - """ - # Get root logger - logging.addLevelName(LOGLV_TRACE, "TRACE") - # PasteScript will have already configured the logger if the - # 'loggers' section was found in the config file, otherwise we do - # some simple setup using the 'log_*' values from the config. - parser = getattr(config, "global_conf_parser", None) - if parser: - paste_configures_logging = config.global_conf_parser.has_section("loggers") - else: - paste_configures_logging = False - auto_configure_logging = not paste_configures_logging and string_as_bool(config.get("auto_configure_logging", "True")) - if auto_configure_logging: - logging_conf = config.get('logging', None) - if logging_conf is None: - # if using the default logging config, honor the log_level setting - logging_conf = LOGGING_CONFIG_DEFAULT - if config.get('log_level', 'DEBUG') != 'DEBUG': - logging_conf['handlers']['console']['level'] = config.get('log_level', 'DEBUG') - # configure logging with logging dict in config, template *FileHandler handler filenames with the `filename_template` option - for name, conf in logging_conf.get('handlers', {}).items(): - if conf['class'].startswith('logging.') and conf['class'].endswith('FileHandler') and 'filename_template' in conf: - conf['filename'] = conf.pop('filename_template').format(**get_stack_facts(config=config)) - logging_conf['handlers'][name] = conf - logging.config.dictConfig(logging_conf) - - -class ConfiguresGalaxyMixin: - """Shared code for configuring Galaxy-like app objects.""" - - config: GalaxyAppConfiguration - tool_cache: "ToolCache" - job_config: "JobConfiguration" - toolbox: "ToolBox" - toolbox_search: "ToolBoxSearch" - container_finder: "ContainerFinder" - - def _configure_genome_builds(self, data_table_name="__dbkeys__", load_old_style=True): - self.genome_builds = GenomeBuilds(self, data_table_name=data_table_name, load_old_style=load_old_style) - - def wait_for_toolbox_reload(self, old_toolbox): - timer = ExecutionTimer() - log.debug('Waiting for toolbox reload') - # Wait till toolbox reload has been triggered (or more than 60 seconds have passed) - while timer.elapsed < 60: - if self.toolbox.has_reloaded(old_toolbox): - log.debug('Finished waiting for toolbox reload %s', timer) - break - time.sleep(0.1) - else: - log.warning('Waiting for toolbox reload timed out after 60 seconds') - - def _configure_tool_config_files(self): - if self.config.shed_tool_config_file not in self.config.tool_configs: - self.config.tool_configs.append(self.config.shed_tool_config_file) - # The value of migrated_tools_config is the file reserved for containing only those tools that have been - # eliminated from the distribution and moved to the tool shed. If migration checking is disabled, only add it if - # it exists (since this may be an existing deployment where migrations were previously run). - if (os.path.exists(self.config.migrated_tools_config) - and self.config.migrated_tools_config not in self.config.tool_configs): - self.config.tool_configs.append(self.config.migrated_tools_config) - - def _configure_toolbox(self): - from galaxy import tools - from galaxy.tools.biotools import get_galaxy_biotools_metadata_source - from galaxy.managers.citations import CitationsManager - from galaxy.tool_util.deps import containers - from galaxy.tool_util.deps.dependencies import AppInfo - import galaxy.tools.search - - if not isinstance(self, BasicSharedApp): - raise Exception("Must inherit from BasicSharedApp") - - self.citations_manager = CitationsManager(self) - self.biotools_metadata_source = get_galaxy_biotools_metadata_source(self.config) - - from galaxy.managers.tools import DynamicToolManager - - self.dynamic_tools_manager = DynamicToolManager(self) - self._toolbox_lock = threading.RLock() - self.toolbox = tools.ToolBox(self.config.tool_configs, self.config.tool_path, self) - galaxy_root_dir = os.path.abspath(self.config.root) - file_path = os.path.abspath(self.config.file_path) - app_info = AppInfo( - galaxy_root_dir=galaxy_root_dir, - default_file_path=file_path, - tool_data_path=self.config.tool_data_path, - shed_tool_data_path=self.config.shed_tool_data_path, - outputs_to_working_directory=self.config.outputs_to_working_directory, - container_image_cache_path=self.config.container_image_cache_path, - library_import_dir=self.config.library_import_dir, - enable_mulled_containers=self.config.enable_mulled_containers, - container_resolvers_config_file=self.config.container_resolvers_config_file, - container_resolvers_config_dict=self.config.container_resolvers, - involucro_path=self.config.involucro_path, - involucro_auto_init=self.config.involucro_auto_init, - mulled_channels=self.config.mulled_channels, - ) - mulled_resolution_cache = None - if self.config.mulled_resolution_cache_type: - cache_opts = { - 'cache.type': self.config.mulled_resolution_cache_type, - 'cache.data_dir': self.config.mulled_resolution_cache_data_dir, - 'cache.lock_dir': self.config.mulled_resolution_cache_lock_dir, - } - mulled_resolution_cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache('mulled_resolution') - self.container_finder = containers.ContainerFinder(app_info, mulled_resolution_cache=mulled_resolution_cache) - self._set_enabled_container_types() - index_help = getattr(self.config, "index_tool_help", True) - self.toolbox_search = galaxy.tools.search.ToolBoxSearch(self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help) - - def reindex_tool_search(self): - # Call this when tools are added or removed. - self.toolbox_search.build_index(tool_cache=self.tool_cache) - self.tool_cache.reset_status() - - def _set_enabled_container_types(self): - container_types_to_destinations = collections.defaultdict(list) - for destinations in self.job_config.destinations.values(): - for destination in destinations: - for enabled_container_type in self.container_finder._enabled_container_types(destination.params): - container_types_to_destinations[enabled_container_type].append(destination) - self.toolbox.dependency_manager.set_enabled_container_types(container_types_to_destinations) - self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.default_container_registry.resolver_classes) - self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.default_container_registry.container_resolvers) - - def _configure_tool_data_tables(self, from_shed_config): - from galaxy.tools.data import ToolDataTableManager - - # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path. - self.tool_data_tables = ToolDataTableManager(tool_data_path=self.config.tool_data_path, - config_filename=self.config.tool_data_table_config_path, - other_config_dict=self.config) - # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables. - try: - self.tool_data_tables.load_from_config_file(config_filename=self.config.shed_tool_data_table_config, - tool_data_path=self.tool_data_tables.tool_data_path, - from_shed_config=from_shed_config) - except OSError as exc: - # Missing shed_tool_data_table_config is okay if it's the default - if exc.errno != errno.ENOENT or self.config.is_set('shed_tool_data_table_config'): - raise - - def _configure_datatypes_registry(self, installed_repository_manager=None): - from galaxy.datatypes import registry - # Create an empty datatypes registry. - self.datatypes_registry = registry.Registry(self.config) - if installed_repository_manager and self.config.load_tool_shed_datatypes: - # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories. We - # load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some - # generic sniffers (eg text,xml) which catch anything, so it's impossible for proprietary sniffers to be used. - # However, if there is a conflict (2 datatypes with the same extension) between a proprietary datatype and a datatype - # in the Galaxy distribution, the datatype in the Galaxy distribution will take precedence. If there is a conflict - # between 2 proprietary datatypes, the datatype from the repository that was installed earliest will take precedence. - installed_repository_manager.load_proprietary_datatypes() - # Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config. - datatypes_configs = self.config.datatypes_config - for datatypes_config in listify(datatypes_configs): - # Setting override=False would make earlier files would take - # precedence - but then they wouldn't override tool shed - # datatypes. - self.datatypes_registry.load_datatypes(self.config.root, datatypes_config, override=True) - - def _configure_object_store(self, **kwds): - from galaxy.objectstore import build_object_store_from_config - self.object_store = build_object_store_from_config(self.config, **kwds) - - def _configure_security(self): - from galaxy.security import idencoding - self.security = idencoding.IdEncodingHelper(id_secret=self.config.id_secret) - BaseDatabaseIdField.security = self.security - - def _configure_tool_shed_registry(self): - import galaxy.tool_shed.tool_shed_registry - - # Set up the tool sheds registry - if os.path.isfile(self.config.tool_sheds_config_file): - self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry(self.config.tool_sheds_config_file) - else: - self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry() - - def _configure_models(self, check_migrate_databases=False, config_file=None): - """Preconditions: object_store must be set on self.""" - db_url = get_database_url(self.config) - install_db_url = self.config.install_database_connection - # TODO: Consider more aggressive check here that this is not the same - # database file under the hood. - combined_install_database = not(install_db_url and install_db_url != db_url) - install_db_url = install_db_url or db_url - install_database_options = self.config.database_engine_options if combined_install_database else self.config.install_database_engine_options - - if self.config.database_wait: - self._wait_for_database(db_url) - - if getattr(self.config, "max_metadata_value_size", None): - from galaxy.model import custom_types - custom_types.MAX_METADATA_VALUE_SIZE = self.config.max_metadata_value_size - - if check_migrate_databases: - # Initialize database / check for appropriate schema version. # If this - # is a new installation, we'll restrict the tool migration messaging. - from galaxy.model.migrate.check import create_or_verify_database - create_or_verify_database(db_url, config_file, self.config.database_engine_options, app=self, map_install_models=combined_install_database) - if not combined_install_database: - tsi_create_or_verify_database(install_db_url, install_database_options, app=self) - - self.model = init_models_from_config( - self.config, - map_install_models=combined_install_database, - object_store=self.object_store, - trace_logger=getattr(self, "trace_logger", None) - ) - if combined_install_database: - log.info("Install database targetting Galaxy's database configuration.") - self.install_model = self.model - else: - from galaxy.model.tool_shed_install import mapping as install_mapping - install_db_url = self.config.install_database_connection - log.info(f"Install database using its own connection {install_db_url}") - self.install_model = install_mapping.init(install_db_url, - install_database_options) - - def _configure_signal_handlers(self, handlers): - for sig, handler in handlers.items(): - signal.signal(sig, handler) - - def _wait_for_database(self, url): - attempts = self.config.database_wait_attempts - pause = self.config.database_wait_sleep - for i in range(1, attempts): - try: - database_exists(url) - break - except Exception: - log.info("Waiting for database: attempt %d of %d" % (i, attempts)) - time.sleep(pause) - - @property - def tool_dependency_dir(self): - return self.toolbox.dependency_manager.default_base_path diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index c41e6f6efdd..a17f3c06003 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -9,6 +9,7 @@ from threading import local from typing import Optional, Type from galaxy import model +from galaxy.config import GalaxyAppConfiguration from galaxy.model import mapper_registry from galaxy.model.base import SharedModelMapping from galaxy.model.migrate.triggers.update_audit_table import install as install_timestamp_triggers @@ -64,3 +65,20 @@ def init(file_path, url, engine_options=None, create_tables=False, map_install_m result.security_agent = GalaxyRBACAgent(result) result.thread_local_log = thread_local_log return result + + +def init_models_from_config(config: GalaxyAppConfiguration, map_install_models=False, object_store=None, trace_logger=None): + model = init( + config.file_path, + config.database_connection, + config.database_engine_options, + map_install_models=map_install_models, + database_query_profiling_proxy=config.database_query_profiling_proxy, + object_store=object_store, + trace_logger=trace_logger, + use_pbkdf2=config.get_bool('use_pbkdf2', True), + slow_query_log_threshold=config.slow_query_log_threshold, + thread_local_log=config.thread_local_log, + log_query_counts=config.database_log_query_counts, + ) + return model diff --git a/lib/galaxy/web/formatting.py b/lib/galaxy/web/formatting.py deleted file mode 100644 index d7b2151b566..00000000000 --- a/lib/galaxy/web/formatting.py +++ /dev/null @@ -1,32 +0,0 @@ -import locale -from string import Template - -DEFAULT_LOCALE_FORMAT = '%a %b %e %H:%M:%S %Y' -ISO_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" - - -def expand_pretty_datetime_format(value): - """ - - >>> expand_pretty_datetime_format("%H:%M:%S %Z") - '%H:%M:%S %Z' - >>> locale_format = expand_pretty_datetime_format("$locale (UTC)") - >>> import locale - >>> expected_format = '%s (UTC)' % locale.nl_langinfo(locale.D_T_FMT) - >>> locale_format == expected_format - True - >>> expand_pretty_datetime_format("$iso8601") - '%Y-%m-%d %H:%M:%S' - """ - locale_format = None - try: - locale_format = locale.nl_langinfo(locale.D_T_FMT) - except AttributeError: # nl_langinfo not available - pass - if not locale_format: - locale_format = DEFAULT_LOCALE_FORMAT - stock_formats = dict( - locale=locale_format, - iso8601=ISO_DATETIME_FORMAT, - ) - return Template(value).safe_substitute(**stock_formats) diff --git a/lib/galaxy/web_stack/__init__.py b/lib/galaxy/web_stack/__init__.py index 2310a8ff63b..66684aeef8c 100644 --- a/lib/galaxy/web_stack/__init__.py +++ b/lib/galaxy/web_stack/__init__.py @@ -633,10 +633,6 @@ def get_app_kwds(config_section, app_name=None): return application_stack_class().get_app_kwds(config_section, app_name=app_name) -def get_stack_facts(config=None): - return application_stack_instance(config=config).facts - - def _uwsgi_configured_mules(): mules = uwsgi.opt.get('mule', []) return [mules] if isinstance(mules, str) or mules is True else mules diff --git a/lib/tool_shed/webapp/config.py b/lib/tool_shed/webapp/config.py index 094186b431b..e738c86f4c5 100644 --- a/lib/tool_shed/webapp/config.py +++ b/lib/tool_shed/webapp/config.py @@ -12,11 +12,11 @@ from galaxy.config import ( CommonConfigurationMixin, get_database_engine_options, ) +from galaxy.config import expand_pretty_datetime_format from galaxy.config.schema import AppSchema from galaxy.exceptions import ConfigurationError from galaxy.util import string_as_bool from galaxy.version import VERSION, VERSION_MAJOR, VERSION_MINOR -from galaxy.web.formatting import expand_pretty_datetime_format try: from importlib.resources import files # type: ignore[attr-defined] diff --git a/packages/app/MANIFEST.in b/packages/app/MANIFEST.in index a4acb5c99e4..d47c145d0e9 100644 --- a/packages/app/MANIFEST.in +++ b/packages/app/MANIFEST.in @@ -1,5 +1,3 @@ include *.rst *.txt LICENSE -include galaxy/config/schemas/*.yml -include galaxy/config/sample/*.sample* include galaxy/jobs/runners/util/job_script/*.sh include galaxy/tools/*tsv diff --git a/packages/app/requirements.txt b/packages/app/requirements.txt index 4d77125b4d6..6e5279f1e2f 100644 --- a/packages/app/requirements.txt +++ b/packages/app/requirements.txt @@ -1,7 +1,9 @@ galaxy-auth +galaxy-config galaxy-data galaxy-job-execution galaxy-job-metrics +galaxy-objectstore galaxy-tool-util[cwl,edam] galaxy-web-framework galaxy-web-stack @@ -18,7 +20,6 @@ Markdown paramiko!=2.9.0,!=2.9.1 pulsar-galaxy-lib>=0.14.13 pydantic -pykwalify refgenconf>=0.12.0 sqlitedict svgwrite diff --git a/packages/app/setup.py b/packages/app/setup.py index b07f8886ee7..f92bc4fe976 100644 --- a/packages/app/setup.py +++ b/packages/app/setup.py @@ -34,7 +34,6 @@ PACKAGES = [ 'galaxy.actions', 'galaxy.app_unittest_utils', 'galaxy.authnz', - 'galaxy.config', 'galaxy.dependencies', 'galaxy.forms', 'galaxy.jobs', @@ -95,14 +94,9 @@ PACKAGES = [ ENTRY_POINTS = ''' [console_scripts] galaxy-main=galaxy.main:main - galaxy-config=galaxy.config.script:main ''' PACKAGE_DATA = { # Be sure to update MANIFEST.in for source dist. - 'galaxy': [ - 'config/schemas/*.yml', - 'config/sample/*', - ], 'tool_shed': [ 'scripts/bootstrap_tool_shed/user_info.xml', ], diff --git a/packages/app/test-requirements.txt b/packages/app/test-requirements.txt index 1fd0bf58c90..f93c7d6a95a 100644 --- a/packages/app/test-requirements.txt +++ b/packages/app/test-requirements.txt @@ -1,3 +1,4 @@ mock-ssh-server +pykwalify pytest testfixtures diff --git a/packages/config/HISTORY.rst b/packages/config/HISTORY.rst new file mode 100644 index 00000000000..3988a66c6d3 --- /dev/null +++ b/packages/config/HISTORY.rst @@ -0,0 +1,12 @@ +.. :changelog: + +History +------- + +.. to_doc + +--------------------- +22.1.0.dev0 +--------------------- + +* Initial release diff --git a/packages/config/LICENSE b/packages/config/LICENSE new file mode 120000 index 00000000000..1ef648f64b3 --- /dev/null +++ b/packages/config/LICENSE @@ -0,0 +1 @@ +../../LICENSE.txt \ No newline at end of file diff --git a/packages/config/MANIFEST.in b/packages/config/MANIFEST.in new file mode 100644 index 00000000000..ef1f736e7c8 --- /dev/null +++ b/packages/config/MANIFEST.in @@ -0,0 +1,3 @@ +include *.rst *.txt LICENSE +include galaxy/config/schemas/*.yml +include galaxy/config/sample/*.sample* diff --git a/packages/config/Makefile b/packages/config/Makefile new file mode 120000 index 00000000000..37af8bae5ba --- /dev/null +++ b/packages/config/Makefile @@ -0,0 +1 @@ +../package.Makefile \ No newline at end of file diff --git a/packages/config/README.rst b/packages/config/README.rst new file mode 100644 index 00000000000..fde16673c89 --- /dev/null +++ b/packages/config/README.rst @@ -0,0 +1,14 @@ + +.. image:: https://badge.fury.io/py/galaxy-config.svg + :target: https://pypi.org/project/galaxy-config/ + + +Overview +-------- + +The Galaxy_ config module. + +* Free software: Academic Free License version 3.0 +* Code: https://github.com/galaxyproject/galaxy + +.. _Galaxy: http://galaxyproject.org/ diff --git a/packages/config/dev-requirements.txt b/packages/config/dev-requirements.txt new file mode 120000 index 00000000000..467b90d7a23 --- /dev/null +++ b/packages/config/dev-requirements.txt @@ -0,0 +1 @@ +../package-dev-requirements.txt \ No newline at end of file diff --git a/packages/config/galaxy/__init__.py b/packages/config/galaxy/__init__.py new file mode 100644 index 00000000000..2e50d9cce89 --- /dev/null +++ b/packages/config/galaxy/__init__.py @@ -0,0 +1 @@ +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore[has-type] diff --git a/packages/app/galaxy/config b/packages/config/galaxy/config similarity index 100% rename from packages/app/galaxy/config rename to packages/config/galaxy/config diff --git a/packages/config/galaxy/project_galaxy_config.py b/packages/config/galaxy/project_galaxy_config.py new file mode 100644 index 00000000000..b7a5fd77095 --- /dev/null +++ b/packages/config/galaxy/project_galaxy_config.py @@ -0,0 +1,9 @@ +__version__ = "22.1.0rc1" + +PROJECT_NAME = "galaxy-config" +PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" +PROJECT_URL = "https://github.com/galaxyproject/galaxy" +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Configuration" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/config/requirements.txt b/packages/config/requirements.txt new file mode 100644 index 00000000000..a3caa0acfea --- /dev/null +++ b/packages/config/requirements.txt @@ -0,0 +1,5 @@ +galaxy-containers +boltons +pykwalify +PyYAML +requests diff --git a/packages/config/scripts b/packages/config/scripts new file mode 120000 index 00000000000..9aec9dc5a06 --- /dev/null +++ b/packages/config/scripts @@ -0,0 +1 @@ +../build_scripts \ No newline at end of file diff --git a/packages/config/setup.cfg b/packages/config/setup.cfg new file mode 120000 index 00000000000..eb7cf09393f --- /dev/null +++ b/packages/config/setup.cfg @@ -0,0 +1 @@ +../../setup.cfg \ No newline at end of file diff --git a/packages/config/setup.py b/packages/config/setup.py new file mode 100644 index 00000000000..46f35f268fe --- /dev/null +++ b/packages/config/setup.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python + +import ast +import os +import re + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +SOURCE_DIR = "galaxy" + +_version_re = re.compile(r"__version__\s+=\s+(.*)") + +project_short_name = os.path.basename(os.path.dirname(os.path.realpath(__file__))) +with open(f"{SOURCE_DIR}/project_galaxy_{project_short_name}.py", "rb") as f: + init_contents = f.read().decode("utf-8") + + def get_var(var_name): + pattern = re.compile(rf"{var_name}\s+=\s+(.*)") + match = pattern.search(init_contents).group(1) + return str(ast.literal_eval(match)) + + version = get_var("__version__") + PROJECT_NAME = get_var("PROJECT_NAME") + PROJECT_URL = get_var("PROJECT_URL") + PROJECT_AUTHOR = get_var("PROJECT_AUTHOR") + PROJECT_EMAIL = get_var("PROJECT_EMAIL") + PROJECT_DESCRIPTION = get_var("PROJECT_DESCRIPTION") + +TEST_DIR = "tests" +PACKAGES = [ + "galaxy", + "galaxy.config", +] +ENTRY_POINTS = """ + [console_scripts] + galaxy-config=galaxy.config.script:main +""" +PACKAGE_DATA = { + # Be sure to update MANIFEST.in for source dist. + "galaxy": [ + "config/schemas/*.yml", + "config/sample/*", + ], +} +PACKAGE_DIR = { + SOURCE_DIR: SOURCE_DIR, +} + +readme = open("README.rst").read() +history = open("HISTORY.rst").read().replace(".. :changelog:", "") + +if os.path.exists("requirements.txt"): + requirements = open("requirements.txt").read().split("\n") +else: + # In tox, it will cover them anyway. + requirements = [] + + +test_requirements = open("test-requirements.txt").read().split("\n") + + +setup( + name=PROJECT_NAME, + version=version, + description=PROJECT_DESCRIPTION, + long_description=readme + "\n\n" + history, + long_description_content_type="text/x-rst", + author=PROJECT_AUTHOR, + author_email=PROJECT_EMAIL, + url=PROJECT_URL, + packages=PACKAGES, + entry_points=ENTRY_POINTS, + package_data=PACKAGE_DATA, + package_dir=PACKAGE_DIR, + include_package_data=True, + install_requires=requirements, + license="AFL", + zip_safe=False, + keywords="galaxy", + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Environment :: Console", + "License :: OSI Approved :: Academic Free License (AFL)", + "Operating System :: POSIX", + "Topic :: Software Development", + "Topic :: Software Development :: Code Generators", + "Topic :: Software Development :: Testing", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + ], + test_suite=TEST_DIR, + tests_require=test_requirements, +) diff --git a/packages/config/test-data b/packages/config/test-data new file mode 120000 index 00000000000..e50575ea576 --- /dev/null +++ b/packages/config/test-data @@ -0,0 +1 @@ +../../test-data \ No newline at end of file diff --git a/packages/config/test-requirements.txt b/packages/config/test-requirements.txt new file mode 100644 index 00000000000..99440d27fbc --- /dev/null +++ b/packages/config/test-requirements.txt @@ -0,0 +1 @@ +pytest<6.1 diff --git a/test/unit/config/config_manage/__init__.py b/packages/config/tests/__init__.py similarity index 100% rename from test/unit/config/config_manage/__init__.py rename to packages/config/tests/__init__.py diff --git a/packages/config/tests/config b/packages/config/tests/config new file mode 120000 index 00000000000..329fc647500 --- /dev/null +++ b/packages/config/tests/config @@ -0,0 +1 @@ +../../../test/unit/config \ No newline at end of file diff --git a/packages/test.sh b/packages/test.sh index f323203da1b..2a9e022611b 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -23,6 +23,7 @@ PACKAGE_DIRS=( objectstore job_metrics containers + config files tool_util data diff --git a/packages/app/galaxy/version.py b/packages/util/galaxy/version.py similarity index 100% rename from packages/app/galaxy/version.py rename to packages/util/galaxy/version.py diff --git a/scripts/cleanup_datasets/cleanup_datasets.py b/scripts/cleanup_datasets/cleanup_datasets.py index ac858881d44..965fd44e2e2 100755 --- a/scripts/cleanup_datasets/cleanup_datasets.py +++ b/scripts/cleanup_datasets/cleanup_datasets.py @@ -18,6 +18,7 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa import galaxy.config from galaxy.datatypes.registry import Registry from galaxy.exceptions import ObjectNotFound +from galaxy.model.mapping import init_models_from_config from galaxy.objectstore import build_object_store_from_config from galaxy.util import unicodify from galaxy.util.script import app_properties_from_args, populate_config_args @@ -519,7 +520,7 @@ class CleanupDatasetsApplication: def __init__(self, config): self.object_store = build_object_store_from_config(config) # Setup the database engine and ORM - self.model = galaxy.config.init_models_from_config(config, object_store=self.object_store) + self.model = init_models_from_config(config, object_store=self.object_store) registry = Registry() registry.load_datatypes() galaxy.model.set_datatypes_registry(registry) diff --git a/scripts/cleanup_datasets/pgcleanup.py b/scripts/cleanup_datasets/pgcleanup.py index b20ca097a46..48721ac3202 100755 --- a/scripts/cleanup_datasets/pgcleanup.py +++ b/scripts/cleanup_datasets/pgcleanup.py @@ -1112,7 +1112,7 @@ class Cleanup: @property def conn(self): if self.__conn is None: - url = make_url(galaxy.config.get_database_url(self.config)) + url = make_url(self.config.database_connection) log.info(f'Connecting to database with URL: {url}') args = url.translate_connect_args(username='user') args.update(url.query) diff --git a/scripts/cleanup_datasets/populate_uuid.py b/scripts/cleanup_datasets/populate_uuid.py index da262653a41..b5f2ea624c5 100755 --- a/scripts/cleanup_datasets/populate_uuid.py +++ b/scripts/cleanup_datasets/populate_uuid.py @@ -15,6 +15,7 @@ import uuid sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) import galaxy.config +from galaxy.model.mapping import init_models_from_config from galaxy.util.script import app_properties_from_args, populate_config_args DESCRIPTION = """ @@ -32,7 +33,7 @@ def main(): app_properties = app_properties_from_args(args) config = galaxy.config.Configuration(**app_properties) - model = galaxy.config.init_models_from_config(config) + model = init_models_from_config(config) for row in model.context.query(model.Dataset): if row.uuid is None: diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 3b586d2da07..b1b12ffec9c 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -18,6 +18,7 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa import galaxy import galaxy.app import galaxy.config +from galaxy.model.mapping import init_models_from_config from galaxy.objectstore import build_object_store_from_config from galaxy.util import ( hash_util, @@ -36,7 +37,7 @@ def _init(args): if not config.database_connection: logging.warning("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml") - model = galaxy.config.init_models_from_config(config, object_store=object_store) + model = init_models_from_config(config, object_store=object_store) return ( model, object_store, diff --git a/scripts/helper.py b/scripts/helper.py index 97b3e394f1d..e52c11ba9ed 100644 --- a/scripts/helper.py +++ b/scripts/helper.py @@ -12,6 +12,7 @@ import sys sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, 'lib')) import galaxy.config +from galaxy.model.mapping import init_models_from_config from galaxy.security import idencoding from galaxy.util.script import app_properties_from_args, populate_config_args @@ -26,7 +27,7 @@ args = parser.parse_args() app_properties = app_properties_from_args(args) config = galaxy.config.Configuration(**app_properties) helper = idencoding.IdEncodingHelper(id_secret=app_properties.get('id_secret')) -model = galaxy.config.init_models_from_config(config) +model = init_models_from_config(config) if args.encode_id: print(f'Encoded "{args.encode_id}": {helper.encode_id(args.encode_id)}') diff --git a/scripts/pages_identifier_conversion.py b/scripts/pages_identifier_conversion.py index b9cec401071..d91c252aac0 100644 --- a/scripts/pages_identifier_conversion.py +++ b/scripts/pages_identifier_conversion.py @@ -10,6 +10,7 @@ import galaxy import galaxy.app import galaxy.config from galaxy.managers.pages import PageContentProcessor, placeholderRenderForSave +from galaxy.model.mapping import init_models_from_config from galaxy.objectstore import build_object_store_from_config from galaxy.security.idencoding import IdEncodingHelper from galaxy.util import unicodify @@ -31,7 +32,7 @@ def main(argv): if not config.database_connection: print("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml") - model = galaxy.config.init_models_from_config(config, object_store=object_store) + model = init_models_from_config(config, object_store=object_store) session = model.context.current pagerevs = session.query(model.PageRevision).all() mock_trans = Bunch(app=Bunch(security=security_helper), model=model, user_is_admin=lambda: True, sa_session=session) diff --git a/scripts/runtime_stats.py b/scripts/runtime_stats.py index 4e358d278dd..f4206874b16 100755 --- a/scripts/runtime_stats.py +++ b/scripts/runtime_stats.py @@ -117,7 +117,7 @@ def parse_arguments(): app_properties = app_properties_from_args(args) config = galaxy.config.Configuration(**app_properties) - uri = args.config.get_database_url(config) + uri = config.database_connection names = {'database': 'dbname', 'username': 'user'} args.connect_args = url.make_url(uri).translate_connect_args(**names) diff --git a/scripts/set_dataset_sizes.py b/scripts/set_dataset_sizes.py index d3a6e355478..c980d755a8f 100644 --- a/scripts/set_dataset_sizes.py +++ b/scripts/set_dataset_sizes.py @@ -7,6 +7,7 @@ import sys sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) import galaxy.config +from galaxy.model.mapping import init_models_from_config from galaxy.objectstore import build_object_store_from_config from galaxy.util.script import app_properties_from_args, populate_config_args @@ -20,7 +21,7 @@ def init(): config = galaxy.config.Configuration(**app_properties) object_store = build_object_store_from_config(config) - model = galaxy.config.init_models_from_config(config, object_store=object_store) + model = init_models_from_config(config, object_store=object_store) return model, object_store diff --git a/scripts/set_user_disk_usage.py b/scripts/set_user_disk_usage.py index bfd1f3d394e..ca8b936a2ee 100755 --- a/scripts/set_user_disk_usage.py +++ b/scripts/set_user_disk_usage.py @@ -7,6 +7,7 @@ import sys sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) import galaxy.config +from galaxy.model.mapping import init_models_from_config from galaxy.objectstore import build_object_store_from_config from galaxy.util import nice_size from galaxy.util.script import app_properties_from_args, populate_config_args @@ -31,8 +32,8 @@ def init(): app_properties = app_properties_from_args(args) config = galaxy.config.Configuration(**app_properties) object_store = build_object_store_from_config(config) - engine = galaxy.config.get_database_url(config).split(":")[0] - return galaxy.config.init_models_from_config(config, object_store=object_store), object_store, engine + engine = config.database_connection.split(":")[0] + return init_models_from_config(config, object_store=object_store), object_store, engine def quotacheck(sa_session, users, engine): diff --git a/test/unit/config/test_config_dataset_storage.py b/test/unit/config/test_config_dataset_storage.py index 8c455c83126..428e4968f0d 100644 --- a/test/unit/config/test_config_dataset_storage.py +++ b/test/unit/config/test_config_dataset_storage.py @@ -1,54 +1,44 @@ import os.path -import pytest - from galaxy import config from galaxy.config import BaseAppConfiguration -@pytest.fixture -def mock_config_file(monkeypatch): - # Patch this; otherwise tempfile.tempdir will be set, which is a global variable that - # defines the value of the default `dir` argument to the functions in Python's - # tempfile module - which breaks multiple tests. - monkeypatch.setattr(config.GalaxyAppConfiguration, '_override_tempdir', lambda a, b: None) - - -def test_object_store_store_by_set(mock_config_file, monkeypatch): +def test_object_store_store_by_set(): # object_store_store_by set by admin - appconfig = config.GalaxyAppConfiguration(object_store_store_by='id') + appconfig = config.GalaxyAppConfiguration(object_store_store_by='id', override_tempdir=False) assert appconfig.object_store_store_by == 'id' -def test_uuid_1(mock_config_file, monkeypatch): +def test_uuid_1(): # object_store_store_by not set # file_path set by admin to `objects` (no need for the dir to exist) - appconfig = config.GalaxyAppConfiguration(file_path='objects') + appconfig = config.GalaxyAppConfiguration(file_path='objects', override_tempdir=False) assert appconfig.object_store_store_by == 'uuid' -def test_uuid_2(mock_config_file, monkeypatch): +def test_uuid_2(monkeypatch): # object_store_store_by not set # file_path not set, `files` dir doesn't exist monkeypatch.setattr(BaseAppConfiguration, '_path_exists', lambda self, path: False) - appconfig = config.GalaxyAppConfiguration() + appconfig = config.GalaxyAppConfiguration(override_tempdir=False) assert appconfig.object_store_store_by == 'uuid' -def test_id_1(mock_config_file, monkeypatch): +def test_id_1(): # object_store_store_by not set # file_path set by admin to `not_objects` (no need for the dir to exist) - appconfig = config.GalaxyAppConfiguration(file_path='not_objects') + appconfig = config.GalaxyAppConfiguration(file_path='not_objects', override_tempdir=False) assert appconfig.object_store_store_by == 'id' -def test_id_2(mock_config_file, monkeypatch): +def test_id_2(monkeypatch): # object_store_store_by not set # file_path not set, `files` dir exists monkeypatch.setattr(BaseAppConfiguration, '_path_exists', lambda self, path: True if os.path.basename(path) == 'files' else False) - appconfig = config.GalaxyAppConfiguration() + appconfig = config.GalaxyAppConfiguration(override_tempdir=False) assert appconfig.object_store_store_by == 'id' diff --git a/test/unit/data/model/mapping/test_model_mapping.py b/test/unit/data/model/mapping/test_model_mapping.py index bbe66d80d1c..2ffb44e040b 100644 --- a/test/unit/data/model/mapping/test_model_mapping.py +++ b/test/unit/data/model/mapping/test_model_mapping.py @@ -60,13 +60,14 @@ class TestPlanet(BaseTest): # BaseTest is a base class; we need it to get the t See other model tests in this module for examples of more complex setups. """ -from datetime import datetime, timedelta +from datetime import timedelta from uuid import UUID, uuid4 import pytest from sqlalchemy import func, select from galaxy import model +from galaxy.model.orm.now import now from .common import ( AbstractBaseTest, collection_consists_of_objects, @@ -92,7 +93,7 @@ class TestAPIKeys(BaseTest): assert cls_.__tablename__ == "api_keys" def test_columns(self, session, cls_, user): - create_time, user_id, key = datetime.now(), user.id, get_unique_value() + create_time, user_id, key = now(), user.id, get_unique_value() obj = cls_(user_id=user_id, key=key, create_time=create_time) with dbcleanup(session, obj) as obj_id: @@ -115,7 +116,7 @@ class TestCleanupEvent(BaseTest): assert cls_.__tablename__ == "cleanup_event" def test_columns(self, session, cls_): - create_time, message = datetime.now(), "a" + create_time, message = now(), "a" obj = cls_(create_time=create_time, message=message) with dbcleanup(session, obj) as obj_id: @@ -130,7 +131,7 @@ class TestCleanupEventDatasetAssociation(BaseTest): assert cls_.__tablename__ == "cleanup_event_dataset_association" def test_columns(self, session, cls_, cleanup_event, dataset): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -150,7 +151,7 @@ class TestCleanupEventHistoryAssociation(BaseTest): assert cls_.__tablename__ == "cleanup_event_history_association" def test_columns(self, session, cls_, cleanup_event, history): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -170,7 +171,7 @@ class TestCleanupEventHistoryDatasetAssociationAssociation(BaseTest): assert cls_.__tablename__ == "cleanup_event_hda_association" def test_columns(self, session, cls_, cleanup_event, history_dataset_association): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -192,7 +193,7 @@ class TestCleanupEventImplicitlyConvertedDatasetAssociationAssociation(BaseTest) def test_columns( self, session, cls_, cleanup_event, implicitly_converted_dataset_association ): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -212,7 +213,7 @@ class TestCleanupEventLibraryAssociation(BaseTest): assert cls_.__tablename__ == "cleanup_event_library_association" def test_columns(self, session, cls_, cleanup_event, library): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -232,7 +233,7 @@ class TestCleanupEventLibraryDatasetAssociation(BaseTest): assert cls_.__tablename__ == "cleanup_event_library_dataset_association" def test_columns(self, session, cls_, cleanup_event, library_dataset): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -254,7 +255,7 @@ class TestCleanupEventLibraryDatasetDatasetAssociationAssociation(BaseTest): def test_columns( self, session, cls_, cleanup_event, library_dataset_dataset_association ): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -274,7 +275,7 @@ class TestCleanupEventLibraryFolderAssociation(BaseTest): assert cls_.__tablename__ == "cleanup_event_library_folder_association" def test_columns(self, session, cls_, cleanup_event, library_folder): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -294,7 +295,7 @@ class TestCleanupEventMetadataFileAssociation(BaseTest): assert cls_.__tablename__ == "cleanup_event_metadata_file_association" def test_columns(self, session, cls_, cleanup_event, metadata_file): - create_time = datetime.now() + create_time = now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, @@ -322,7 +323,7 @@ class TestCustosAuthnzToken(BaseTest): access_token = "c" id_token = "d" refresh_token = "e" - expiration_time = datetime.now() + expiration_time = now() refresh_expiration_time = expiration_time + timedelta(hours=1) obj = cls_() @@ -391,7 +392,7 @@ class TestDataManagerHistoryAssociation(BaseTest): assert cls_.__tablename__ == "data_manager_history_association" def test_columns(self, session, cls_, history, user): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.create_time = create_time @@ -424,7 +425,7 @@ class TestDataManagerJobAssociation(BaseTest): assert has_index(cls_.__table__, ("data_manager_id",)) def test_columns(self, session, cls_, job): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) data_manager_id = "a" obj = cls_() @@ -457,7 +458,7 @@ class TestDataset(BaseTest): assert cls_.table.name == "dataset" def test_columns(self, session, cls_, job): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) state = "a" deleted = True @@ -561,7 +562,7 @@ class TestDatasetCollection(BaseTest): populated_state = "b" populated_state_message = "c" element_count = 1 - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() @@ -704,7 +705,7 @@ class TestDatasetPermissions(BaseTest): def test_columns(self, session, cls_, dataset, role): action = "a" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(action, dataset, role) obj.create_time = create_time @@ -817,7 +818,7 @@ class TestDefaultQuotaAssociation(BaseTest): def test_columns(self, session, cls_, quota): type_ = cls_.types.REGISTERED - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(type_, quota) obj.create_time = create_time @@ -877,7 +878,7 @@ class TestDynamicTool(BaseTest): active = True hidden = True value = "f" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() @@ -930,7 +931,7 @@ class TestEvent(BaseTest): def test_columns(self, session, cls_, history, galaxy_session, user): message, tool_id = "a", "b" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.message = message @@ -1018,7 +1019,7 @@ class TestFormDefinition(BaseTest): def test_columns(self, session, cls_, form_definition_current): name, desc, fields, type, layout = "a", "b", "c", "d", "e" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.name = name @@ -1055,7 +1056,7 @@ class TestFormDefinitionCurrent(BaseTest): assert cls_.__tablename__ == "form_definition_current" def test_columns(self, session, cls_, form_definition): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) deleted = True obj = cls_() @@ -1088,7 +1089,7 @@ class TestFormValues(BaseTest): def test_columns(self, session, cls_, form_definition): content = "a" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.form_definition = form_definition @@ -1118,7 +1119,7 @@ class TestGalaxySession(BaseTest): def test_columns(self, session, cls_, user, history, galaxy_session): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) remote_host = "a" remote_addr = "b" @@ -1179,7 +1180,7 @@ class TestGalaxySessionToHistoryAssociation(BaseTest): assert cls_.__tablename__ == "galaxy_session_to_history" def test_columns(self, session, cls_, galaxy_session, history): - create_time = datetime.now() + create_time = now() obj = cls_(galaxy_session, history) obj.create_time = create_time @@ -1205,7 +1206,7 @@ class TestGenomeIndexToolData(BaseTest): def test_columns(self, session, cls_, job, dataset, user): fasta_path = "a" - created_time = datetime.now() + created_time = now() modified_time = created_time + timedelta(hours=1) indexer = "b" obj = cls_() @@ -1246,7 +1247,7 @@ class TestGroup(BaseTest): assert cls_.__tablename__ == "galaxy_group" def test_columns(self, session, cls_): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) name = get_unique_value() deleted = True @@ -1297,7 +1298,7 @@ class TestGroupQuotaAssociation(BaseTest): assert cls_.__tablename__ == "group_quota_association" def test_columns(self, session, cls_, group, quota): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(group, quota) obj.create_time = create_time @@ -1326,7 +1327,7 @@ class TestGroupRoleAssociation(BaseTest): def test_columns(self, session, cls_, group, role): obj = cls_(group, role) - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj.create_time = create_time obj.update_time = update_time @@ -1354,7 +1355,7 @@ class TestHistory(BaseTest): assert has_index(cls_.__table__, ("slug",)) def test_columns(self, session, cls_, user): - create_time = datetime.now() + create_time = now() name = "a" hid_counter = 2 deleted = True @@ -1529,7 +1530,7 @@ class TestHistoryDatasetAssociation(BaseTest): copied_from_hda = history_dataset_association_factory() parent = history_dataset_association_factory() - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) state = "a" name = "b" @@ -1760,7 +1761,7 @@ class TestHistoryDatasetAssociationDisplayAtAuthorization(BaseTest): ) def test_columns(self, session, cls_, history_dataset_association, user): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) site = "a" obj = cls_(history_dataset_association, user, site) @@ -1798,7 +1799,7 @@ class TestHistoryDatasetAssociationHistory(BaseTest): ): name, update_time, version, extension, metadata = ( "a", - datetime.now(), + now(), 2, "b", {"key": "value"}, @@ -1965,7 +1966,7 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): visible = True deleted = True implicit_output_name = "b" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() @@ -2339,7 +2340,7 @@ class TestImplicitlyConvertedDatasetAssociation(BaseTest): history_dataset_association, library_dataset_dataset_association, ): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) deleted = True metadata_safe = False @@ -2453,7 +2454,7 @@ class TestInteractiveToolEntryPoint(BaseTest): info = "f" configured = True deleted = True - created_time = datetime.now() + created_time = now() modified_time = created_time + timedelta(hours=1) obj = cls_() @@ -2513,7 +2514,7 @@ class TestJob(BaseTest): galaxy_session, user, ): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) tool_id = "a" tool_version = "b" @@ -2756,7 +2757,7 @@ class TestJobContainerAssociation(BaseTest): container_type = "a" container_name = "b" container_info = "c" - created_time = datetime.now() + created_time = now() modified_time = created_time + timedelta(hours=1) session.add(job) # must be bound to a session for lazy load of attributes @@ -2988,7 +2989,7 @@ class TestJobStateHistory(BaseTest): def test_columns(self, session, cls_, job): state, info = job.state, job.info - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(job) obj.create_time = create_time @@ -3246,7 +3247,7 @@ class TestLibrary(BaseTest): def test_columns(self, session, cls_, library_folder): name, deleted, purged, description, synopsis = "a", True, True, "b", "c" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(name, description, synopsis, library_folder) obj.create_time = create_time @@ -3290,7 +3291,7 @@ class TestLibraryDataset(BaseTest): def test_columns(self, session, cls_, library_folder): order_id = 9 - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) name = "a" info = "b" @@ -3533,7 +3534,7 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): extended_metadata, user, ): - create_time = datetime.now() + create_time = now() state = "a" name = "b" info = "c" @@ -3715,7 +3716,7 @@ class TestLibraryDatasetDatasetAssociationPermissions(BaseTest): def test_columns(self, session, cls_, library_dataset_dataset_association, role): action = "a" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(action, library_dataset_dataset_association, role) obj.create_time = create_time @@ -3843,7 +3844,7 @@ class TestLibraryDatasetPermissions(BaseTest): def test_columns(self, session, cls_, library_dataset, role): action = "a" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(action, library_dataset, role) obj.create_time = create_time @@ -3874,7 +3875,7 @@ class TestLibraryFolder(BaseTest): def test_columns(self, session, cls_, library_folder): parent = library_folder - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) name = "a" description = "b" @@ -3984,7 +3985,7 @@ class TestLibraryFolderPermissions(BaseTest): def test_columns(self, session, cls_, library_folder, role): action = "a" obj = cls_(action, library_folder, role) - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj.create_time = create_time obj.update_time = update_time @@ -4041,7 +4042,7 @@ class TestLibraryPermissions(BaseTest): def test_columns(self, session, cls_, library, role): action = "a" obj = cls_(action, library, role) - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(action, library, role) obj.create_time = create_time @@ -4077,7 +4078,7 @@ class TestMetadataFile(BaseTest): library_dataset_dataset_association, ): name = "a" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) object_store_id = "b" uuid = uuid4() @@ -4215,7 +4216,7 @@ class TestPage(BaseTest): def test_columns(self, session, cls_, user, page_revision): title, deleted, importable, slug, published = "a", True, True, "b", True - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.user = user @@ -4345,7 +4346,7 @@ class TestPageRevision(BaseTest): def test_columns(self, session, cls_, page): title, content = "a", "b" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.page = page @@ -4438,7 +4439,7 @@ class TestPasswordResetToken(BaseTest): def test_columns_and_relationships(self, session, cls_, user): token = get_unique_value() - expiration_time = datetime.now() + expiration_time = now() obj = cls_(user, token) obj.expiration_time = expiration_time @@ -4508,7 +4509,7 @@ class TestQuota(BaseTest): def test_columns(self, session, cls_): name, description, amount, operation = get_unique_value(), "b", 42, "+" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(name, description, amount, operation) obj.create_time = create_time @@ -4567,7 +4568,7 @@ class TestRole(BaseTest): cls_.types.SYSTEM, True, ) - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(name, description, type_, deleted) obj.create_time = create_time @@ -4616,7 +4617,7 @@ class TestStoredWorkflow(BaseTest): assert has_index(cls_.__table__, ("slug",)) def test_columns(self, session, cls_, workflow, user): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) name = "a" deleted = True @@ -4905,7 +4906,7 @@ class TestTask(BaseTest): assert cls_.__tablename__ == "task" def test_columns(self, session, cls_, job): - create_time = datetime.now() + create_time = now() execution_time = create_time + timedelta(hours=1) update_time = execution_time + timedelta(hours=1) state = "p" @@ -5051,7 +5052,7 @@ class TestUser(BaseTest): assert cls_.__tablename__ == "galaxy_user" def test_columns(self, session, cls_, form_values): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) email = get_unique_value() username = get_unique_value() @@ -5225,7 +5226,7 @@ class TestUserAction(BaseTest): def test_columns(self, session, cls_, user, galaxy_session): action, params, context = "a", "b", "c" - create_time = datetime.now() + create_time = now() obj = cls_() obj.user = user obj.session_id = galaxy_session.id @@ -5271,7 +5272,7 @@ class TestUserAddress(BaseTest): deleted, purged, ) = ("a", "b", "c", "d", "e", "f", "g", "h", "i", True, False) - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.user = user @@ -5349,7 +5350,7 @@ class TestUserGroupAssociation(BaseTest): assert cls_.__tablename__ == "user_group_association" def test_columns(self, session, cls_, user, group): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(user, group) obj.create_time = create_time @@ -5396,7 +5397,7 @@ class TestUserQuotaAssociation(BaseTest): assert cls_.__tablename__ == "user_quota_association" def test_columns(self, session, cls_, user, quota): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(user, quota) obj.create_time = create_time @@ -5424,7 +5425,7 @@ class TestUserRoleAssociation(BaseTest): assert cls_.__tablename__ == "user_role_association" def test_columns(self, session, cls_, user, role): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_(user, role) obj.create_time = create_time @@ -5452,7 +5453,7 @@ class TestVault(BaseTest): assert cls_.__tablename__ == "vault" def test_columns(self, session, cls_): - create_time = update_time = datetime.now() + create_time = update_time = now() key = '/some/path' parent_key = '/some' value = 'helloworld' @@ -5474,7 +5475,7 @@ class TestVisualization(BaseTest): assert has_index(cls_.__table__, ("slug",)) def test_columns(self, session, cls_, user, visualization_revision): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) title = "a" type = "b" @@ -5627,7 +5628,7 @@ class TestVisualizationRevision(BaseTest): def test_columns(self, session, cls_, visualization): visualization, title, dbkey, config = visualization, "a", "b", "c" - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) obj = cls_() obj.visualization = visualization @@ -5720,7 +5721,7 @@ class TestWorkerProcess(BaseTest): def test_columns(self, session, cls_): server_name, hostname = get_unique_value(), "a" - update_time = datetime.now() + update_time = now() obj = cls_() obj.server_name = server_name obj.hostname = hostname @@ -5740,7 +5741,7 @@ class TestWorkflow(BaseTest): assert cls_.__tablename__ == "workflow" def test_columns(self, session, cls_, stored_workflow, workflow): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) name = "a" has_cycles = True @@ -5805,7 +5806,7 @@ class TestWorkflowInvocation(BaseTest): assert cls_.__tablename__ == "workflow_invocation" def test_columns(self, session, cls_, workflow, history): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) state = "a" scheduler = "b" @@ -6095,7 +6096,7 @@ class TestWorkflowInvocationStep(BaseTest): job, implicit_collection_jobs, ): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) state, action = "a", "b" @@ -6581,7 +6582,7 @@ class TestWorkflowStep(BaseTest): assert cls_.__tablename__ == "workflow_step" def test_columns(self, session, cls_, workflow, dynamic_tool, workflow_factory): - create_time = datetime.now() + create_time = now() update_time = create_time + timedelta(hours=1) type = "a" tool_id = "b" diff --git a/test/unit/config/config_manage/1607_root_filters/config/galaxy.ini b/test/unit/webapps/config_manage/1607_root_filters/config/galaxy.ini similarity index 100% rename from test/unit/config/config_manage/1607_root_filters/config/galaxy.ini rename to test/unit/webapps/config_manage/1607_root_filters/config/galaxy.ini diff --git a/test/unit/config/config_manage/1607_root_filters/config/tool_shed.ini b/test/unit/webapps/config_manage/1607_root_filters/config/tool_shed.ini similarity index 100% rename from test/unit/config/config_manage/1607_root_filters/config/tool_shed.ini rename to test/unit/webapps/config_manage/1607_root_filters/config/tool_shed.ini diff --git a/test/unit/config/config_manage/1607_root_samples/config/galaxy.ini b/test/unit/webapps/config_manage/1607_root_samples/config/galaxy.ini similarity index 100% rename from test/unit/config/config_manage/1607_root_samples/config/galaxy.ini rename to test/unit/webapps/config_manage/1607_root_samples/config/galaxy.ini diff --git a/test/unit/config/config_manage/1607_root_samples/config/reports.ini b/test/unit/webapps/config_manage/1607_root_samples/config/reports.ini similarity index 100% rename from test/unit/config/config_manage/1607_root_samples/config/reports.ini rename to test/unit/webapps/config_manage/1607_root_samples/config/reports.ini diff --git a/test/unit/config/config_manage/1607_root_samples/config/tool_shed.ini b/test/unit/webapps/config_manage/1607_root_samples/config/tool_shed.ini similarity index 100% rename from test/unit/config/config_manage/1607_root_samples/config/tool_shed.ini rename to test/unit/webapps/config_manage/1607_root_samples/config/tool_shed.ini diff --git a/test/unit/config/config_manage/1705_allow_path_paste/config/galaxy.ini b/test/unit/webapps/config_manage/1705_allow_path_paste/config/galaxy.ini similarity index 100% rename from test/unit/config/config_manage/1705_allow_path_paste/config/galaxy.ini rename to test/unit/webapps/config_manage/1705_allow_path_paste/config/galaxy.ini diff --git a/test/unit/webapps/config_manage/__init__.py b/test/unit/webapps/config_manage/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/unit/config/config_manage/embedded/config/galaxy.yml b/test/unit/webapps/config_manage/embedded/config/galaxy.yml similarity index 100% rename from test/unit/config/config_manage/embedded/config/galaxy.yml rename to test/unit/webapps/config_manage/embedded/config/galaxy.yml diff --git a/test/unit/config/config_manage/simple/config/galaxy.yml b/test/unit/webapps/config_manage/simple/config/galaxy.yml similarity index 100% rename from test/unit/config/config_manage/simple/config/galaxy.yml rename to test/unit/webapps/config_manage/simple/config/galaxy.yml diff --git a/test/unit/config/config_manage/test_config_manage.py b/test/unit/webapps/config_manage/test_config_manage.py similarity index 100% rename from test/unit/config/config_manage/test_config_manage.py rename to test/unit/webapps/config_manage/test_config_manage.py diff --git a/test/unit/config/test_config_values.py b/test/unit/webapps/test_config_values.py similarity index 94% rename from test/unit/config/test_config_values.py rename to test/unit/webapps/test_config_values.py index 6a4306be3c9..661752c9837 100644 --- a/test/unit/config/test_config_values.py +++ b/test/unit/webapps/test_config_values.py @@ -5,23 +5,19 @@ from datetime import timedelta import pytest from galaxy import config +from galaxy.config import expand_pretty_datetime_format from galaxy.util import listify -from galaxy.web.formatting import expand_pretty_datetime_format TestData = namedtuple('TestData', ('key', 'expected', 'loaded')) @pytest.fixture(scope='module') def appconfig(): - return config.GalaxyAppConfiguration() + return config.GalaxyAppConfiguration(override_tempdir=False) @pytest.fixture def mock_config_file(monkeypatch): - # Patch this; otherwise tempfile.tempdir will be set, which is a global variable that - # defines the value of the default `dir` argument to the functions in Python's - # tempfile module - which breaks multiple tests. - monkeypatch.setattr(config.GalaxyAppConfiguration, '_override_tempdir', lambda a, b: None) # Set this to return None to force the creation of base config directories # in _set_config_directories(). Used to test the values of these directories only. monkeypatch.setattr(config, 'find_config_file', lambda x: None) @@ -39,7 +35,7 @@ def test_common_base_config(appconfig): def test_base_config_if_running_from_source(monkeypatch, mock_config_file): # Simulated condition: running from source, config_file is None. monkeypatch.setattr(config, 'running_from_source', True) - appconfig = config.GalaxyAppConfiguration() + appconfig = config.GalaxyAppConfiguration(override_tempdir=False) assert not appconfig.config_file assert appconfig.config_dir == os.path.join(appconfig.root, 'config') assert appconfig.data_dir == os.path.join(appconfig.root, 'database') @@ -49,7 +45,7 @@ def test_base_config_if_running_from_source(monkeypatch, mock_config_file): def test_base_config_if_running_not_from_source(monkeypatch, mock_config_file): # Simulated condition: running not from source, config_file is None. monkeypatch.setattr(config, 'running_from_source', False) - appconfig = config.GalaxyAppConfiguration() + appconfig = config.GalaxyAppConfiguration(override_tempdir=False) assert not appconfig.config_file assert appconfig.config_dir == os.getcwd() assert appconfig.data_dir == os.path.join(appconfig.config_dir, 'data') @@ -226,8 +222,7 @@ class ExpectedValues: def get_config_data(): - config.GalaxyAppConfiguration._override_tempdir = lambda a, b: None # method must be mocked - configuration = config.GalaxyAppConfiguration() + configuration = config.GalaxyAppConfiguration(override_tempdir=False) ev = ExpectedValues(configuration) items = ((k, v) for k, v in configuration.schema.app_schema.items()) for key, data in items: