diff --git a/.ci/flake8_wrapper.sh b/.ci/flake8_wrapper.sh index b6fb1a2e0f6..522a7b2abb2 100755 --- a/.ci/flake8_wrapper.sh +++ b/.ci/flake8_wrapper.sh @@ -5,4 +5,4 @@ set -e flake8 --exclude $(paste -sd, .ci/flake8_ignorelist.txt) . # Apply stricter rules for the directories shared with Pulsar -flake8 --ignore=D --max-line-length=150 lib/galaxy/jobs/runners/util/ +flake8 --ignore=E203,D --max-line-length=150 lib/galaxy/jobs/runners/util/ diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 9dfd554cf3c..2e8b61d85b1 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -43,3 +43,4 @@ jobs: run: tox -e lint_docstring_include_list - name: Run mypy checks run: tox -e mypy + - uses: psf/black@stable diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 00000000000..b3778fdf7c4 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,14 @@ +[settings] +extend_skip=doc/source/conf.py,lib/galaxy/util/jstree.py +force_alphabetical_sort_within_sections=true +# Override force_grid_wrap value from profile=black, but black is still happy +force_grid_wrap=2 +# Same line length as for black +line_length=120 +no_lines_before=LOCALFOLDER +profile=black +reverse_relative=true +skip_gitignore=true +# Make isort run faster by skipping database +skip_glob=database/* +src_paths=lib diff --git a/Makefile b/Makefile index 93e632030d2..1c8d1e0a6d1 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ VENV?=.venv # Source virtualenv to execute command (flake8, sphinx, twine, etc...) IN_VENV=if [ -f "$(VENV)/bin/activate" ]; then . "$(VENV)/bin/activate"; fi; -RELEASE_CURR:=22.01 +RELEASE_CURR:=22.05 RELEASE_UPSTREAM:=upstream TARGET_BRANCH=$(RELEASE_UPSTREAM)/dev CONFIG_MANAGE=$(IN_VENV) python lib/galaxy/config/config_manage.py @@ -41,6 +41,10 @@ setup-venv: diff-format: $(IN_VENV) darker -r $(TARGET_BRANCH) +format: + $(IN_VENV) isort . + $(IN_VENV) black . + list-dependency-updates: setup-venv $(IN_VENV) pip list --outdated --format=columns diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 49073dfd3fa..81a728403df 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -61,7 +61,10 @@ from galaxy.util.properties import ( ) from galaxy.web.formatting import expand_pretty_datetime_format from galaxy.web_stack import get_stack_facts -from ..version import VERSION_MAJOR, VERSION_MINOR +from ..version import ( + VERSION_MAJOR, + VERSION_MINOR, +) try: from importlib.resources import files # type: ignore[attr-defined] @@ -78,60 +81,60 @@ if TYPE_CHECKING: log = logging.getLogger(__name__) -GALAXY_APP_NAME = 'galaxy' -GALAXY_SCHEMAS_PATH = files('galaxy.config') / 'schemas' -GALAXY_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / 'config_schema.yml' -UWSGI_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / 'uwsgi_schema.yml' +GALAXY_APP_NAME = "galaxy" +GALAXY_SCHEMAS_PATH = files("galaxy.config") / "schemas" +GALAXY_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "config_schema.yml" +UWSGI_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "uwsgi_schema.yml" LOGGING_CONFIG_DEFAULT: Dict[str, Any] = { - 'disable_existing_loggers': False, - 'version': 1, - 'root': { - 'handlers': ['console'], - 'level': 'DEBUG', + "disable_existing_loggers": False, + "version": 1, + "root": { + "handlers": ["console"], + "level": "DEBUG", }, - 'loggers': { - 'paste.httpserver.ThreadPool': { - 'level': 'WARN', - 'qualname': 'paste.httpserver.ThreadPool', + "loggers": { + "paste.httpserver.ThreadPool": { + "level": "WARN", + "qualname": "paste.httpserver.ThreadPool", }, - 'sqlalchemy_json.track': { - 'level': 'WARN', - 'qualname': 'sqlalchemy_json.track', + "sqlalchemy_json.track": { + "level": "WARN", + "qualname": "sqlalchemy_json.track", }, - 'urllib3.connectionpool': { - 'level': 'WARN', - 'qualname': 'urllib3.connectionpool', + "urllib3.connectionpool": { + "level": "WARN", + "qualname": "urllib3.connectionpool", }, - 'routes.middleware': { - 'level': 'WARN', - 'qualname': 'routes.middleware', + "routes.middleware": { + "level": "WARN", + "qualname": "routes.middleware", }, - 'amqp': { - 'level': 'INFO', - 'qualname': 'amqp', + "amqp": { + "level": "INFO", + "qualname": "amqp", }, - 'botocore': { - 'level': 'INFO', - 'qualname': 'botocore', + "botocore": { + "level": "INFO", + "qualname": "botocore", }, }, - 'filters': { - 'stack': { - '()': 'galaxy.web_stack.application_stack_log_filter', + "filters": { + "stack": { + "()": "galaxy.web_stack.application_stack_log_filter", }, }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'stack', - 'level': 'DEBUG', - 'stream': 'ext://sys.stderr', - 'filters': ['stack'], + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "stack", + "level": "DEBUG", + "stream": "ext://sys.stderr", + "filters": ["stack"], }, }, - 'formatters': { - 'stack': { - '()': 'galaxy.web_stack.application_stack_log_formatter', + "formatters": { + "stack": { + "()": "galaxy.web_stack.application_stack_log_formatter", }, }, } @@ -139,7 +142,7 @@ LOGGING_CONFIG_DEFAULT: Dict[str, Any] = { def find_root(kwargs): - return os.path.abspath(kwargs.get('root_dir', '.')) + return os.path.abspath(kwargs.get("root_dir", ".")) OptStr = TypeVar("OptStr", None, str) @@ -150,7 +153,9 @@ class BaseAppConfiguration(HasDynamicProperties): # If VALUE == first directory in a user-supplied path that resolves to KEY, it will be stripped from that path renamed_options: Optional[Dict[str, str]] = None deprecated_dirs: Dict[str, str] = {} - paths_to_check_against_root: Set[str] = set() # backward compatibility: if resolved path doesn't exist, try resolving w.r.t root + paths_to_check_against_root: Set[ + str + ] = set() # backward compatibility: if resolved path doesn't exist, try resolving w.r.t root add_sample_file_to_defaults: Set[str] = set() # for these options, add sample config files to their defaults listify_options: Set[str] = set() # values for these options are processed as lists of values object_store_store_by: str @@ -188,19 +193,21 @@ class BaseAppConfiguration(HasDynamicProperties): Fix deprecated database URLs (postgres... >> postgresql...) https://docs.sqlalchemy.org/en/14/changelog/changelog_14.html#change-3687655465c25a39b968b4f5f6e9170b """ - old_dialect, new_dialect = 'postgres', 'postgresql' - old_prefixes = (f'{old_dialect}:', f'{old_dialect}+') # check for postgres://foo and postgres+driver//foo + old_dialect, new_dialect = "postgres", "postgresql" + old_prefixes = (f"{old_dialect}:", f"{old_dialect}+") # check for postgres://foo and postgres+driver//foo offset = len(old_dialect) - keys = ('database_connection', 'install_database_connection') + keys = ("database_connection", "install_database_connection") for key in keys: if key in kwargs: value = kwargs[key] for prefix in old_prefixes: if value.startswith(prefix): - value = f'{new_dialect}{value[offset:]}' + value = f"{new_dialect}{value[offset:]}" kwargs[key] = value - log.warning('PostgreSQL database URLs of the form "postgres://" have been ' - 'deprecated. Please use "postgresql://".') + log.warning( + 'PostgreSQL database URLs of the form "postgres://" have been ' + 'deprecated. Please use "postgresql://".' + ) def is_set(self, key): """Check if a configuration option has been explicitly set.""" @@ -215,13 +222,12 @@ class BaseAppConfiguration(HasDynamicProperties): return self._in_root_dir(path) def _set_config_base(self, config_kwargs): - def _set_global_conf(): - self.config_file = find_config_file('galaxy') - self.global_conf = config_kwargs.get('global_conf') + self.config_file = find_config_file("galaxy") + self.global_conf = config_kwargs.get("global_conf") self.global_conf_parser = configparser.ConfigParser() if not self.config_file and self.global_conf and "__file__" in self.global_conf: - self.config_file = os.path.join(self.root, self.global_conf['__file__']) + self.config_file = os.path.join(self.root, self.global_conf["__file__"]) if self.config_file is None: log.warning("No Galaxy config file found, running from current working directory: %s", os.getcwd()) @@ -236,40 +242,40 @@ class BaseAppConfiguration(HasDynamicProperties): def _set_config_directories(): # Set config_dir to value from kwargs OR dirname of config_file OR None _config_dir = os.path.dirname(self.config_file) if self.config_file else None - self.config_dir = config_kwargs.get('config_dir', _config_dir) + self.config_dir = config_kwargs.get("config_dir", _config_dir) # Make path absolute before using it as base for other paths if self.config_dir: self.config_dir = os.path.abspath(self.config_dir) - self.data_dir = config_kwargs.get('data_dir') + self.data_dir = config_kwargs.get("data_dir") if self.data_dir: self.data_dir = os.path.abspath(self.data_dir) - self.sample_config_dir = os.path.join(os.path.dirname(__file__), 'sample') + self.sample_config_dir = os.path.join(os.path.dirname(__file__), "sample") if self.sample_config_dir: self.sample_config_dir = os.path.abspath(self.sample_config_dir) - self.managed_config_dir = config_kwargs.get('managed_config_dir') + self.managed_config_dir = config_kwargs.get("managed_config_dir") if self.managed_config_dir: self.managed_config_dir = os.path.abspath(self.managed_config_dir) if running_from_source: if not self.config_dir: - self.config_dir = os.path.join(self.root, 'config') + self.config_dir = os.path.join(self.root, "config") if not self.data_dir: - self.data_dir = os.path.join(self.root, 'database') + self.data_dir = os.path.join(self.root, "database") if not self.managed_config_dir: self.managed_config_dir = self.config_dir else: if not self.config_dir: self.config_dir = os.getcwd() if not self.data_dir: - self.data_dir = self._in_config_dir('data') + self.data_dir = self._in_config_dir("data") if not self.managed_config_dir: - self.managed_config_dir = self._in_data_dir('config') + self.managed_config_dir = self._in_data_dir("config") # TODO: do we still need to support ../shed_tools when running_from_source? - self.shed_tools_dir = self._in_data_dir('shed_tools') + self.shed_tools_dir = self._in_data_dir("shed_tools") log.debug("Configuration directory is %s", self.config_dir) log.debug("Data directory is %s", self.data_dir) @@ -280,7 +286,7 @@ class BaseAppConfiguration(HasDynamicProperties): def _load_schema(self): # Override in subclasses - raise Exception('Not implemented') + raise Exception("Not implemented") def _preprocess_paths_to_resolve(self): # For these options, if option is not set, listify its defaults and add a sample config file. @@ -288,17 +294,20 @@ class BaseAppConfiguration(HasDynamicProperties): for key in self.add_sample_file_to_defaults: if not self.is_set(key): defaults = listify(getattr(self, key), do_strip=True) - sample = f'{defaults[-1]}.sample' # if there are multiple defaults, use last as template + sample = f"{defaults[-1]}.sample" # if there are multiple defaults, use last as template sample = self._in_sample_dir(sample) # resolve w.r.t sample_dir defaults.append(sample) setattr(self, key, defaults) def _postprocess_paths_to_resolve(self): - def select_one_path_from_list(): # To consider: options with a sample file added to defaults except options that can have multiple values. # If value is not set, check each path in list; set to first path that exists; if none exist, set to last path in list. - keys = self.add_sample_file_to_defaults - self.listify_options if self.listify_options else self.add_sample_file_to_defaults + keys = ( + self.add_sample_file_to_defaults - self.listify_options + if self.listify_options + else self.add_sample_file_to_defaults + ) for key in keys: if not self.is_set(key): paths = getattr(self, key) @@ -307,7 +316,9 @@ class BaseAppConfiguration(HasDynamicProperties): setattr(self, key, path) break else: - setattr(self, key, paths[-1]) # TODO: we assume it exists; but we've already checked in the loop! Raise error instead? + setattr( + self, key, paths[-1] + ) # TODO: we assume it exists; but we've already checked in the loop! Raise error instead? def select_one_or_all_paths_from_list(): # Values for these options are lists of paths. If value is not set, use defaults if all paths in list exist; @@ -320,7 +331,9 @@ class BaseAppConfiguration(HasDynamicProperties): setattr(self, key, [paths[-1]]) # value is a list break - if self.add_sample_file_to_defaults: # Currently, this is the ONLY case when we need to pick one file from a list + if ( + self.add_sample_file_to_defaults + ): # Currently, this is the ONLY case when we need to pick one file from a list select_one_path_from_list() if self.listify_options: select_one_or_all_paths_from_list() @@ -346,7 +359,7 @@ class BaseAppConfiguration(HasDynamicProperties): } def convert_datatype(key, value): - datatype = self.schema.app_schema[key].get('type') + datatype = self.schema.app_schema[key].get("type") # check for `not None` explicitly (value can be falsy) if value is not None and datatype in type_converters: # convert value or each item in value to type `datatype` @@ -367,14 +380,18 @@ class BaseAppConfiguration(HasDynamicProperties): ignore = first_dir + os.sep log.warning( "Paths for the '%s' option are now relative to '%s', remove the leading '%s' " - "to suppress this warning: %s", key, resolves_to, ignore, path + "to suppress this warning: %s", + key, + resolves_to, + ignore, + path, ) - paths[i] = path[len(ignore):] + paths[i] = path[len(ignore) :] # return list or string, depending on type of `value` if isinstance(value, list): return paths - return ','.join(paths) + return ",".join(paths) return value for key, value in kwargs.items(): @@ -387,7 +404,7 @@ class BaseAppConfiguration(HasDynamicProperties): def _create_attributes_from_raw_config(self): # `base_configs` are a special case: these attributes have been created and will be ignored # by the code below. Trying to overwrite any other existing attributes will raise an error. - base_configs = {'config_dir', 'data_dir', 'managed_config_dir'} + base_configs = {"config_dir", "data_dir", "managed_config_dir"} for key, value in self._raw_config.items(): if not hasattr(self, key): setattr(self, key, value) @@ -395,7 +412,6 @@ class BaseAppConfiguration(HasDynamicProperties): raise ConfigurationError(f"Attempting to override existing attribute '{key}'") def _resolve_paths(self): - def resolve(key): if key in _cache: # resolve each path only once return _cache[key] @@ -420,7 +436,7 @@ class BaseAppConfiguration(HasDynamicProperties): # Check if value is a list or should be listified; if so, listify and resolve each item separately. if type(value) is list or (self.listify_options and key in self.listify_options): saved_values = listify(getattr(self, key), do_strip=True) # listify and save original value - setattr(self, key, '_') # replace value with temporary placeholder + setattr(self, key, "_") # replace value with temporary placeholder resolve(key) # resolve temporary value (`_` becomes `parent-path/_`) resolved_base = getattr(self, key)[:-1] # get rid of placeholder in resolved path # apply resolved base to saved values @@ -433,7 +449,6 @@ class BaseAppConfiguration(HasDynamicProperties): self._check_against_root(key) def _check_against_root(self, key): - def get_path(current_path, initial_path): # if path does not exist and was set as relative: if not self._path_exists(current_path) and not os.path.isabs(initial_path): @@ -531,64 +546,71 @@ class CommonConfigurationMixin: class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): - deprecated_options = ('database_file', 'track_jobs_in_database', 'blacklist_file', 'whitelist_file', - 'sanitize_whitelist_file', 'user_library_import_symlink_whitelist', 'fetch_url_whitelist', - 'containers_resolvers_config_file') + deprecated_options = ( + "database_file", + "track_jobs_in_database", + "blacklist_file", + "whitelist_file", + "sanitize_whitelist_file", + "user_library_import_symlink_whitelist", + "fetch_url_whitelist", + "containers_resolvers_config_file", + ) renamed_options = { - 'blacklist_file': 'email_domain_blocklist_file', - 'whitelist_file': 'email_domain_allowlist_file', - 'sanitize_whitelist_file': 'sanitize_allowlist_file', - 'user_library_import_symlink_whitelist': 'user_library_import_symlink_allowlist', - 'fetch_url_whitelist': 'fetch_url_allowlist', - 'containers_resolvers_config_file': 'container_resolvers_config_file', + "blacklist_file": "email_domain_blocklist_file", + "whitelist_file": "email_domain_allowlist_file", + "sanitize_whitelist_file": "sanitize_allowlist_file", + "user_library_import_symlink_whitelist": "user_library_import_symlink_allowlist", + "fetch_url_whitelist": "fetch_url_allowlist", + "containers_resolvers_config_file": "container_resolvers_config_file", } - default_config_file_name = 'galaxy.yml' - deprecated_dirs = {'config_dir': 'config', 'data_dir': 'database'} + default_config_file_name = "galaxy.yml" + deprecated_dirs = {"config_dir": "config", "data_dir": "database"} paths_to_check_against_root = { - 'auth_config_file', - 'build_sites_config_file', - 'containers_config_file', - 'data_manager_config_file', - 'datatypes_config_file', - 'dependency_resolvers_config_file', - 'error_report_file', - 'job_config_file', - 'job_metrics_config_file', - 'job_resource_params_file', - 'local_conda_mapping_file', - 'migrated_tools_config', - 'modules_mapping_files', - 'object_store_config_file', - 'oidc_backends_config_file', - 'oidc_config_file', - 'shed_data_manager_config_file', - 'shed_tool_config_file', - 'shed_tool_data_table_config', - 'tool_destinations_config_file', - 'tool_sheds_config_file', - 'user_preferences_extra_conf_path', - 'workflow_resource_params_file', - 'workflow_schedulers_config_file', - 'markdown_export_css', - 'markdown_export_css_pages', - 'markdown_export_css_invocation_reports', - 'file_path', - 'tool_data_table_config_path', - 'tool_config_file', + "auth_config_file", + "build_sites_config_file", + "containers_config_file", + "data_manager_config_file", + "datatypes_config_file", + "dependency_resolvers_config_file", + "error_report_file", + "job_config_file", + "job_metrics_config_file", + "job_resource_params_file", + "local_conda_mapping_file", + "migrated_tools_config", + "modules_mapping_files", + "object_store_config_file", + "oidc_backends_config_file", + "oidc_config_file", + "shed_data_manager_config_file", + "shed_tool_config_file", + "shed_tool_data_table_config", + "tool_destinations_config_file", + "tool_sheds_config_file", + "user_preferences_extra_conf_path", + "workflow_resource_params_file", + "workflow_schedulers_config_file", + "markdown_export_css", + "markdown_export_css_pages", + "markdown_export_css_invocation_reports", + "file_path", + "tool_data_table_config_path", + "tool_config_file", } add_sample_file_to_defaults = { - 'build_sites_config_file', - 'datatypes_config_file', - 'job_metrics_config_file', - 'tool_data_table_config_path', - 'tool_config_file', + "build_sites_config_file", + "datatypes_config_file", + "job_metrics_config_file", + "tool_data_table_config_path", + "tool_config_file", } listify_options = { - 'tool_data_table_config_path', - 'tool_config_file', + "tool_data_table_config_path", + "tool_config_file", } database_connection: str tool_path: str @@ -638,17 +660,20 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): This method should be deleted after migration to SQLAlchemy 2.0 is complete. To enable warnings, set `GALAXY_CONFIG_SQLALCHEMY_WARN_20=1`, """ - warn = string_as_bool(kwargs.get('sqlalchemy_warn_20', False)) + warn = string_as_bool(kwargs.get("sqlalchemy_warn_20", False)) if warn: import sqlalchemy + sqlalchemy.util.deprecations.SQLALCHEMY_WARN_20 = True self._setup_sqlalchemy20_warnings_filters() def _setup_sqlalchemy20_warnings_filters(self): import warnings + from sqlalchemy.exc import RemovedIn20Warning + # Always display RemovedIn20Warning warnings. - warnings.filterwarnings('always', category=RemovedIn20Warning) + warnings.filterwarnings("always", category=RemovedIn20Warning) # Optionally, enable filters for specific warnings (raise error, or log, etc.) # messages = [ # r"replace with warning text to match", @@ -694,13 +719,13 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.version_minor = VERSION_MINOR # Database related configuration - self.check_migrate_databases = kwargs.get('check_migrate_databases', True) + self.check_migrate_databases = kwargs.get("check_migrate_databases", True) if not self.database_connection: # Provide default if not supplied by user - db_path = self._in_data_dir('universe.sqlite') - self.database_connection = f'sqlite:///{db_path}?isolation_level=IMMEDIATE' + db_path = self._in_data_dir("universe.sqlite") + self.database_connection = f"sqlite:///{db_path}?isolation_level=IMMEDIATE" self.database_engine_options = get_database_engine_options(kwargs) - self.database_create_tables = string_as_bool(kwargs.get('database_create_tables', 'True')) - self.database_encoding = kwargs.get('database_encoding') # Create new databases with this encoding + self.database_create_tables = string_as_bool(kwargs.get("database_create_tables", "True")) + self.database_encoding = kwargs.get("database_encoding") # Create new databases with this encoding self.thread_local_log = None if self.enable_per_request_sql_debugging: self.thread_local_log = threading.local() @@ -711,12 +736,12 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.tool_path = self._in_root_dir(self.tool_path) self.tool_data_path = self._in_root_dir(self.tool_data_path) if not running_from_source and kwargs.get("tool_data_path") is None: - self.tool_data_path = self._in_data_dir(self.schema.defaults['tool_data_path']) + self.tool_data_path = self._in_data_dir(self.schema.defaults["tool_data_path"]) self.builds_file_path = os.path.join(self.tool_data_path, self.builds_file_path) self.len_file_path = os.path.join(self.tool_data_path, self.len_file_path) self.oidc = {} self.integrated_tool_panel_config = self._in_managed_config_dir(self.integrated_tool_panel_config) - integrated_tool_panel_tracking_directory = kwargs.get('integrated_tool_panel_tracking_directory') + integrated_tool_panel_tracking_directory = kwargs.get("integrated_tool_panel_tracking_directory") if integrated_tool_panel_tracking_directory: self.integrated_tool_panel_tracking_directory = self._in_root_dir(integrated_tool_panel_tracking_directory) else: @@ -729,18 +754,18 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.user_tool_filters = listify(self.user_tool_filters, do_strip=True) self.user_tool_label_filters = listify(self.user_tool_label_filters, do_strip=True) self.user_tool_section_filters = listify(self.user_tool_section_filters, do_strip=True) - self.has_user_tool_filters = bool(self.user_tool_filters or self.user_tool_label_filters or self.user_tool_section_filters) - - self.password_expiration_period = timedelta( - days=int(cast(SupportsInt, self.password_expiration_period)) + self.has_user_tool_filters = bool( + self.user_tool_filters or self.user_tool_label_filters or self.user_tool_section_filters ) + self.password_expiration_period = timedelta(days=int(cast(SupportsInt, self.password_expiration_period))) + if self.shed_tool_data_path: self.shed_tool_data_path = self._in_root_dir(self.shed_tool_data_path) else: self.shed_tool_data_path = self.tool_data_path - self.running_functional_tests = string_as_bool(kwargs.get('running_functional_tests', False)) + self.running_functional_tests = string_as_bool(kwargs.get("running_functional_tests", False)) if isinstance(self.hours_between_check, str): self.hours_between_check = float(self.hours_between_check) try: @@ -766,9 +791,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.use_remote_user = self.use_remote_user or self.single_user self.fetch_url_allowlist_ips = [ ipaddress.ip_network(unicodify(ip.strip())) # If it has a slash, assume 127.0.0.1/24 notation - if '/' in ip else - ipaddress.ip_address(unicodify(ip.strip())) # Otherwise interpret it as an ip address. - for ip in kwargs.get("fetch_url_allowlist", "").split(',') + if "/" in ip + else ipaddress.ip_address(unicodify(ip.strip())) # Otherwise interpret it as an ip address. + for ip in kwargs.get("fetch_url_allowlist", "").split(",") if len(ip.strip()) > 0 ] self.job_queue_cleanup_interval = int(kwargs.get("job_queue_cleanup_interval", "5")) @@ -781,36 +806,47 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.preserve_python_environment = "legacy_only" self.nodejs_path = kwargs.get("nodejs_path") self.container_image_cache_path = self._in_data_dir(kwargs.get("container_image_cache_path", "container_cache")) - self.output_size_limit = int(kwargs.get('output_size_limit', 0)) + self.output_size_limit = int(kwargs.get("output_size_limit", 0)) # activation_email was used until release_15.03 - activation_email = kwargs.get('activation_email') + activation_email = kwargs.get("activation_email") self.email_from = self.email_from or activation_email - self.email_domain_blocklist_content = self._load_list_from_file(self._in_config_dir(self.email_domain_blocklist_file)) if self.email_domain_blocklist_file else None - self.email_domain_allowlist_content = self._load_list_from_file(self._in_config_dir(self.email_domain_allowlist_file)) if self.email_domain_allowlist_file else None + self.email_domain_blocklist_content = ( + self._load_list_from_file(self._in_config_dir(self.email_domain_blocklist_file)) + if self.email_domain_blocklist_file + else None + ) + self.email_domain_allowlist_content = ( + self._load_list_from_file(self._in_config_dir(self.email_domain_allowlist_file)) + if self.email_domain_allowlist_file + else None + ) # These are not even beta - just experiments - don't use them unless # you want yours tools to be broken in the future. - self.enable_beta_tool_formats = string_as_bool(kwargs.get('enable_beta_tool_formats', 'False')) + self.enable_beta_tool_formats = string_as_bool(kwargs.get("enable_beta_tool_formats", "False")) - if self.workflow_resource_params_mapper and ':' not in self.workflow_resource_params_mapper: + if self.workflow_resource_params_mapper and ":" not in self.workflow_resource_params_mapper: # Assume it is not a Python function, so a file; else: a Python function self.workflow_resource_params_mapper = self._in_root_dir(self.workflow_resource_params_mapper) - self.pbs_application_server = kwargs.get('pbs_application_server', "") - self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "") - self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "") - self.pbs_stage_path = kwargs.get('pbs_stage_path', "") + self.pbs_application_server = kwargs.get("pbs_application_server", "") + self.pbs_dataset_server = kwargs.get("pbs_dataset_server", "") + self.pbs_dataset_path = kwargs.get("pbs_dataset_path", "") + self.pbs_stage_path = kwargs.get("pbs_stage_path", "") _sanitize_allowlist_path = self._in_managed_config_dir(self.sanitize_allowlist_file) if not os.path.isfile(_sanitize_allowlist_path): # then check old default location for deprecated in ( - self._in_managed_config_dir('sanitize_whitelist.txt'), - self._in_root_dir('config/sanitize_whitelist.txt')): + self._in_managed_config_dir("sanitize_whitelist.txt"), + self._in_root_dir("config/sanitize_whitelist.txt"), + ): if os.path.isfile(deprecated): - log.warning("The path '%s' for the 'sanitize_allowlist_file' config option is " + log.warning( + "The path '%s' for the 'sanitize_allowlist_file' config option is " "deprecated and will be no longer checked in a future release. Please consult " - "the latest version of the sample configuration file." % deprecated) + "the latest version of the sample configuration file." % deprecated + ) _sanitize_allowlist_path = deprecated break self.sanitize_allowlist_file = _sanitize_allowlist_path @@ -819,18 +855,24 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): if "trust_jupyter_notebook_conversion" not in kwargs: # if option not set, check IPython-named alternative, falling back to schema default if not set either _default = self.trust_jupyter_notebook_conversion - self.trust_jupyter_notebook_conversion = string_as_bool(kwargs.get('trust_ipython_notebook_conversion', _default)) + self.trust_jupyter_notebook_conversion = string_as_bool( + kwargs.get("trust_ipython_notebook_conversion", _default) + ) # Configuration for the message box directly below the masthead. - self.blog_url = kwargs.get('blog_url') + self.blog_url = kwargs.get("blog_url") self.user_library_import_symlink_allowlist = listify(self.user_library_import_symlink_allowlist, do_strip=True) - self.user_library_import_dir_auto_creation = self.user_library_import_dir_auto_creation if self.user_library_import_dir else False + self.user_library_import_dir_auto_creation = ( + self.user_library_import_dir_auto_creation if self.user_library_import_dir else False + ) # Searching data libraries - self.ftp_upload_dir_template = kwargs.get('ftp_upload_dir_template', '${ftp_upload_dir}%s${ftp_upload_dir_identifier}' % os.path.sep) + self.ftp_upload_dir_template = kwargs.get( + "ftp_upload_dir_template", "${ftp_upload_dir}%s${ftp_upload_dir_identifier}" % os.path.sep + ) # Support older library-specific path paste option but just default to the new # allow_path_paste value. - self.allow_library_path_paste = string_as_bool(kwargs.get('allow_library_path_paste', self.allow_path_paste)) - self.disable_library_comptypes = kwargs.get('disable_library_comptypes', '').lower().split(',') - self.check_upload_content = string_as_bool(kwargs.get('check_upload_content', True)) + self.allow_library_path_paste = string_as_bool(kwargs.get("allow_library_path_paste", self.allow_path_paste)) + self.disable_library_comptypes = kwargs.get("disable_library_comptypes", "").lower().split(",") + self.check_upload_content = string_as_bool(kwargs.get("check_upload_content", True)) # On can mildly speed up Galaxy startup time by disabling index of help, # not needed on production systems but useful if running many functional tests. self.index_tool_help = string_as_bool(kwargs.get("index_tool_help", True)) @@ -840,7 +882,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): # Deployers may either specify a complete list of mapping files or get the default for free and just # specify a local mapping file to adapt and extend the default one. if "conda_mapping_files" not in kwargs: - _default_mapping = self._in_root_dir(os.path.join("lib", "galaxy", "tool_util", "deps", "resolvers", "default_conda_mapping.yml")) + _default_mapping = self._in_root_dir( + os.path.join("lib", "galaxy", "tool_util", "deps", "resolvers", "default_conda_mapping.yml") + ) # dependency resolution options are consumed via config_dict - so don't populate # self, populate config_dict self.config_dict["conda_mapping_files"] = [self.local_conda_mapping_file, _default_mapping] @@ -849,16 +893,16 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.container_resolvers_config_file = self._in_config_dir(self.container_resolvers_config_file) # tool_dependency_dir can be "none" (in old configs). If so, set it to None - if self.tool_dependency_dir and self.tool_dependency_dir.lower() == 'none': + if self.tool_dependency_dir and self.tool_dependency_dir.lower() == "none": self.tool_dependency_dir = None if self.involucro_path is None: - target_dir = self.tool_dependency_dir or self.schema.defaults['tool_dependency_dir'] + target_dir = self.tool_dependency_dir or self.schema.defaults["tool_dependency_dir"] self.involucro_path = self._in_data_dir(os.path.join(target_dir, "involucro")) self.involucro_path = self._in_root_dir(self.involucro_path) if self.mulled_channels: self.mulled_channels = [c.strip() for c in self.mulled_channels.split(",")] # type: ignore[attr-defined] - default_job_resubmission_condition = kwargs.get('default_job_resubmission_condition', '') + default_job_resubmission_condition = kwargs.get("default_job_resubmission_condition", "") if not default_job_resubmission_condition.strip(): default_job_resubmission_condition = None self.default_job_resubmission_condition = default_job_resubmission_condition @@ -870,92 +914,100 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): if self.tus_upload_store: self.tus_upload_store = os.path.abspath(self.tus_upload_store) - self.object_store = kwargs.get('object_store', 'disk') - self.object_store_check_old_style = string_as_bool(kwargs.get('object_store_check_old_style', False)) - self.object_store_cache_path = self._in_root_dir(kwargs.get("object_store_cache_path", self._in_data_dir("object_store_cache"))) + self.object_store = kwargs.get("object_store", "disk") + self.object_store_check_old_style = string_as_bool(kwargs.get("object_store_check_old_style", False)) + self.object_store_cache_path = self._in_root_dir( + kwargs.get("object_store_cache_path", self._in_data_dir("object_store_cache")) + ) self._configure_dataset_storage() # Handle AWS-specific config options for backward compatibility - if kwargs.get('aws_access_key') is not None: - self.os_access_key = kwargs.get('aws_access_key') - self.os_secret_key = kwargs.get('aws_secret_key') - self.os_bucket_name = kwargs.get('s3_bucket') - self.os_use_reduced_redundancy = kwargs.get('use_reduced_redundancy', False) + if kwargs.get("aws_access_key") is not None: + self.os_access_key = kwargs.get("aws_access_key") + self.os_secret_key = kwargs.get("aws_secret_key") + self.os_bucket_name = kwargs.get("s3_bucket") + self.os_use_reduced_redundancy = kwargs.get("use_reduced_redundancy", False) else: - self.os_access_key = kwargs.get('os_access_key') - self.os_secret_key = kwargs.get('os_secret_key') - self.os_bucket_name = kwargs.get('os_bucket_name') - self.os_use_reduced_redundancy = kwargs.get('os_use_reduced_redundancy', False) - self.os_host = kwargs.get('os_host') - self.os_port = kwargs.get('os_port') - self.os_is_secure = string_as_bool(kwargs.get('os_is_secure', True)) - self.os_conn_path = kwargs.get('os_conn_path', '/') - self.object_store_cache_size = float(kwargs.get('object_store_cache_size', -1)) - self.distributed_object_store_config_file = kwargs.get('distributed_object_store_config_file') + self.os_access_key = kwargs.get("os_access_key") + self.os_secret_key = kwargs.get("os_secret_key") + self.os_bucket_name = kwargs.get("os_bucket_name") + self.os_use_reduced_redundancy = kwargs.get("os_use_reduced_redundancy", False) + self.os_host = kwargs.get("os_host") + self.os_port = kwargs.get("os_port") + self.os_is_secure = string_as_bool(kwargs.get("os_is_secure", True)) + self.os_conn_path = kwargs.get("os_conn_path", "/") + self.object_store_cache_size = float(kwargs.get("object_store_cache_size", -1)) + self.distributed_object_store_config_file = kwargs.get("distributed_object_store_config_file") if self.distributed_object_store_config_file is not None: self.distributed_object_store_config_file = self._in_root_dir(self.distributed_object_store_config_file) - self.irods_root_collection_path = kwargs.get('irods_root_collection_path') - self.irods_default_resource = kwargs.get('irods_default_resource') + self.irods_root_collection_path = kwargs.get("irods_root_collection_path") + self.irods_default_resource = kwargs.get("irods_default_resource") # Heartbeat log file name override - if self.global_conf is not None and 'heartbeat_log' in self.global_conf: - self.heartbeat_log = self.global_conf['heartbeat_log'] + if self.global_conf is not None and "heartbeat_log" in self.global_conf: + self.heartbeat_log = self.global_conf["heartbeat_log"] # Determine which 'server:' this is - self.server_name = 'main' + self.server_name = "main" for arg in sys.argv: # Crummy, but PasteScript does not give you a way to determine this - if arg.lower().startswith('--server-name='): - self.server_name = arg.split('=', 1)[-1] + if arg.lower().startswith("--server-name="): + self.server_name = arg.split("=", 1)[-1] # Allow explicit override of server name in config params if "server_name" in kwargs: self.server_name = kwargs.get("server_name") # The application stack code may manipulate the server name. It also needs to be accessible via the get() method # for galaxy.util.facts() - self.config_dict['base_server_name'] = self.base_server_name = self.server_name + self.config_dict["base_server_name"] = self.base_server_name = self.server_name # Store all configured server names for the message queue routing self.server_names = [] for section in self.global_conf_parser.sections(): - if section.startswith('server:'): - self.server_names.append(section.replace('server:', '', 1)) + if section.startswith("server:"): + self.server_names.append(section.replace("server:", "", 1)) self._set_galaxy_infrastructure_url(kwargs) # Asynchronous execution process pools - limited functionality for now, attach_to_pools is designed to allow # webless Galaxy server processes to attach to arbitrary message queues (e.g. as job handlers) so they do not # have to be explicitly defined as such in the job configuration. - self.attach_to_pools = kwargs.get('attach_to_pools', []) or [] + self.attach_to_pools = kwargs.get("attach_to_pools", []) or [] # Store advanced job management config - self.job_handlers = [x.strip() for x in kwargs.get('job_handlers', self.server_name).split(',')] - self.default_job_handlers = [x.strip() for x in kwargs.get('default_job_handlers', ','.join(self.job_handlers)).split(',')] + self.job_handlers = [x.strip() for x in kwargs.get("job_handlers", self.server_name).split(",")] + self.default_job_handlers = [ + x.strip() for x in kwargs.get("default_job_handlers", ",".join(self.job_handlers)).split(",") + ] # Galaxy internal control queue configuration. # If specified in universe, use it, otherwise we use whatever 'real' # database is specified. Lastly, we create and use new sqlite database # (to minimize locking) as a final option. - if 'amqp_internal_connection' in kwargs: - self.amqp_internal_connection = kwargs.get('amqp_internal_connection') + if "amqp_internal_connection" in kwargs: + self.amqp_internal_connection = kwargs.get("amqp_internal_connection") # TODO Get extra amqp args as necessary for ssl - elif 'database_connection' in kwargs: + elif "database_connection" in kwargs: self.amqp_internal_connection = f"sqlalchemy+{self.database_connection}" else: - self.amqp_internal_connection = f"sqlalchemy+sqlite:///{self._in_data_dir('control.sqlite')}?isolation_level=IMMEDIATE" + self.amqp_internal_connection = ( + f"sqlalchemy+sqlite:///{self._in_data_dir('control.sqlite')}?isolation_level=IMMEDIATE" + ) self.pretty_datetime_format = expand_pretty_datetime_format(self.pretty_datetime_format) try: with open(self.user_preferences_extra_conf_path) as stream: self.user_preferences_extra = yaml.safe_load(stream) except Exception: - if self.is_set('user_preferences_extra_conf_path'): - log.warning(f'Config file ({self.user_preferences_extra_conf_path}) could not be found or is malformed.') - self.user_preferences_extra = {'preferences': {}} + if self.is_set("user_preferences_extra_conf_path"): + log.warning( + f"Config file ({self.user_preferences_extra_conf_path}) could not be found or is malformed." + ) + self.user_preferences_extra = {"preferences": {}} # Experimental: This will not be enabled by default and will hide # nonproduction code. # The api_folders refers to whether the API exposes the /folders section. - self.api_folders = string_as_bool(kwargs.get('api_folders', False)) + self.api_folders = string_as_bool(kwargs.get("api_folders", False)) # This is for testing new library browsing capabilities. - self.new_lib_browse = string_as_bool(kwargs.get('new_lib_browse', False)) + self.new_lib_browse = string_as_bool(kwargs.get("new_lib_browse", False)) # Logging configuration with logging.config.configDict: # Statistics and profiling with statsd - self.statsd_host = kwargs.get('statsd_host', '') + self.statsd_host = kwargs.get("statsd_host", "") ie_dirs = self.interactive_environment_plugins_directory self.gie_dirs = [d.strip() for d in (ie_dirs.split(",") if ie_dirs else [])] @@ -966,7 +1018,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.manage_dynamic_proxy = self.dynamic_proxy_manage # Set to false if being launched externally # InteractiveTools propagator mapping file - self.interactivetools_map = self._in_root_dir(kwargs.get("interactivetools_map", self._in_data_dir("interactivetools_map.sqlite"))) + self.interactivetools_map = self._in_root_dir( + kwargs.get("interactivetools_map", self._in_data_dir("interactivetools_map.sqlite")) + ) self.containers_conf = parse_containers_config(self.containers_config_file) @@ -993,58 +1047,56 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.redact_user_address_during_deletion = True self.allow_user_deletion = True - LOGGING_CONFIG_DEFAULT['formatters']['brief'] = { - 'format': '%(asctime)s %(levelname)-8s %(name)-15s %(message)s' + LOGGING_CONFIG_DEFAULT["formatters"]["brief"] = { + "format": "%(asctime)s %(levelname)-8s %(name)-15s %(message)s" } - LOGGING_CONFIG_DEFAULT['handlers']['compliance_log'] = { - 'class': 'logging.handlers.RotatingFileHandler', - 'formatter': 'brief', - 'filename': 'compliance.log', - 'backupCount': 0, + LOGGING_CONFIG_DEFAULT["handlers"]["compliance_log"] = { + "class": "logging.handlers.RotatingFileHandler", + "formatter": "brief", + "filename": "compliance.log", + "backupCount": 0, } - LOGGING_CONFIG_DEFAULT['loggers']['COMPLIANCE'] = { - 'handlers': ['compliance_log'], - 'level': 'DEBUG', - 'qualname': 'COMPLIANCE' + LOGGING_CONFIG_DEFAULT["loggers"]["COMPLIANCE"] = { + "handlers": ["compliance_log"], + "level": "DEBUG", + "qualname": "COMPLIANCE", } log_destination = kwargs.get("log_destination") - galaxy_daemon_log_destination = os.environ.get('GALAXY_DAEMON_LOG') + galaxy_daemon_log_destination = os.environ.get("GALAXY_DAEMON_LOG") if log_destination == "stdout": - LOGGING_CONFIG_DEFAULT['handlers']['console'] = { - 'class': 'logging.StreamHandler', - 'formatter': 'stack', - 'level': 'DEBUG', - 'stream': 'ext://sys.stdout', - 'filters': ['stack'] + LOGGING_CONFIG_DEFAULT["handlers"]["console"] = { + "class": "logging.StreamHandler", + "formatter": "stack", + "level": "DEBUG", + "stream": "ext://sys.stdout", + "filters": ["stack"], } elif log_destination: - LOGGING_CONFIG_DEFAULT['handlers']['console'] = { - 'class': 'logging.FileHandler', - 'formatter': 'stack', - 'level': 'DEBUG', - 'filename': log_destination, - 'filters': ['stack'] + LOGGING_CONFIG_DEFAULT["handlers"]["console"] = { + "class": "logging.FileHandler", + "formatter": "stack", + "level": "DEBUG", + "filename": log_destination, + "filters": ["stack"], } if galaxy_daemon_log_destination: - LOGGING_CONFIG_DEFAULT['handlers']['files'] = { - 'class': 'logging.FileHandler', - 'formatter': 'stack', - 'level': 'DEBUG', - 'filename': galaxy_daemon_log_destination, - 'filters': ['stack'] + LOGGING_CONFIG_DEFAULT["handlers"]["files"] = { + "class": "logging.FileHandler", + "formatter": "stack", + "level": "DEBUG", + "filename": galaxy_daemon_log_destination, + "filters": ["stack"], } - LOGGING_CONFIG_DEFAULT['root']['handlers'].append('files') + LOGGING_CONFIG_DEFAULT["root"]["handlers"].append("files") def _configure_dataset_storage(self): # The default for `file_path` has changed in 20.05; we may need to fall back to the old default - self._set_alt_paths('file_path', self._in_data_dir('files')) # this is called BEFORE guessing id/uuid - ID, UUID = 'id', 'uuid' - if self.is_set('object_store_store_by'): + self._set_alt_paths("file_path", self._in_data_dir("files")) # this is called BEFORE guessing id/uuid + ID, UUID = "id", "uuid" + if self.is_set("object_store_store_by"): if self.object_store_store_by not in [ID, UUID]: - raise Exception( - f"Invalid value for object_store_store_by [{self.object_store_store_by}]" - ) + raise Exception(f"Invalid value for object_store_store_by [{self.object_store_store_by}]") elif os.path.basename(self.file_path) == "objects": self.object_store_store_by = UUID else: @@ -1057,35 +1109,41 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): def _set_galaxy_infrastructure_url(self, kwargs): # indicate if this was not set explicitly, so dependending on the context a better default # can be used (request url in a web thread, Docker parent in IE stuff, etc.) - self.galaxy_infrastructure_url_set = kwargs.get('galaxy_infrastructure_url') is not None + self.galaxy_infrastructure_url_set = kwargs.get("galaxy_infrastructure_url") is not None if "HOST_IP" in self.galaxy_infrastructure_url: - self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute({ - 'HOST_IP': socket.gethostbyname(socket.gethostname()) - }) + self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute( + {"HOST_IP": socket.gethostbyname(socket.gethostname())} + ) if "GALAXY_WEB_PORT" in self.galaxy_infrastructure_url: - port = os.environ.get('GALAXY_WEB_PORT') + port = os.environ.get("GALAXY_WEB_PORT") if not port: - raise Exception('$GALAXY_WEB_PORT set in galaxy_infrastructure_url, but environment variable not set') - self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute({ - 'GALAXY_WEB_PORT': port - }) + raise Exception("$GALAXY_WEB_PORT set in galaxy_infrastructure_url, but environment variable not set") + self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute( + {"GALAXY_WEB_PORT": port} + ) if "UWSGI_PORT" in self.galaxy_infrastructure_url: import uwsgi - http = unicodify(uwsgi.opt['http']) + + http = unicodify(uwsgi.opt["http"]) host, port = http.split(":", 1) assert port, "galaxy_infrastructure_url depends on dynamic PORT determination but port unknown" - self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute({ - 'UWSGI_PORT': port - }) + self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute( + {"UWSGI_PORT": port} + ) def reload_sanitize_allowlist(self, explicit=True): self.sanitize_allowlist = [] if not os.path.exists(self.sanitize_allowlist_file): if explicit: - log.warning("Sanitize log file explicitly specified as '%s' but does not exist, continuing with no tools allowlisted.", self.sanitize_allowlist_file) + log.warning( + "Sanitize log file explicitly specified as '%s' but does not exist, continuing with no tools allowlisted.", + self.sanitize_allowlist_file, + ) else: with open(self.sanitize_allowlist_file) as f: - self.sanitize_allowlist = sorted(line.strip() for line in f.readlines() if line.strip() and not line.startswith('#')) + self.sanitize_allowlist = sorted( + line.strip() for line in f.readlines() if line.strip() and not line.startswith("#") + ) def ensure_tempdir(self): self._ensure_directory(self.new_file_path) @@ -1118,7 +1176,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): # Check for deprecated options. for key in self.config_dict.keys(): if key in self.deprecated_options: - log.warning(f"Config option '{key}' is deprecated and will be removed in a future release. Please consult the latest version of the sample configuration file.") + log.warning( + f"Config option '{key}' is deprecated and will be removed in a future release. Please consult the latest version of the sample configuration file." + ) @staticmethod def _parse_allowed_origin_hostnames(allowed_origin_hostnames): @@ -1132,7 +1192,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): def parse(string): # a string enclosed in fwd slashes will be parsed as a regexp: e.g. // - if string[0] == '/' and string[-1] == '/': + if string[0] == "/" and string[-1] == "/": string = string[1:-1] return re.compile(string, flags=(re.UNICODE)) return string @@ -1153,24 +1213,24 @@ def reload_config_options(current_config): if current_config._raw_config[option] != modified_config[option]: current_config._raw_config[option] = modified_config[option] setattr(current_config, option, modified_config[option]) - log.info(f'Reloaded {option}') + log.info(f"Reloaded {option}") -def get_database_engine_options(kwargs, model_prefix=''): +def get_database_engine_options(kwargs, model_prefix=""): """ Allow options for the SQLAlchemy database engine to be passed by using the prefix "database_engine_option". """ conversions: Dict[str, Callable[[Any], Union[bool, int]]] = { - 'convert_unicode': string_as_bool, - 'pool_timeout': int, - 'echo': string_as_bool, - 'echo_pool': string_as_bool, - 'pool_recycle': int, - 'pool_size': int, - 'max_overflow': int, - 'pool_threadlocal': string_as_bool, - 'server_side_cursors': string_as_bool + "convert_unicode": string_as_bool, + "pool_timeout": int, + "echo": string_as_bool, + "echo_pool": string_as_bool, + "pool_recycle": int, + "pool_size": int, + "max_overflow": int, + "pool_threadlocal": string_as_bool, + "server_side_cursors": string_as_bool, } prefix = f"{model_prefix}database_engine_option_" prefix_len = len(prefix) @@ -1199,7 +1259,7 @@ def init_models_from_config(config, map_install_models=False, object_store=None, database_query_profiling_proxy=config.database_query_profiling_proxy, object_store=object_store, trace_logger=trace_logger, - use_pbkdf2=config.get_bool('use_pbkdf2', True), + use_pbkdf2=config.get_bool("use_pbkdf2", True), slow_query_log_threshold=config.slow_query_log_threshold, thread_local_log=config.thread_local_log, log_query_counts=config.database_log_query_counts, @@ -1223,19 +1283,25 @@ def configure_logging(config): paste_configures_logging = config.global_conf_parser.has_section("loggers") else: paste_configures_logging = False - auto_configure_logging = not paste_configures_logging and string_as_bool(config.get("auto_configure_logging", "True")) + auto_configure_logging = not paste_configures_logging and string_as_bool( + config.get("auto_configure_logging", "True") + ) if auto_configure_logging: - logging_conf = config.get('logging', None) + logging_conf = config.get("logging", None) if logging_conf is None: # if using the default logging config, honor the log_level setting logging_conf = LOGGING_CONFIG_DEFAULT - if config.get('log_level', 'DEBUG') != 'DEBUG': - logging_conf['handlers']['console']['level'] = config.get('log_level', 'DEBUG') + if config.get("log_level", "DEBUG") != "DEBUG": + logging_conf["handlers"]["console"]["level"] = config.get("log_level", "DEBUG") # configure logging with logging dict in config, template *FileHandler handler filenames with the `filename_template` option - for name, conf in logging_conf.get('handlers', {}).items(): - if conf['class'].startswith('logging.') and conf['class'].endswith('FileHandler') and 'filename_template' in conf: - conf['filename'] = conf.pop('filename_template').format(**get_stack_facts(config=config)) - logging_conf['handlers'][name] = conf + for name, conf in logging_conf.get("handlers", {}).items(): + if ( + conf["class"].startswith("logging.") + and conf["class"].endswith("FileHandler") + and "filename_template" in conf + ): + conf["filename"] = conf.pop("filename_template").format(**get_stack_facts(config=config)) + logging_conf["handlers"][name] = conf logging.config.dictConfig(logging_conf) @@ -1254,15 +1320,15 @@ class ConfiguresGalaxyMixin: def wait_for_toolbox_reload(self, old_toolbox): timer = ExecutionTimer() - log.debug('Waiting for toolbox reload') + log.debug("Waiting for toolbox reload") # Wait till toolbox reload has been triggered (or more than 60 seconds have passed) while timer.elapsed < 60: if self.toolbox.has_reloaded(old_toolbox): - log.debug('Finished waiting for toolbox reload %s', timer) + log.debug("Finished waiting for toolbox reload %s", timer) break time.sleep(0.1) else: - log.warning('Waiting for toolbox reload timed out after 60 seconds') + log.warning("Waiting for toolbox reload timed out after 60 seconds") def _configure_tool_config_files(self): if self.config.shed_tool_config_file not in self.config.tool_configs: @@ -1270,17 +1336,19 @@ class ConfiguresGalaxyMixin: # The value of migrated_tools_config is the file reserved for containing only those tools that have been # eliminated from the distribution and moved to the tool shed. If migration checking is disabled, only add it if # it exists (since this may be an existing deployment where migrations were previously run). - if (os.path.exists(self.config.migrated_tools_config) - and self.config.migrated_tools_config not in self.config.tool_configs): + if ( + os.path.exists(self.config.migrated_tools_config) + and self.config.migrated_tools_config not in self.config.tool_configs + ): self.config.tool_configs.append(self.config.migrated_tools_config) def _configure_toolbox(self): + import galaxy.tools.search from galaxy import tools - from galaxy.tools.biotools import get_galaxy_biotools_metadata_source from galaxy.managers.citations import CitationsManager from galaxy.tool_util.deps import containers from galaxy.tool_util.deps.dependencies import AppInfo - import galaxy.tools.search + from galaxy.tools.biotools import get_galaxy_biotools_metadata_source if not isinstance(self, BasicSharedApp): raise Exception("Must inherit from BasicSharedApp") @@ -1313,15 +1381,19 @@ class ConfiguresGalaxyMixin: mulled_resolution_cache = None if self.config.mulled_resolution_cache_type: cache_opts = { - 'cache.type': self.config.mulled_resolution_cache_type, - 'cache.data_dir': self.config.mulled_resolution_cache_data_dir, - 'cache.lock_dir': self.config.mulled_resolution_cache_lock_dir, + "cache.type": self.config.mulled_resolution_cache_type, + "cache.data_dir": self.config.mulled_resolution_cache_data_dir, + "cache.lock_dir": self.config.mulled_resolution_cache_lock_dir, } - mulled_resolution_cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache('mulled_resolution') + mulled_resolution_cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache( + "mulled_resolution" + ) self.container_finder = containers.ContainerFinder(app_info, mulled_resolution_cache=mulled_resolution_cache) self._set_enabled_container_types() index_help = getattr(self.config, "index_tool_help", True) - self.toolbox_search = galaxy.tools.search.ToolBoxSearch(self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help) + self.toolbox_search = galaxy.tools.search.ToolBoxSearch( + self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help + ) def reindex_tool_search(self): # Call this when tools are added or removed. @@ -1335,28 +1407,37 @@ class ConfiguresGalaxyMixin: for enabled_container_type in self.container_finder._enabled_container_types(destination.params): container_types_to_destinations[enabled_container_type].append(destination) self.toolbox.dependency_manager.set_enabled_container_types(container_types_to_destinations) - self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.default_container_registry.resolver_classes) - self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.default_container_registry.container_resolvers) + self.toolbox.dependency_manager.resolver_classes.update( + self.container_finder.default_container_registry.resolver_classes + ) + self.toolbox.dependency_manager.dependency_resolvers.extend( + self.container_finder.default_container_registry.container_resolvers + ) def _configure_tool_data_tables(self, from_shed_config): from galaxy.tools.data import ToolDataTableManager # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path. - self.tool_data_tables = ToolDataTableManager(tool_data_path=self.config.tool_data_path, - config_filename=self.config.tool_data_table_config_path, - other_config_dict=self.config) + self.tool_data_tables = ToolDataTableManager( + tool_data_path=self.config.tool_data_path, + config_filename=self.config.tool_data_table_config_path, + other_config_dict=self.config, + ) # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables. try: - self.tool_data_tables.load_from_config_file(config_filename=self.config.shed_tool_data_table_config, - tool_data_path=self.tool_data_tables.tool_data_path, - from_shed_config=from_shed_config) + self.tool_data_tables.load_from_config_file( + config_filename=self.config.shed_tool_data_table_config, + tool_data_path=self.tool_data_tables.tool_data_path, + from_shed_config=from_shed_config, + ) except OSError as exc: # Missing shed_tool_data_table_config is okay if it's the default - if exc.errno != errno.ENOENT or self.config.is_set('shed_tool_data_table_config'): + if exc.errno != errno.ENOENT or self.config.is_set("shed_tool_data_table_config"): raise def _configure_datatypes_registry(self, installed_repository_manager=None): from galaxy.datatypes import registry + # Create an empty datatypes registry. self.datatypes_registry = registry.Registry(self.config) if installed_repository_manager and self.config.load_tool_shed_datatypes: @@ -1377,10 +1458,12 @@ class ConfiguresGalaxyMixin: def _configure_object_store(self, **kwds): from galaxy.objectstore import build_object_store_from_config + self.object_store = build_object_store_from_config(self.config, **kwds) def _configure_security(self): from galaxy.security import idencoding + self.security = idencoding.IdEncodingHelper(id_secret=self.config.id_secret) BaseDatabaseIdField.security = self.security @@ -1399,22 +1482,34 @@ class ConfiguresGalaxyMixin: install_db_url = self.config.install_database_connection # TODO: Consider more aggressive check here that this is not the same # database file under the hood. - combined_install_database = not(install_db_url and install_db_url != db_url) + combined_install_database = not (install_db_url and install_db_url != db_url) install_db_url = install_db_url or db_url - install_database_options = self.config.database_engine_options if combined_install_database else self.config.install_database_engine_options + install_database_options = ( + self.config.database_engine_options + if combined_install_database + else self.config.install_database_engine_options + ) if self.config.database_wait: self._wait_for_database(db_url) if getattr(self.config, "max_metadata_value_size", None): from galaxy.model import custom_types + custom_types.MAX_METADATA_VALUE_SIZE = self.config.max_metadata_value_size if check_migrate_databases: # Initialize database / check for appropriate schema version. # If this # is a new installation, we'll restrict the tool migration messaging. from galaxy.model.migrate.check import create_or_verify_database - create_or_verify_database(db_url, config_file, self.config.database_engine_options, app=self, map_install_models=combined_install_database) + + create_or_verify_database( + db_url, + config_file, + self.config.database_engine_options, + app=self, + map_install_models=combined_install_database, + ) if not combined_install_database: tsi_create_or_verify_database(install_db_url, install_database_options, app=self) @@ -1422,17 +1517,17 @@ class ConfiguresGalaxyMixin: self.config, map_install_models=combined_install_database, object_store=self.object_store, - trace_logger=getattr(self, "trace_logger", None) + trace_logger=getattr(self, "trace_logger", None), ) if combined_install_database: log.info("Install database targetting Galaxy's database configuration.") self.install_model = self.model else: from galaxy.model.tool_shed_install import mapping as install_mapping + install_db_url = self.config.install_database_connection log.info(f"Install database using its own connection {install_db_url}") - self.install_model = install_mapping.init(install_db_url, - install_database_options) + self.install_model = install_mapping.init(install_db_url, install_database_options) def _configure_signal_handlers(self, handlers): for sig, handler in handlers.items(): diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 1513ba45325..80742143f6f 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -20,7 +20,7 @@ bdbag==1.6.3; (python_version >= "2.7" and python_full_version < "3.0.0") or (py beaker==1.11.0 billiard==3.6.4.0; python_version >= "3.7" bioblend==0.16.0; python_version >= "3.6" -black==22.1.0; python_full_version >= "3.6.2" and python_version >= "3.6" +black==22.1.0; python_full_version >= "3.6.2" bleach==4.1.0; python_version >= "3.6" boltons==21.0.0 boto==2.49.0 @@ -30,19 +30,19 @@ cached-property==1.5.2; python_version < "3.8" and python_version >= "3.7" celery==5.2.3; python_version >= "3.7" certifi==2021.10.8; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7" cffi==1.15.0 -charset-normalizer==2.0.11; python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4" +charset-normalizer==2.0.11; python_full_version >= "3.6.0" and python_version >= "3.7" and python_version < "4" cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") circus==0.17.1 click-didyoumean==0.3.0; python_full_version >= "3.6.2" and python_full_version < "4.0.0" and python_version >= "3.7" click-plugins==1.1.1; python_version >= "3.7" click-repl==0.2.0; python_version >= "3.7" -click==8.0.3; python_full_version >= "3.6.2" and python_full_version < "4.0.0" and python_version >= "3.7" +click==8.0.3; python_version >= "3.7" and python_full_version >= "3.6.2" and python_full_version < "4.0.0" cloudauthz==0.6.0 cloudbridge==3.0.0 colorama==0.4.4; sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.6.2" and platform_system == "Windows" and python_full_version < "4.0.0" and (python_version >= "2.7" and python_full_version < "3.0.0" and platform_system == "Windows" or python_full_version >= "3.5.0" and platform_system == "Windows") and (python_version >= "3.7" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.5.0") and (python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") or sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") and python_full_version >= "3.5.0") coloredlogs==15.0.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0" commonmark==0.9.1; python_full_version >= "3.6.2" and python_full_version < "4.0.0" -coverage==6.3; python_version >= "3.7" +coverage==6.3.1; python_version >= "3.7" cryptography==36.0.1; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7" cwltest==2.2.20210901154959; python_version >= "3.6" and python_version < "4" cwltool==3.1.20211107152837; python_version >= "3.6" and python_version < "4" @@ -70,16 +70,17 @@ gunicorn==20.1.0; python_version >= "3.5" gxformat2==0.15.0 h11==0.12.0; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.1" h5py==3.6.0; python_version >= "3.7" -httpcore==0.14.5; python_version >= "3.6" +httpcore==0.14.6; python_version >= "3.6" httpx==0.22.0; python_version >= "3.6" humanfriendly==10.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0" idna==3.3 imagesize==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -importlib-metadata==4.10.1; python_version == "3.7" and (python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") +importlib-metadata==4.10.1; python_version == "3.7" and (python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.7") and python_full_version >= "3.6.2" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") importlib-resources==5.4.0; python_version >= "3.6" iniconfig==1.1.1; python_version >= "3.7" isa-rwval==0.10.10 isodate==0.6.1; python_version >= "3.7" and python_version < "4" +isort==5.10.1; python_full_version >= "3.6.1" and python_version < "4.0" jinja2==3.0.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" jsonschema==4.4.0; python_version >= "3.7" junit-xml==1.9; python_version >= "3.6" and python_version < "4" @@ -91,7 +92,7 @@ mako==1.1.6; (python_version >= "2.7" and python_full_version < "3.0.0") or (pyt markdown-it-reporter==0.0.2 markdown==3.3.6; python_version >= "3.6" markupsafe==2.0.1; python_version >= "3.6" -mercurial==6.0.1 +mercurial==6.0.2 mirakuru==2.4.1; python_version >= "3.7" mistune==0.8.4; python_version >= "3.7" and python_version < "4" mrcfile==1.3.0 @@ -112,7 +113,7 @@ paste==3.5.0 pastedeploy==2.1.1 pathspec==0.9.0; python_full_version >= "3.6.2" and python_version >= "3.6" pbr==5.8.0; python_version >= "2.6" -platformdirs==2.4.1; python_full_version >= "3.6.2" and python_version >= "3.7" +platformdirs==2.4.1; python_version >= "3.7" and python_full_version >= "3.6.2" pluggy==1.0.0; python_version >= "3.7" port-for==0.6.1; python_version >= "3.7" prettytable==3.0.0; python_version >= "3.7" @@ -127,7 +128,7 @@ pycryptodome==3.14.0; (python_version >= "2.7" and python_full_version < "3.0.0" pydantic==1.9.0; python_full_version >= "3.6.1" pydot==1.4.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.4.0" pyeventsystem==0.1.0 -pyfaidx==0.6.3.1 +pyfaidx==0.6.4 pygithub==1.55; python_version >= "3.6" pygments==2.11.2; python_full_version >= "3.6.2" and python_full_version < "4.0.0" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") pyjwt==2.3.0; python_version >= "3.6" @@ -151,7 +152,7 @@ pytest-pythonpath==0.7.3 pytest-shard==0.1.2; python_version >= "3.6" pytest==6.2.5; python_version >= "3.6" python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.3.0" -python-irodsclient==1.1.0 +python-irodsclient==1.1.1 python-jose==3.3.0 python-multipart==0.0.5 python3-openid==3.2.0; python_version >= "3.0" @@ -166,7 +167,7 @@ repoze.lru==0.7 requests-oauthlib==1.3.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" requests-toolbelt==0.9.1; python_version >= "3.6" requests==2.27.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") -responses==0.17.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") +responses==0.18.0; python_version >= "3.7" rfc3986==1.5.0; python_version >= "3.6" rich==11.1.0; python_full_version >= "3.6.2" and python_full_version < "4.0.0" routes==2.5.1 @@ -177,7 +178,7 @@ schema-salad==8.2.20220103095339; python_version >= "3.7" and python_version < " selenium==4.1.0; python_version >= "3.7" and python_version < "4.0" setuptools-scm==5.0.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" shellescape==3.8.1; python_version >= "3.6" and python_version < "4" -six==1.16.0; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.5.0" and python_version >= "3.7" and python_version < "4" +six==1.16.0; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.3.0" and python_version >= "3.7" and python_version < "4" sniffio==1.2.0; python_version >= "3.7" and python_full_version >= "3.6.2" and python_version < "4.0" snowballstemmer==2.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" social-auth-core==4.0.3 @@ -206,7 +207,7 @@ testfixtures==6.18.3 tifffile==2021.11.2; python_version >= "3.7" tinydb==4.6.1; python_version >= "3.6" and python_version < "4.0" toml==0.10.2; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7" -tomli==2.0.0; python_full_version >= "3.6.2" and python_version >= "3.7" +tomli==2.0.0; python_version >= "3.7" and python_full_version >= "3.6.2" tornado==6.1; python_version >= "3.5" tqdm==4.62.3; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" trio-websocket==0.9.2; python_version >= "3.7" and python_version < "4.0" @@ -230,5 +231,5 @@ whoosh==2.7.4 wrapt==1.13.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" wsproto==1.0.0; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.1" yacman==0.8.4 -zipp==3.7.0; python_version == "3.7" and (python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.7") +zipp==3.7.0; python_version == "3.7" and (python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.7") and python_full_version >= "3.6.2" zipstream-new==1.1.8 diff --git a/lib/galaxy/dependencies/lint-requirements.txt b/lib/galaxy/dependencies/lint-requirements.txt index 4360f8e7166..b629fbe1728 100644 --- a/lib/galaxy/dependencies/lint-requirements.txt +++ b/lib/galaxy/dependencies/lint-requirements.txt @@ -1,6 +1,5 @@ flake8 flake8-bugbear -flake8-import-order mypy==0.910 types-bleach types-boto diff --git a/lib/galaxy/dependencies/pinned-lint-requirements.txt b/lib/galaxy/dependencies/pinned-lint-requirements.txt index f6212039a2f..f0ee2ef900f 100644 --- a/lib/galaxy/dependencies/pinned-lint-requirements.txt +++ b/lib/galaxy/dependencies/pinned-lint-requirements.txt @@ -1,7 +1,6 @@ attrs==21.4.0 flake8==4.0.1 flake8-bugbear==22.1.11 -flake8-import-order==0.18.1 importlib-metadata==4.2.0 mccabe==0.6.1 mypy==0.910 diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 1aaed9f9450..808cc037714 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -75,7 +75,7 @@ lxml==4.7.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (pyt mako==1.1.6; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") markdown==3.3.6; python_version >= "3.6" markupsafe==2.0.1; python_version >= "3.6" -mercurial==6.0.1 +mercurial==6.0.2 mistune==0.8.4; python_version >= "3.7" and python_version < "4" mrcfile==1.3.0 msgpack==1.0.3; python_version >= "3.7" and python_version < "4" @@ -103,7 +103,7 @@ pycryptodome==3.14.0; (python_version >= "2.7" and python_full_version < "3.0.0" pydantic==1.9.0; python_full_version >= "3.6.1" pydot==1.4.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.4.0" pyeventsystem==0.1.0 -pyfaidx==0.6.3.1 +pyfaidx==0.6.4 pygments==2.11.2; python_full_version >= "3.6.2" and python_full_version < "4.0.0" and python_version >= "3.5" pyjwt==2.3.0; python_version >= "3.6" pykwalify==1.8.0 diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index ec4e6adc6c9..fa306149b7c 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -13,7 +13,12 @@ import sys import time import traceback from json import loads -from typing import Any, Dict, List, TYPE_CHECKING +from typing import ( + Any, + Dict, + List, + TYPE_CHECKING, +) import packaging.version import yaml @@ -36,11 +41,10 @@ from galaxy.job_execution.output_collect import ( collect_extra_files, collect_shrinked_content_from_path, ) -from galaxy.job_execution.setup import ( # noqa: F401 +from galaxy.job_execution.setup import ( # noqa: F401; This is read by certain misbehaving tool wrappers that import Galaxy internals create_working_directory_for_job, ensure_configs_directory, JobIO, - # This is read by certain misbehaving tool wrappers that import Galaxy internals TOOL_PROVIDED_JOB_METADATA_FILE, TOOL_PROVIDED_JOB_METADATA_KEYS, ) @@ -48,7 +52,10 @@ from galaxy.jobs.mapper import ( JobMappingException, JobRunnerMapper, ) -from galaxy.jobs.runners import BaseJobRunner, JobState +from galaxy.jobs.runners import ( + BaseJobRunner, + JobState, +) from galaxy.metadata import get_metadata_compute_strategy from galaxy.model import store from galaxy.model.store.discover import MaxDiscoveredFilesExceededError @@ -67,7 +74,7 @@ from galaxy.util import ( parse_xml_string, RWXRWXRWX, safe_makedirs, - unicodify + unicodify, ) from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext @@ -82,7 +89,7 @@ if TYPE_CHECKING: log = logging.getLogger(__name__) # Override with config.default_job_shell. -DEFAULT_JOB_SHELL = '/bin/bash' +DEFAULT_JOB_SHELL = "/bin/bash" DEFAULT_LOCAL_WORKERS = 4 DEFAULT_CLEANUP_JOB = "always" @@ -95,22 +102,22 @@ class JobDestination(Bunch): """ def __init__(self, **kwds): - self['id'] = None - self['url'] = None - self['tags'] = None - self['runner'] = None - self['legacy'] = False - self['converted'] = False - self['shell'] = None - self['env'] = [] - self['resubmit'] = [] + self["id"] = None + self["url"] = None + self["tags"] = None + self["runner"] = None + self["legacy"] = False + self["converted"] = False + self["shell"] = None + self["env"] = [] + self["resubmit"] = [] # dict is appropriate (rather than a bunch) since keys may not be valid as attributes - self['params'] = dict() + self["params"] = dict() # Use the values persisted in an existing job - if 'from_job' in kwds and kwds['from_job'].destination_id is not None: - self['id'] = kwds['from_job'].destination_id - self['params'] = kwds['from_job'].destination_params + if "from_job" in kwds and kwds["from_job"].destination_id is not None: + self["id"] = kwds["from_job"].destination_id + self["params"] = kwds["from_job"].destination_params super().__init__(**kwds) @@ -124,9 +131,9 @@ class JobToolConfiguration(Bunch): """ def __init__(self, **kwds): - self['handler'] = None - self['destination'] = None - self['params'] = dict() + self["handler"] = None + self["destination"] = None + self["params"] = dict() super().__init__(**kwds) def get_resource_group(self): @@ -135,8 +142,8 @@ class JobToolConfiguration(Bunch): def config_exception(e, file): abs_path = os.path.abspath(file) - message = f'Problem parsing the XML in file {abs_path}, ' - message += 'please correct the indicated portion of the file and restart Galaxy. ' + message = f"Problem parsing the XML in file {abs_path}, " + message += "please correct the indicated portion of the file and restart Galaxy. " message += unicodify(e) log.exception(message) return Exception(message) @@ -149,23 +156,20 @@ def job_config_xml_to_dict(config, root): config_dict["runners"] = runners # Parser plugins section populate 'runners' and 'dynamic' in config_dict. - plugins = root.find('plugins') + plugins = root.find("plugins") if plugins is not None: - for plugin in ConfiguresHandlers._findall_with_required(plugins, 'plugin', ('id', 'type', 'load')): - if plugin.get('type') == 'runner': - workers = plugin.get('workers', plugins.get('workers', JobConfiguration.DEFAULT_NWORKERS)) + for plugin in ConfiguresHandlers._findall_with_required(plugins, "plugin", ("id", "type", "load")): + if plugin.get("type") == "runner": + workers = plugin.get("workers", plugins.get("workers", JobConfiguration.DEFAULT_NWORKERS)) runner_kwds = JobConfiguration.get_params(config, plugin) - plugin_id = plugin.get('id') - runner_info = dict(id=plugin_id, - load=plugin.get('load'), - workers=int(workers), - kwds=runner_kwds) + plugin_id = plugin.get("id") + runner_info = dict(id=plugin_id, load=plugin.get("load"), workers=int(workers), kwds=runner_kwds) runners[plugin_id] = runner_info else: log.error(f"Unknown plugin type: {plugin.get('type')}") - for plugin in ConfiguresHandlers._findall_with_required(plugins, 'plugin', ('id', 'type')): - if plugin.get('id') == 'dynamic' and plugin.get('type') == 'runner': + for plugin in ConfiguresHandlers._findall_with_required(plugins, "plugin", ("id", "type")): + if plugin.get("id") == "dynamic" and plugin.get("type") == "runner": config_dict["dynamic"] = JobConfiguration.get_params(config, plugin) handling_config_dict = ConfiguresHandlers.xml_to_dict(config, root.find("handlers")) @@ -174,9 +178,9 @@ def job_config_xml_to_dict(config, root): # Parse destinations environments = [] - destinations = root.find('destinations') - for destination in ConfiguresHandlers._findall_with_required(destinations, 'destination', ('id', 'runner')): - destination_id = destination.get('id') + destinations = root.find("destinations") + for destination in ConfiguresHandlers._findall_with_required(destinations, "destination", ("id", "runner")): + destination_id = destination.get("id") destination_metrics = destination.get("metrics", None) environment = {"id": destination_id} @@ -188,9 +192,9 @@ def job_config_xml_to_dict(config, root): else: metrics_to_dict = {"src": "path", "path": destination_metrics} else: - metrics_elements = ConfiguresHandlers._findall_with_required(destination, 'job_metrics', ()) + metrics_elements = ConfiguresHandlers._findall_with_required(destination, "job_metrics", ()) if metrics_elements: - metrics_to_dict = {"src": "xml_element", 'xml_element': metrics_elements[0]} + metrics_to_dict = {"src": "xml_element", "xml_element": metrics_elements[0]} environment["metrics"] = metrics_to_dict @@ -200,45 +204,45 @@ def job_config_xml_to_dict(config, root): params["docker_sudo"] = "true" # TODO: handle enabled/disabled in configure_from - environment['params'] = params - environment['env'] = JobConfiguration.get_envs(destination) + environment["params"] = params + environment["env"] = JobConfiguration.get_envs(destination) destination_resubmits = JobConfiguration.get_resubmits(destination) if destination_resubmits: - environment['resubmit'] = destination_resubmits + environment["resubmit"] = destination_resubmits # TODO: handle empty resubmits defaults in configure_from - runner = destination.get('runner') + runner = destination.get("runner") if runner: - environment['runner'] = runner + environment["runner"] = runner - tags = destination.get('tags') + tags = destination.get("tags") # Store tags as a list if tags is not None: - tags = [x.strip() for x in tags.split(',')] - environment['tags'] = tags + tags = [x.strip() for x in tags.split(",")] + environment["tags"] = tags environments.append(environment) - config_dict['execution'] = { - 'environments': environments, + config_dict["execution"] = { + "environments": environments, } default_destination = ConfiguresHandlers.get_xml_default(config, destinations) if default_destination: - config_dict['execution']['default'] = default_destination + config_dict["execution"]["default"] = default_destination resources_config_dict = {} resource_groups = {} # Parse resources... - resources = root.find('resources') + resources = root.find("resources") if resources is not None: default_resource_group = resources.get("default", None) if default_resource_group: resources_config_dict["default"] = default_resource_group - for group in ConfiguresHandlers._findall_with_required(resources, 'group'): - group_id = group.get('id') - fields_str = group.get('fields', None) or group.text or '' + for group in ConfiguresHandlers._findall_with_required(resources, "group"): + group_id = group.get("id") + fields_str = group.get("fields", None) or group.text or "" fields = [f for f in fields_str.split(",") if f] resource_groups[group_id] = fields @@ -246,36 +250,36 @@ def job_config_xml_to_dict(config, root): config_dict["resources"] = resources_config_dict # Parse tool mappings - tools = root.find('tools') - config_dict['tools'] = [] + tools = root.find("tools") + config_dict["tools"] = [] if tools is not None: - for tool in tools.findall('tool'): + for tool in tools.findall("tool"): # There can be multiple definitions with identical ids, but different params tool_mapping_conf = {} - for key in ['handler', 'destination', 'id', 'resources', 'class']: + for key in ["handler", "destination", "id", "resources", "class"]: value = tool.get(key) if value: if key == "destination": key = "environment" tool_mapping_conf[key] = value tool_mapping_conf["params"] = JobConfiguration.get_params(config, tool) - config_dict['tools'].append(tool_mapping_conf) + config_dict["tools"].append(tool_mapping_conf) limits_config = [] - limits = root.find('limits') + limits = root.find("limits") if limits is not None: - for limit in JobConfiguration._findall_with_required(limits, 'limit', ('type',)): + for limit in JobConfiguration._findall_with_required(limits, "limit", ("type",)): limit_dict = {} - for key in ['type', 'tag', 'id', 'window']: - if key == 'type' and key.startswith('destination_'): + for key in ["type", "tag", "id", "window"]: + if key == "type" and key.startswith("destination_"): key = f"environment_{key[len('destination_'):]}" value = limit.get(key) if value: limit_dict[key] = value - limit_dict['value'] = limit.text + limit_dict["value"] = limit.text limits_config.append(limit_dict) - config_dict['limits'] = limits_config + config_dict["limits"] = limits_config return config_dict @@ -284,6 +288,7 @@ class JobConfiguration(ConfiguresHandlers): These features are configured in the job configuration, by default, ``job_conf.xml`` """ + runner_plugins: List[dict] handlers: dict handler_runner_plugins: Dict[str, str] @@ -292,7 +297,7 @@ class JobConfiguration(ConfiguresHandlers): resource_groups: Dict[str, list] destinations: Dict[str, tuple] resource_parameters: Dict[str, Any] - DEFAULT_BASE_HANDLER_POOLS = ('job-handlers',) + DEFAULT_BASE_HANDLER_POOLS = ("job-handlers",) DEFAULT_NWORKERS = 4 @@ -308,8 +313,7 @@ class JobConfiguration(ConfiguresHandlers): """ def __init__(self, app: MinimalManagerApp): - """Parse the job configuration XML. - """ + """Parse the job configuration XML.""" self.app = app self.runner_plugins = [] self.dynamic_params = None @@ -327,34 +331,38 @@ class JobConfiguration(ConfiguresHandlers): self.resource_groups = {} self.default_resource_group = None self.resource_parameters = {} - self.limits = Bunch(registered_user_concurrent_jobs=None, - anonymous_user_concurrent_jobs=None, - walltime=None, - walltime_delta=None, - total_walltime={}, - output_size=None, - destination_user_concurrent_jobs={}, - destination_total_concurrent_jobs={}) + self.limits = Bunch( + registered_user_concurrent_jobs=None, + anonymous_user_concurrent_jobs=None, + walltime=None, + walltime_delta=None, + total_walltime={}, + output_size=None, + destination_user_concurrent_jobs={}, + destination_total_concurrent_jobs={}, + ) default_resubmits = [] default_resubmit_condition = self.app.config.default_job_resubmission_condition if default_resubmit_condition: - default_resubmits.append(dict( - environment=None, - condition=default_resubmit_condition, - handler=None, - delay=None, - )) + default_resubmits.append( + dict( + environment=None, + condition=default_resubmit_condition, + handler=None, + delay=None, + ) + ) self.default_resubmits = default_resubmits self.__parse_resource_parameters() # Initialize the config try: - if 'job_config' in self.app.config.config_dict: + if "job_config" in self.app.config.config_dict: job_config_dict = self.app.config.config_dict["job_config"] else: job_config_file = self.app.config.job_config_file - if '.xml' in job_config_file: + if ".xml" in job_config_file: tree = load(job_config_file) job_config_dict = self.__parse_job_conf_xml(tree) else: @@ -363,15 +371,19 @@ class JobConfiguration(ConfiguresHandlers): # Load tasks if configured if self.app.config.use_tasked_jobs: - job_config_dict["runners"]["tasks"] = dict(id='tasks', load='tasks', workers=self.app.config.local_task_queue_workers, kwds={}) + job_config_dict["runners"]["tasks"] = dict( + id="tasks", load="tasks", workers=self.app.config.local_task_queue_workers, kwds={} + ) self._configure_from_dict(job_config_dict) - log.debug('Done loading job configuration') + log.debug("Done loading job configuration") except OSError: - log.warning('Job configuration "%s" does not exist, using default job configuration', - self.app.config.job_config_file) + log.warning( + 'Job configuration "%s" does not exist, using default job configuration', + self.app.config.job_config_file, + ) self.__set_default_job_conf() except Exception as e: raise config_exception(e, job_config_file) @@ -383,7 +395,7 @@ class JobConfiguration(ConfiguresHandlers): # with a flat dictionary. kwds = {} for key, value in runner_info.items(): - if key in ['id', 'load', 'workers']: + if key in ["id", "load", "workers"]: continue kwds[key] = value runner_info["kwds"] = kwds @@ -392,7 +404,7 @@ class JobConfiguration(ConfiguresHandlers): continue runner_info["id"] = runner_id if runner_id == "dynamic": - log.warning('Deprecated treatment of dynamic running configuration as an actual job runner.') + log.warning("Deprecated treatment of dynamic running configuration as an actual job runner.") self.dynamic_params = runner_info["kwds"] continue self.runner_plugins.append(runner_info) @@ -407,17 +419,20 @@ class JobConfiguration(ConfiguresHandlers): self._set_default_handler_assignment_methods() else: self.app.application_stack.init_job_handling(self) - log.info("Job handler assignment methods set to: %s", ', '.join(self.handler_assignment_methods)) + log.info("Job handler assignment methods set to: %s", ", ".join(self.handler_assignment_methods)) for tag, handlers in [(t, h) for t, h in self.handlers.items() if isinstance(h, list)]: - log.info("Tag [%s] handlers: %s", tag, ', '.join(handlers)) - self.handler_ready_window_size = int(handling_config_dict.get( - 'ready_window_size', JobConfiguration.DEFAULT_HANDLER_READY_WINDOW_SIZE)) + log.info("Tag [%s] handlers: %s", tag, ", ".join(handlers)) + self.handler_ready_window_size = int( + handling_config_dict.get("ready_window_size", JobConfiguration.DEFAULT_HANDLER_READY_WINDOW_SIZE) + ) # Parse environments job_metrics = self.app.job_metrics - execution_dict = job_config_dict.get('execution', {}) + execution_dict = job_config_dict.get("execution", {}) environments = execution_dict.get("environments", []) - enviroment_iter = map(lambda e: (e["id"], e), environments) if isinstance(environments, list) else environments.items() + enviroment_iter = ( + map(lambda e: (e["id"], e), environments) if isinstance(environments, list) else environments.items() + ) for environment_id, environment_dict in enviroment_iter: metrics = environment_dict.get("metrics") if metrics is None: @@ -445,12 +460,12 @@ class JobConfiguration(ConfiguresHandlers): # allowing a flat configuration of these things. params = {} for key, value in environment_dict.items(): - if key in ['id', 'tags', 'runner', 'shell', 'env', 'resubmit']: + if key in ["id", "tags", "runner", "shell", "env", "resubmit"]: continue params[key] = value environment_dict["params"] = params - for key in ['tags', 'runner', 'shell', 'env', 'resubmit', 'params']: + for key in ["tags", "runner", "shell", "env", "resubmit", "params"]: if key in environment_dict: destination_kwds[key] = environment_dict[key] destination_kwds["id"] = environment_id @@ -470,7 +485,9 @@ class JobConfiguration(ConfiguresHandlers): self.destinations[tag].append(job_destination) # Determine the default destination - self.default_destination_id = self._ensure_default_set(execution_dict.get("default"), list(self.destinations.keys()), auto=True) + self.default_destination_id = self._ensure_default_set( + execution_dict.get("default"), list(self.destinations.keys()), auto=True + ) # Read in resources resources = job_config_dict.get("resources", {}) @@ -478,13 +495,13 @@ class JobConfiguration(ConfiguresHandlers): for group_id, fields in resources.get("groups", {}).items(): self.resource_groups[group_id] = fields - tools = job_config_dict.get('tools', []) + tools = job_config_dict.get("tools", []) for tool in tools: - raw_tool_id = tool.get('id') - tool_class = tool.get('class') + raw_tool_id = tool.get("id") + tool_class = tool.get("class") if raw_tool_id is not None: assert tool_class is None - tool_id = raw_tool_id.lower().rstrip('/') + tool_id = raw_tool_id.lower().rstrip("/") if tool_id not in self.tools: self.tools[tool_id] = list() else: @@ -509,46 +526,45 @@ class JobConfiguration(ConfiguresHandlers): else: self.tool_classes[tool_class].append(jtc) - types = dict(registered_user_concurrent_jobs=int, - anonymous_user_concurrent_jobs=int, - walltime=str, - total_walltime=str, - output_size=util.size_to_bytes) + types = dict( + registered_user_concurrent_jobs=int, + anonymous_user_concurrent_jobs=int, + walltime=str, + total_walltime=str, + output_size=util.size_to_bytes, + ) # Parse job limits for limit_dict in job_config_dict.get("limits", []): - limit_type = limit_dict.get('type') + limit_type = limit_dict.get("type") if limit_type.startswith("environment_"): limit_type = f"destination_{limit_type[len('environment_'):]}" limit_value = limit_dict.get("value") # concurrent_jobs renamed to destination_user_concurrent_jobs in job_conf.xml - if limit_type in ('destination_user_concurrent_jobs', 'concurrent_jobs', 'destination_total_concurrent_jobs'): - id = limit_dict.get('tag', None) or limit_dict.get('id') - if limit_type == 'destination_total_concurrent_jobs': + if limit_type in ( + "destination_user_concurrent_jobs", + "concurrent_jobs", + "destination_total_concurrent_jobs", + ): + id = limit_dict.get("tag", None) or limit_dict.get("id") + if limit_type == "destination_total_concurrent_jobs": self.limits.destination_total_concurrent_jobs[id] = int(limit_value) else: self.limits.destination_user_concurrent_jobs[id] = int(limit_value) - elif limit_type == 'total_walltime': - self.limits.total_walltime["window"] = ( - int(limit_dict.get('window')) or 30 - ) - self.limits.total_walltime["raw"] = ( - types.get(limit_type, str)(limit_value) - ) + elif limit_type == "total_walltime": + self.limits.total_walltime["window"] = int(limit_dict.get("window")) or 30 + self.limits.total_walltime["raw"] = types.get(limit_type, str)(limit_value) elif limit_value: self.limits.__dict__[limit_type] = types.get(limit_type, str)(limit_value) if self.limits.walltime is not None: - h, m, s = (int(v) for v in self.limits.walltime.split(':')) + h, m, s = (int(v) for v in self.limits.walltime.split(":")) self.limits.walltime_delta = datetime.timedelta(0, s, 0, 0, m, h) if "raw" in self.limits.total_walltime: - h, m, s = (int(v) for v in - self.limits.total_walltime["raw"].split(':')) - self.limits.total_walltime["delta"] = datetime.timedelta( - 0, s, 0, 0, m, h - ) + h, m, s = (int(v) for v in self.limits.total_walltime["raw"].split(":")) + self.limits.total_walltime["delta"] = datetime.timedelta(0, s, 0, 0, m, h) def __parse_job_conf_xml(self, tree): """Loads the new-style job configuration from options in the job config file (by default, job_conf.xml). @@ -557,7 +573,7 @@ class JobConfiguration(ConfiguresHandlers): :type tree: ``lxml.etree._Element`` """ root = tree.getroot() - log.debug(f'Loading job configuration from {self.app.config.job_config_file}') + log.debug(f"Loading job configuration from {self.app.config.job_config_file}") job_config_dict = job_config_xml_to_dict(self.app.config, root) return job_config_dict @@ -570,10 +586,10 @@ class JobConfiguration(ConfiguresHandlers): def __set_default_job_conf(self): # Run jobs locally - self.runner_plugins = [dict(id='local', load='local', workers=DEFAULT_LOCAL_WORKERS)] + self.runner_plugins = [dict(id="local", load="local", workers=DEFAULT_LOCAL_WORKERS)] # Load tasks if configured if self.app.config.use_tasked_jobs: - self.runner_plugins.append(dict(id='tasks', load='tasks', workers=DEFAULT_LOCAL_WORKERS)) + self.runner_plugins.append(dict(id="tasks", load="tasks", workers=DEFAULT_LOCAL_WORKERS)) # Set the handlers self._init_handler_assignment_methods() if not self.handler_assignment_methods_configured: @@ -582,12 +598,12 @@ class JobConfiguration(ConfiguresHandlers): self.app.application_stack.init_job_handling(self) self.handler_ready_window_size = JobConfiguration.DEFAULT_HANDLER_READY_WINDOW_SIZE # Set the destination - self.default_destination_id = 'local' - self.destinations['local'] = [JobDestination(id='local', runner='local')] - log.debug('Done loading job configuration') + self.default_destination_id = "local" + self.destinations["local"] = [JobDestination(id="local", runner="local")] + log.debug("Done loading job configuration") def get_tool_resource_xml(self, tool_id, tool_type): - """ Given a tool id, return XML elements describing parameters to + """Given a tool id, return XML elements describing parameters to insert into job resources. :tool id: A tool ID (a string) @@ -595,7 +611,7 @@ class JobConfiguration(ConfiguresHandlers): :returns: List of parameter elements. """ - if tool_id and tool_type in ('default', 'manage_data'): + if tool_id and tool_type in ("default", "manage_data"): # TODO: Only works with exact matches, should handle different kinds of ids # the way destination lookup does. resource_group = None @@ -615,7 +631,7 @@ class JobConfiguration(ConfiguresHandlers): if fields: conditional_element = parse_xml_string(self.JOB_RESOURCE_CONDITIONAL_XML) - when_yes_elem = conditional_element.findall('when')[1] + when_yes_elem = conditional_element.findall("when")[1] for parameter in fields: when_yes_elem.append(parameter) return conditional_element @@ -626,19 +642,19 @@ class JobConfiguration(ConfiguresHandlers): @staticmethod def get_params(config, parent): rval = {} - for param in parent.findall('param'): - key = param.get('id') + for param in parent.findall("param"): + key = param.get("id") if key in ["container", "container_override"]: - containers = map(requirements.container_from_element, param.findall('container')) + containers = map(requirements.container_from_element, param.findall("container")) param_value = list(map(lambda c: c.to_dict(), containers)) else: param_value = param.text - if 'from_environ' in param.attrib: - environ_var = param.attrib['from_environ'] + if "from_environ" in param.attrib: + environ_var = param.attrib["from_environ"] param_value = os.environ.get(environ_var, param_value) - elif 'from_config' in param.attrib: - config_val = param.attrib['from_config'] + elif "from_config" in param.attrib: + config_val = param.attrib["from_config"] param_value = config.config_dict.get(config_val, param_value) rval[key] = param_value @@ -664,14 +680,16 @@ class JobConfiguration(ConfiguresHandlers): :returns: dict """ rval = [] - for param in parent.findall('env'): - rval.append(dict( - name=param.get('id'), - file=param.get('file'), - execute=param.get('exec'), - value=param.text, - raw=util.asbool(param.get('raw', 'false')) - )) + for param in parent.findall("env"): + rval.append( + dict( + name=param.get("id"), + file=param.get("file"), + execute=param.get("exec"), + value=param.text, + raw=util.asbool(param.get("raw", "false")), + ) + ) return rval @staticmethod @@ -684,13 +702,15 @@ class JobConfiguration(ConfiguresHandlers): :returns: dict """ rval = [] - for resubmit in parent.findall('resubmit'): - rval.append(dict( - condition=resubmit.get('condition'), - environment=resubmit.get('destination'), - handler=resubmit.get('handler'), - delay=resubmit.get('delay'), - )) + for resubmit in parent.findall("resubmit"): + rval.append( + dict( + condition=resubmit.get("condition"), + environment=resubmit.get("destination"), + handler=resubmit.get("handler"), + delay=resubmit.get("delay"), + ) + ) return rval def __is_enabled(self, params): @@ -711,7 +731,9 @@ class JobConfiguration(ConfiguresHandlers): :returns: JobToolConfiguration -- a representation of a element that uses the default handler and destination """ - return JobToolConfiguration(id='_default_', handler=self.default_handler_id, destination=self.default_destination_id) + return JobToolConfiguration( + id="_default_", handler=self.default_handler_id, destination=self.default_destination_id + ) # Called upon instantiation of a Tool object def get_job_tool_configurations(self, ids, tool_classes): @@ -793,24 +815,28 @@ class JobConfiguration(ConfiguresHandlers): """ rval = {} if handler_id in self.handler_runner_plugins: - plugins_to_load = [rp for rp in self.runner_plugins if rp['id'] in self.handler_runner_plugins[handler_id]] - log.info("Handler '%s' will load specified runner plugins: %s", handler_id, ', '.join(rp['id'] for rp in plugins_to_load)) + plugins_to_load = [rp for rp in self.runner_plugins if rp["id"] in self.handler_runner_plugins[handler_id]] + log.info( + "Handler '%s' will load specified runner plugins: %s", + handler_id, + ", ".join(rp["id"] for rp in plugins_to_load), + ) else: plugins_to_load = self.runner_plugins log.info("Handler '%s' will load all configured runner plugins", handler_id) for runner in plugins_to_load: class_names = [] module = None - id = runner['id'] - load = runner['load'] - if ':' in load: + id = runner["id"] + load = runner["load"] + if ":" in load: # Name to load was specified as ':' - module_name, class_name = load.rsplit(':', 1) + module_name, class_name = load.rsplit(":", 1) class_names = [class_name] module = __import__(module_name) else: # Name to load was specified as '' - if '.' not in load: + if "." not in load: # For legacy reasons, try from galaxy.jobs.runners first if there's no '.' in the name module_name = f"galaxy.jobs.runners.{load}" try: @@ -844,13 +870,20 @@ class JobConfiguration(ConfiguresHandlers): log.warning(f"A non-class name was found in __all__, ignoring: {id}") continue except AssertionError: - log.warning(f"Job runner classes must be subclassed from BaseJobRunner, {id} has bases: {runner_class.__bases__}") + log.warning( + f"Job runner classes must be subclassed from BaseJobRunner, {id} has bases: {runner_class.__bases__}" + ) continue try: - rval[id] = runner_class(self.app, runner.get('workers', JobConfiguration.DEFAULT_NWORKERS), **runner.get('kwds', {})) + rval[id] = runner_class( + self.app, runner.get("workers", JobConfiguration.DEFAULT_NWORKERS), **runner.get("kwds", {}) + ) except TypeError: - log.exception("Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load", - module_name, class_name) + log.exception( + "Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load", + module_name, + class_name, + ) rval[id] = runner_class(self.app) log.debug(f"Loaded job runner '{module_name}:{class_name}' as '{id}'") return rval @@ -881,7 +914,9 @@ class JobConfiguration(ConfiguresHandlers): :param job_runners: All loaded job runner plugins. :type job_runners: list of job runner plugins """ - for id, destination in [(id, destinations[0]) for id, destinations in self.destinations.items() if self.is_id(destinations)]: + for id, destination in [ + (id, destinations[0]) for id, destinations in self.destinations.items() if self.is_id(destinations) + ]: # Only need to deal with real destinations, not members of tags if destination.legacy and not destination.converted: if destination.runner in job_runners: @@ -894,11 +929,12 @@ class JobConfiguration(ConfiguresHandlers): else: log.debug(f"Legacy destination with id '{id}', url '{destination.url}' converted, got params:") else: - log.warning(f"Legacy destination with id '{id}' could not be converted: Unknown runner plugin: {destination.runner}") + log.warning( + f"Legacy destination with id '{id}' could not be converted: Unknown runner plugin: {destination.runner}" + ) class HasResourceParameters: - def get_resource_parameters(self, job=None): # Find the dymically inserted resource parameters and give them # to rule. @@ -925,9 +961,10 @@ class JobWrapper(HasResourceParameters): Wraps a 'model.Job' with convenience methods for running processes and state management. """ + is_task = False - def __init__(self, job, queue: 'JobHandlerQueue', use_persisted_destination=False): + def __init__(self, job, queue: "JobHandlerQueue", use_persisted_destination=False): self.job_id = job.id self.session_id = job.session_id self.user_id = job.user_id @@ -972,19 +1009,28 @@ class JobWrapper(HasResourceParameters): def external_output_metadata(self): if self.__external_output_metadata is None: try: - metadata_strategy_override = self.get_destination_configuration('metadata_strategy', None) + metadata_strategy_override = self.get_destination_configuration("metadata_strategy", None) except JobMappingException: metadata_strategy_override = None if self.__has_tasks: metadata_strategy_override = "directory" - self.__external_output_metadata = get_metadata_compute_strategy(self.app.config, self.job_id, metadata_strategy_override=metadata_strategy_override, tool_id=self.tool.id) + self.__external_output_metadata = get_metadata_compute_strategy( + self.app.config, + self.job_id, + metadata_strategy_override=metadata_strategy_override, + tool_id=self.tool.id, + ) return self.__external_output_metadata @property def remote_command_line(self): - use_remote = self.get_destination_configuration('tool_evaluation_strategy') == 'remote' + use_remote = self.get_destination_configuration("tool_evaluation_strategy") == "remote" # It wouldn't be hard to support history export, but we want to do this in task queue workers anyway ... - return use_remote and self.external_output_metadata.extended and not self.sa_session.query(model.JobExportHistoryArchive).filter_by(job=self.get_job()).first() + return ( + use_remote + and self.external_output_metadata.extended + and not self.sa_session.query(model.JobExportHistoryArchive).filter_by(job=self.get_job()).first() + ) def tool_directory(self): tool_dir = self.tool.tool_dir @@ -1028,8 +1074,7 @@ class JobWrapper(HasResourceParameters): @property def outputs_directory(self): - """Default location of ``outputs_to_working_directory``. - """ + """Default location of ``outputs_to_working_directory``.""" return None if self.created_with_galaxy_version < packaging.version.parse("20.01") else "outputs" @property @@ -1052,8 +1097,7 @@ class JobWrapper(HasResourceParameters): @property def cleanup_job(self): - """ Remove the job after it is complete, should return "always", "onsuccess", or "never". - """ + """Remove the job after it is complete, should return "always", "onsuccess", or "never".""" return self.get_destination_configuration("cleanup_job", DEFAULT_CLEANUP_JOB) @property @@ -1062,14 +1106,14 @@ class JobWrapper(HasResourceParameters): @property def use_metadata_binary(self): - return util.asbool(self.get_destination_configuration('use_metadata_binary', "False")) + return util.asbool(self.get_destination_configuration("use_metadata_binary", "False")) def can_split(self): # Should the job handler split this job up? return self.app.config.use_tasked_jobs and self.tool.parallelism def get_job_runner_url(self): - log.warning(f'({self.job_id}) Job runner URLs are deprecated, use destinations instead.') + log.warning(f"({self.job_id}) Job runner URLs are deprecated, use destinations instead.") return self.job_destination.url def get_parallelism(self): @@ -1077,7 +1121,7 @@ class JobWrapper(HasResourceParameters): @property def shell(self): - return self.job_destination.shell or getattr(self.app.config, 'default_job_shell', DEFAULT_JOB_SHELL) + return self.job_destination.shell or getattr(self.app.config, "default_job_shell", DEFAULT_JOB_SHELL) def disable_commands_in_new_shell(self): """Provide an extension point to disable this isolation, @@ -1101,7 +1145,7 @@ class JobWrapper(HasResourceParameters): @property def galaxy_virtual_env(self): - return os.environ.get('VIRTUAL_ENV', None) + return os.environ.get("VIRTUAL_ENV", None) # legacy naming get_job_runner = get_job_runner_url @@ -1157,10 +1201,9 @@ class JobWrapper(HasResourceParameters): return os.path.abspath(os.path.join(self.working_directory, COMMAND_VERSION_FILENAME)) def __prepare_upload_paramfile(self, job): - """Special case paramfile handling for the upload tool. Copies the paramfile to the working directory - """ - new = os.path.join(self.working_directory, 'upload_params.json') - param_file_path = json.loads(next(iter(param.value for param in job.parameters if param.name == 'paramfile'))) + """Special case paramfile handling for the upload tool. Copies the paramfile to the working directory""" + new = os.path.join(self.working_directory, "upload_params.json") + param_file_path = json.loads(next(iter(param.value for param in job.parameters if param.name == "paramfile"))) try: shutil.copy2(param_file_path, new) except OSError as exc: @@ -1187,13 +1230,18 @@ class JobWrapper(HasResourceParameters): return self.sa_session.query(model.GenomeIndexToolData).filter_by(job=job).first() # TODO: The upload tool actions that create the paramfile can probably be turned in to a configfile to remove this special casing - if job.tool_id == 'upload1': + if job.tool_id == "upload1": self.__prepare_upload_paramfile(job) tool_evaluator = self._get_tool_evaluator(job) compute_environment = compute_environment or self.default_compute_environment(job) tool_evaluator.set_compute_environment(compute_environment, get_special=get_special) - self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables = tool_evaluator.build() + ( + self.command_line, + self.version_command_line, + self.extra_filenames, + self.environment_variables, + ) = tool_evaluator.build() job.command_line = self.command_line self.interactivetools = tool_evaluator.populate_interactivetools() self.app.interactivetool_manager.create_interactivetool(job, self.tool, self.interactivetools) @@ -1202,11 +1250,13 @@ class JobWrapper(HasResourceParameters): self.galaxy_lib_dir if self.tool.requires_galaxy_python_environment or self.remote_command_line: # These tools (upload, metadata, data_source) may need access to the datatypes registry. - self.app.datatypes_registry.to_xml_file(os.path.join(self.working_directory, 'registry.xml')) + self.app.datatypes_registry.to_xml_file(os.path.join(self.working_directory, "registry.xml")) if self.remote_command_line: - os.makedirs(os.path.join(self.working_directory, 'metadata', 'outputs_new'), exist_ok=True) - self.job_io.to_json(path=os.path.join(self.working_directory, 'metadata', 'outputs_new', 'job_io.json')) - self.app.tool_data_tables.to_json(path=os.path.join(self.working_directory, 'metadata', 'outputs_new', 'tool_data_tables.json')) + os.makedirs(os.path.join(self.working_directory, "metadata", "outputs_new"), exist_ok=True) + self.job_io.to_json(path=os.path.join(self.working_directory, "metadata", "outputs_new", "job_io.json")) + self.app.tool_data_tables.to_json( + path=os.path.join(self.working_directory, "metadata", "outputs_new", "tool_data_tables.json") + ) job.dependencies = self.tool.dependencies self.sa_session.add(job) self.sa_session.flush() @@ -1221,18 +1271,16 @@ class JobWrapper(HasResourceParameters): # The tool execution is given a working directory beneath the # "job" working directory. safe_makedirs(self.tool_working_directory) - safe_makedirs(os.path.join(working_directory, 'outputs')) - log.debug('(%s) Working directory for job is: %s', - self.job_id, self.working_directory) + safe_makedirs(os.path.join(working_directory, "outputs")) + log.debug("(%s) Working directory for job is: %s", self.job_id, self.working_directory) except ObjectInvalid: - raise Exception('(%s) Unable to create job working directory', - job.id) + raise Exception("(%s) Unable to create job working directory", job.id) @property def guest_ports(self): if hasattr(self, "interactivetools"): # This works when the job is being prepared - guest_ports = [ep.get('port') for ep in self.interactivetools] + guest_ports = [ep.get("port") for ep in self.interactivetools] return guest_ports else: # This works when handling a running job @@ -1252,12 +1300,13 @@ class JobWrapper(HasResourceParameters): self._set_object_store_ids(job) self.__working_directory = self.app.object_store.get_filename( - job, base_dir='job_work', dir_only=True, obj_dir=True) + job, base_dir="job_work", dir_only=True, obj_dir=True + ) return self.__working_directory def working_directory_exists(self): job = self.get_job() - return self.app.object_store.exists(job, base_dir='job_work', dir_only=True, obj_dir=True) + return self.app.object_store.exists(job, base_dir="job_work", dir_only=True, obj_dir=True) @property def tool_working_directory(self): @@ -1269,24 +1318,22 @@ class JobWrapper(HasResourceParameters): def clear_working_directory(self): job = self.get_job() if not os.path.exists(self.working_directory): - log.warning('(%s): Working directory clear requested but %s does ' - 'not exist', - self.job_id, - self.working_directory) + log.warning( + "(%s): Working directory clear requested but %s does " "not exist", self.job_id, self.working_directory + ) return self.object_store.create( - job, base_dir='job_work', dir_only=True, obj_dir=True, - extra_dir='_cleared_contents', extra_dir_at_root=True) + job, base_dir="job_work", dir_only=True, obj_dir=True, extra_dir="_cleared_contents", extra_dir_at_root=True + ) base = self.object_store.get_filename( - job, base_dir='job_work', dir_only=True, obj_dir=True, - extra_dir='_cleared_contents', extra_dir_at_root=True) - date_str = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') + job, base_dir="job_work", dir_only=True, obj_dir=True, extra_dir="_cleared_contents", extra_dir_at_root=True + ) + date_str = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") arc_dir = os.path.join(base, date_str) shutil.move(self.working_directory, arc_dir) self._setup_working_directory(job=job) - log.debug('(%s) Previous working directory moved to %s', - self.job_id, arc_dir) + log.debug("(%s) Previous working directory moved to %s", self.job_id, arc_dir) def default_compute_environment(self, job=None): if not job: @@ -1298,7 +1345,7 @@ class JobWrapper(HasResourceParameters): # Restore parameters from the database job = self.get_job() if job.user is None and job.galaxy_session is None: - raise Exception(f'Job {job.id} has no user and no session.') + raise Exception(f"Job {job.id} has no user and no session.") return job def _get_tool_evaluator(self, job): @@ -1316,7 +1363,9 @@ class JobWrapper(HasResourceParameters): if os.path.exists(path): util.umask_fix_perms(path, self.app.config.umask, 0o666, self.app.config.gid) - def fail(self, message, exception=False, tool_stdout="", tool_stderr="", exit_code=None, job_stdout=None, job_stderr=None): + def fail( + self, message, exception=False, tool_stdout="", tool_stderr="", exit_code=None, job_stdout=None, job_stderr=None + ): """ Indicate job failure by setting state and message on all output datasets. @@ -1330,9 +1379,13 @@ class JobWrapper(HasResourceParameters): try: self.job_destination except JobMappingException as exc: - log.debug("(%s) fail(): Job destination raised JobMappingException('%s'), caching fake '__fail__' " - "destination for completion of fail method", self.get_id_tag(), unicodify(exc.failure_message)) - self.job_runner_mapper.cached_job_destination = JobDestination(id='__fail__') + log.debug( + "(%s) fail(): Job destination raised JobMappingException('%s'), caching fake '__fail__' " + "destination for completion of fail method", + self.get_id_tag(), + unicodify(exc.failure_message), + ) + self.job_runner_mapper.cached_job_destination = JobDestination(id="__fail__") # Might be AssertionError or other exception message = str(message) @@ -1360,30 +1413,38 @@ class JobWrapper(HasResourceParameters): dataset = dataset_assoc.dataset self.sa_session.refresh(dataset) dataset.state = dataset.states.ERROR - dataset.blurb = 'tool error' + dataset.blurb = "tool error" dataset.info = message dataset.set_size() dataset.dataset.set_total_size() dataset.mark_unhidden() - if dataset.ext == 'auto': - dataset.extension = 'data' + if dataset.ext == "auto": + dataset.extension = "data" try: self.__update_output(job, dataset) except Exception: # Failure to update the output of a failed job should not prevent completion of the failure method - log.exception("(%s) fail(): Failed to update job output dataset with id: %s", self.get_id_tag(), - dataset.dataset.id) + log.exception( + "(%s) fail(): Failed to update job output dataset with id: %s", + self.get_id_tag(), + dataset.dataset.id, + ) # Pause any dependent jobs (and those jobs' outputs) for dep_job_assoc in dataset.dependent_jobs: - self.pause(dep_job_assoc.job, "Execution of this dataset's job is paused because its input datasets are in an error state.") - job.set_final_state(job.states.ERROR, supports_skip_locked=self.app.application_stack.supports_skip_locked()) + self.pause( + dep_job_assoc.job, + "Execution of this dataset's job is paused because its input datasets are in an error state.", + ) + job.set_final_state( + job.states.ERROR, supports_skip_locked=self.app.application_stack.supports_skip_locked() + ) job.command_line = self.command_line job.info = message # TODO: Put setting the stdout, stderr, and exit code in one place # (not duplicated with the finish method). job.set_streams(tool_stdout, tool_stderr, job_stdout=job_stdout, job_stderr=job_stderr) # Let the exit code be Null if one is not provided: - if (exit_code is not None): + if exit_code is not None: job.exit_code = exit_code self.sa_session.add(job) @@ -1399,7 +1460,9 @@ class JobWrapper(HasResourceParameters): self._fix_output_permissions() self._report_error() # Perform email action even on failure. - for pja in [pjaa.post_job_action for pjaa in job.post_job_actions if pjaa.post_job_action.action_type == "EmailAction"]: + for pja in [ + pjaa.post_job_action for pjaa in job.post_job_actions if pjaa.post_job_action.action_type == "EmailAction" + ]: ActionBox.execute(self.app, self.sa_session, pja, job) # If the job was deleted, call tool specific fail actions (used for e.g. external metadata) and clean up if self.tool: @@ -1408,7 +1471,7 @@ class JobWrapper(HasResourceParameters): except Exception: log.exception(f"Error occured while calling tool specific fail actions for job {job.id}") cleanup_job = self.cleanup_job - delete_files = cleanup_job == 'always' or (cleanup_job == 'onsuccess' and job.state == job.states.DELETED) + delete_files = cleanup_job == "always" or (cleanup_job == "onsuccess" and job.state == job.states.DELETED) self.cleanup(delete_files=delete_files) def pause(self, job=None, message=None): @@ -1456,8 +1519,12 @@ class JobWrapper(HasResourceParameters): # thread and no other threads are working on the job yet - so don't refresh. if job.state in model.Job.terminal_states: - log.warning("(%s) Ignoring state change from '%s' to '%s' for job " - "that is already terminal", job.id, job.state, state) + log.warning( + "(%s) Ignoring state change from '%s' to '%s' for job " "that is already terminal", + job.id, + job.state, + state, + ) return if info: job.info = info @@ -1473,7 +1540,7 @@ class JobWrapper(HasResourceParameters): return job.state def set_runner(self, runner_url, external_id): - log.warning('set_runner() is deprecated, use set_job_destination()') + log.warning("set_runner() is deprecated, use set_job_destination()") self.set_job_destination(self.job_destination, external_id) def set_job_destination(self, job_destination, external_id=None, flush=True, job=None): @@ -1485,7 +1552,7 @@ class JobWrapper(HasResourceParameters): """ if job is None: job = self.get_job() - log.debug(f'({job.id}) Persisting job destination (destination id: {job_destination.id})') + log.debug(f"({job.id}) Persisting job destination (destination id: {job_destination.id})") job.destination_id = job_destination.id job.destination_params = job_destination.params job.job_runner_name = job_destination.runner @@ -1512,13 +1579,11 @@ class JobWrapper(HasResourceParameters): return self.tool.tmp_target def get_destination_configuration(self, key, default=None): - """ Get a destination parameter that can be defaulted back + """Get a destination parameter that can be defaulted back in app.config if it needs to be applied globally. """ dest_params = self.job_destination.params - return self.get_job().get_destination_configuration( - dest_params, self.app.config, key, default - ) + return self.get_job().get_destination_configuration(dest_params, self.app.config, key, default) def enqueue(self): job = self.get_job() @@ -1567,23 +1632,23 @@ class JobWrapper(HasResourceParameters): trynum = self.app.config.retry_job_output_collection except (OSError, ObjectNotFound) as e: trynum += 1 - log.warning('Error accessing dataset with ID %i, will retry: %s', dataset.dataset.id, unicodify(e)) + log.warning("Error accessing dataset with ID %i, will retry: %s", dataset.dataset.id, unicodify(e)) time.sleep(2) if getattr(dataset, "hidden_beneath_collection_instance", None): dataset.visible = False - dataset.blurb = 'done' - dataset.peek = 'no peek' - dataset.info = (dataset.info or '') - if context['stdout'].strip(): + dataset.blurb = "done" + dataset.peek = "no peek" + dataset.info = dataset.info or "" + if context["stdout"].strip(): # Ensure white space between entries dataset.info = f"{dataset.info.rstrip()}\n{context['stdout'].strip()}" - if context['stderr'].strip(): + if context["stderr"].strip(): # Ensure white space between entries dataset.info = f"{dataset.info.rstrip()}\n{context['stderr'].strip()}" dataset.tool_version = self.version_string dataset.set_size() - if 'uuid' in context: - dataset.dataset.uuid = context['uuid'] + if "uuid" in context: + dataset.dataset.uuid = context["uuid"] self.__update_output(job, dataset) if not purged: collect_extra_files(self.object_store, dataset, self.working_directory) @@ -1594,30 +1659,42 @@ class JobWrapper(HasResourceParameters): dataset.mark_unhidden() elif not purged: # If the tool was expected to set the extension, attempt to retrieve it - if dataset.ext == 'auto': - dataset.extension = context.get('ext', 'data') + if dataset.ext == "auto": + dataset.extension = context.get("ext", "data") dataset.init_meta(copy_from=dataset) # if a dataset was copied, it won't appear in our dictionary: # either use the metadata from originating output dataset, or call set_meta on the copies # it would be quicker to just copy the metadata from the originating output dataset, # but somewhat trickier (need to recurse up the copied_from tree), for now we'll call set_meta() retry_internally = util.asbool(self.get_destination_configuration("retry_metadata_internally", True)) - if not retry_internally and self.tool.tool_type == 'interactive': - retry_internally = util.asbool(self.get_destination_configuration("retry_interactivetool_metadata_internally", retry_internally)) - metadata_set_successfully = self.external_output_metadata.external_metadata_set_successfully(dataset, output_name, self.sa_session, working_directory=self.working_directory) + if not retry_internally and self.tool.tool_type == "interactive": + retry_internally = util.asbool( + self.get_destination_configuration("retry_interactivetool_metadata_internally", retry_internally) + ) + metadata_set_successfully = self.external_output_metadata.external_metadata_set_successfully( + dataset, output_name, self.sa_session, working_directory=self.working_directory + ) if retry_internally and not metadata_set_successfully: # If Galaxy was expected to sniff type and didn't - do so. if dataset.ext == "_sniff_": - extension = sniff.handle_uploaded_dataset_file(dataset.dataset.file_name, self.app.datatypes_registry) + extension = sniff.handle_uploaded_dataset_file( + dataset.dataset.file_name, self.app.datatypes_registry + ) dataset.extension = extension # call datatype.set_meta directly for the initial set_meta call during dataset creation dataset.datatype.set_meta(dataset, overwrite=False) - elif (job.states.ERROR != final_job_state and not metadata_set_successfully): + elif job.states.ERROR != final_job_state and not metadata_set_successfully: dataset._state = model.Dataset.states.FAILED_METADATA else: - self.external_output_metadata.load_metadata(dataset, output_name, self.sa_session, working_directory=self.working_directory, remote_metadata_directory=remote_metadata_directory) - line_count = context.get('line_count', None) + self.external_output_metadata.load_metadata( + dataset, + output_name, + self.sa_session, + working_directory=self.working_directory, + remote_metadata_directory=remote_metadata_directory, + ) + line_count = context.get("line_count", None) try: # Certain datatype's set_peek methods contain a line_count argument dataset.set_peek(line_count=line_count) @@ -1627,8 +1704,8 @@ class JobWrapper(HasResourceParameters): else: # Handle purged datasets. dataset.blurb = "empty" - if dataset.ext == 'auto': - dataset.extension = context.get('ext', 'txt') + if dataset.ext == "auto": + dataset.extension = context.get("ext", "txt") for context_key in TOOL_PROVIDED_JOB_METADATA_KEYS: if context_key in context: @@ -1654,8 +1731,7 @@ class JobWrapper(HasResourceParameters): the contents of the output files. """ finish_timer = self.app.execution_timer_factory.get_timer( - 'internals.galaxy.jobs.job_wrapper_finish', - 'job_wrapper.finish for job ${job_id} executed' + "internals.galaxy.jobs.job_wrapper_finish", "job_wrapper.finish for job ${job_id} executed" ) # default post job setup @@ -1666,13 +1742,21 @@ class JobWrapper(HasResourceParameters): if not isinstance(exception, (AssertionError, MessageException)): # Only attach MessageException and AssertionErrors to job.traceback exception = None - return self.fail(message, tool_stdout=tool_stdout, tool_stderr=tool_stderr, exit_code=tool_exit_code, job_stdout=job_stdout, job_stderr=job_stderr, exception=exception) + return self.fail( + message, + tool_stdout=tool_stdout, + tool_stderr=tool_stderr, + exit_code=tool_exit_code, + job_stdout=job_stdout, + job_stderr=job_stderr, + exception=exception, + ) # TODO: After failing here, consider returning from the function. try: self.reclaim_ownership() except Exception: - log.exception(f'({job.id}) Failed to change ownership of {self.working_directory}, failing') + log.exception(f"({job.id}) Failed to change ownership of {self.working_directory}, failing") return fail() # if the job was deleted, don't finish it @@ -1684,7 +1768,7 @@ class JobWrapper(HasResourceParameters): # the tasks failed. So include the stderr, stdout, and exit code: return fail() - extended_metadata = self.external_output_metadata.extended and not self.tool.tool_type == 'interactive' + extended_metadata = self.external_output_metadata.extended and not self.tool.tool_type == "interactive" # We collect the stderr from tools that write their stderr to galaxy.json tool_provided_metadata = self.get_tool_provided_job_metadata() @@ -1696,7 +1780,14 @@ class JobWrapper(HasResourceParameters): # We set final_job_state to use for dataset management, but *don't* set # job.state until after dataset discovery to prevent history issues if check_output_detected_state is None: - check_output_detected_state = self.check_tool_output(tool_stdout, tool_stderr, tool_exit_code=tool_exit_code, job=job, job_stdout=job_stdout, job_stderr=job_stderr) + check_output_detected_state = self.check_tool_output( + tool_stdout, + tool_stderr, + tool_exit_code=tool_exit_code, + job=job, + job_stdout=job_stdout, + job_stderr=job_stderr, + ) if check_output_detected_state == DETECTED_JOB_STATE.OK and not tool_provided_metadata.has_failed_outputs(): final_job_state = job.states.OK @@ -1714,8 +1805,10 @@ class JobWrapper(HasResourceParameters): # finish method - the false_path file has already moved, # and when the job is recovered, it won't be found. if os.path.exists(dataset_path.real_path) and os.stat(dataset_path.real_path).st_size > 0: - log.warning("finish(): %s not found, but %s is not empty, so it will be used instead" - % (dataset_path.false_path, dataset_path.real_path)) + log.warning( + "finish(): %s not found, but %s is not empty, so it will be used instead" + % (dataset_path.false_path, dataset_path.real_path) + ) else: # Prior to fail we need to set job.state job.set_state(final_job_state) @@ -1726,7 +1819,7 @@ class JobWrapper(HasResourceParameters): try: import_options = store.ImportOptions(allow_dataset_object_edit=True, allow_edit=True) import_model_store = store.get_import_model_store_for_directory( - os.path.join(self.working_directory, 'metadata', 'outputs_populated'), + os.path.join(self.working_directory, "metadata", "outputs_populated"), app=self.app, import_options=import_options, user=job.user, @@ -1757,13 +1850,14 @@ class JobWrapper(HasResourceParameters): # should this also be checking library associations? - can a library item be added from a history before the job has ended? - # lets not allow this to occur # need to update all associated output hdas, i.e. history was shared with job running - for dataset in dataset_assoc.dataset.dataset.history_associations + dataset_assoc.dataset.dataset.library_associations: + for dataset in ( + dataset_assoc.dataset.dataset.history_associations + + dataset_assoc.dataset.dataset.library_associations + ): output_name = dataset_assoc.name # Handles retry internally on error for instance... - self._finish_dataset( - output_name, dataset, job, context, final_job_state, remote_metadata_directory - ) + self._finish_dataset(output_name, dataset, job, context, final_job_state, remote_metadata_directory) if not final_job_state == job.states.ERROR: dataset_assoc.dataset.dataset.state = model.Dataset.states.OK try: @@ -1779,7 +1873,10 @@ class JobWrapper(HasResourceParameters): dataset_assoc.dataset.dataset.state = model.Dataset.states.ERROR # Pause any dependent jobs (and those jobs' outputs) for dep_job_assoc in dataset_assoc.dataset.dependent_jobs: - self.pause(dep_job_assoc.job, "Execution of this dataset's job is paused because its input datasets are in an error state.") + self.pause( + dep_job_assoc.job, + "Execution of this dataset's job is paused because its input datasets are in an error state.", + ) for pja in job.post_job_actions: ActionBox.execute(self.app, self.sa_session, pja.post_job_action, job, final_job_state=final_job_state) @@ -1804,15 +1901,24 @@ class JobWrapper(HasResourceParameters): # ( this used to be performed in the "exec_after_process" hook, but hooks are deprecated ). param_dict = self.get_param_dict(job) try: - self.tool.exec_after_process(self.app, inp_data, out_data, param_dict, job=job, final_job_state=final_job_state) + self.tool.exec_after_process( + self.app, inp_data, out_data, param_dict, job=job, final_job_state=final_job_state + ) except Exception as e: log.exception(f"exec_after_process hook failed for job {self.job_id}") return fail("exec_after_process hook failed", exception=e) # Call 'exec_after_process' hook - self.tool.call_hook('exec_after_process', self.app, inp_data=inp_data, - out_data=out_data, param_dict=param_dict, - tool=self.tool, stdout=job.stdout, stderr=job.stderr) + self.tool.call_hook( + "exec_after_process", + self.app, + inp_data=inp_data, + out_data=out_data, + param_dict=param_dict, + tool=self.tool, + stdout=job.stdout, + stderr=job.stderr, + ) self._fix_output_permissions() @@ -1836,7 +1942,7 @@ class JobWrapper(HasResourceParameters): if job.state == job.states.ERROR: self._report_error() cleanup_job = self.cleanup_job - delete_files = cleanup_job == 'always' or (job.state == job.states.OK and cleanup_job == 'onsuccess') + delete_files = cleanup_job == "always" or (job.state == job.states.OK and cleanup_job == "onsuccess") self.cleanup(delete_files=delete_files) log.debug(finish_timer.to_str(job_id=self.job_id, tool_id=job.tool_id)) @@ -1852,14 +1958,14 @@ class JobWrapper(HasResourceParameters): input_dbkey = loads(input_dbkey) else: # Legacy jobs without __input_ext. - input_ext = 'data' - input_dbkey = '?' + input_ext = "data" + input_dbkey = "?" for _, data in inp_data.items(): # For loop odd, but sort simulating behavior in galaxy.tools.actions if not data: continue input_ext = data.ext - input_dbkey = data.dbkey or '?' + input_dbkey = data.dbkey or "?" # Create generated output children and primary datasets. tool_working_directory = self.tool_working_directory @@ -1881,11 +1987,15 @@ class JobWrapper(HasResourceParameters): if job is not None: job_id_tag = job.get_id_tag() - state, tool_stdout, tool_stderr, job_messages = check_output(self.tool.stdio_regexes, self.tool.stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code, job_id_tag) + state, tool_stdout, tool_stderr, job_messages = check_output( + self.tool.stdio_regexes, self.tool.stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code, job_id_tag + ) # Store the modified stdout and stderr in the job: if job is not None: - job.set_streams(tool_stdout, tool_stderr, job_messages=job_messages, job_stdout=job_stdout, job_stderr=job_stderr) + job.set_streams( + tool_stdout, tool_stderr, job_messages=job_messages, job_stdout=job_stdout, job_stderr=job_stderr + ) return state @@ -1903,16 +2013,22 @@ class JobWrapper(HasResourceParameters): raise self.external_output_metadata.cleanup_external_metadata(self.sa_session) if delete_files: - self.object_store.delete(self.get_job(), base_dir='job_work', entire_dir=True, dir_only=True, obj_dir=True) + self.object_store.delete( + self.get_job(), base_dir="job_work", entire_dir=True, dir_only=True, obj_dir=True + ) except Exception: log.exception("Unable to cleanup job %d", self.job_id) def _collect_metrics(self, has_metrics, job_metrics_directory=None): job = has_metrics.get_job() job_metrics_directory = job_metrics_directory or self.working_directory - per_plugin_properties = self.app.job_metrics.collect_properties(job.destination_id, self.job_id, job_metrics_directory) + per_plugin_properties = self.app.job_metrics.collect_properties( + job.destination_id, self.job_id, job_metrics_directory + ) if per_plugin_properties: - log.info(f"Collecting metrics for {type(has_metrics).__name__} {getattr(has_metrics, 'id', None)} in {job_metrics_directory}") + log.info( + f"Collecting metrics for {type(has_metrics).__name__} {getattr(has_metrics, 'id', None)} in {job_metrics_directory}" + ) for plugin, properties in per_plugin_properties.items(): for metric_name, metric_value in properties.items(): if metric_value is not None: @@ -1932,16 +2048,28 @@ class JobWrapper(HasResourceParameters): if self.app.job_config.limits.output_size and self.app.job_config.limits.output_size > 0: for outfile, size in self.get_output_sizes(): if size > self.app.job_config.limits.output_size: - log.warning('(%s) Job output size %s has exceeded the global output size limit', self.get_id_tag(), os.path.basename(outfile)) - return (JobState.runner_states.OUTPUT_SIZE_LIMIT, - 'Job output file grew too large (greater than %s), please try different inputs or parameters' - % util.nice_size(self.app.job_config.limits.output_size)) + log.warning( + "(%s) Job output size %s has exceeded the global output size limit", + self.get_id_tag(), + os.path.basename(outfile), + ) + return ( + JobState.runner_states.OUTPUT_SIZE_LIMIT, + "Job output file grew too large (greater than %s), please try different inputs or parameters" + % util.nice_size(self.app.job_config.limits.output_size), + ) if self.app.job_config.limits.walltime_delta is not None and runtime is not None: if runtime > self.app.job_config.limits.walltime_delta: - log.warning('(%s) Job runtime %s has exceeded the global walltime, it will be terminated', self.get_id_tag(), runtime) - return (JobState.runner_states.GLOBAL_WALLTIME_REACHED, - 'Job ran longer than the maximum allowed execution time (runtime: %s, limit: %s), please try different inputs or parameters' - % (str(runtime).split('.')[0], self.app.job_config.limits.walltime)) + log.warning( + "(%s) Job runtime %s has exceeded the global walltime, it will be terminated", + self.get_id_tag(), + runtime, + ) + return ( + JobState.runner_states.GLOBAL_WALLTIME_REACHED, + "Job ran longer than the maximum allowed execution time (runtime: %s, limit: %s), please try different inputs or parameters" + % (str(runtime).split(".")[0], self.app.job_config.limits.walltime), + ) return None def has_limits(self): @@ -1960,7 +2088,7 @@ class JobWrapper(HasResourceParameters): def get_env_setup_clause(self): if self.app.config.environment_setup_file is None: - return '' + return "" return f'[ -f "{self.app.config.environment_setup_file}" ] && . {self.app.config.environment_setup_file}' @property @@ -1973,7 +2101,9 @@ class JobWrapper(HasResourceParameters): try: if not tmp_dir or util.asbool(tmp_dir): working_directory = self.working_directory - return '''$([ ! -e '{0}/tmp' ] || mv '{0}/tmp' '{0}'/tmp.$(date +%Y%m%d-%H%M%S) ; mkdir '{0}/tmp'; echo '{0}/tmp')'''.format(working_directory) + return """$([ ! -e '{0}/tmp' ] || mv '{0}/tmp' '{0}'/tmp.$(date +%Y%m%d-%H%M%S) ; mkdir '{0}/tmp'; echo '{0}/tmp')""".format( + working_directory + ) else: return tmp_dir except ValueError: @@ -2022,23 +2152,33 @@ class JobWrapper(HasResourceParameters): def invalidate_external_metadata(self): job = self.get_job() - self.external_output_metadata.invalidate_external_metadata([output_dataset_assoc.dataset for - output_dataset_assoc in - job.output_datasets + job.output_library_datasets], - self.sa_session) + self.external_output_metadata.invalidate_external_metadata( + [ + output_dataset_assoc.dataset + for output_dataset_assoc in job.output_datasets + job.output_library_datasets + ], + self.sa_session, + ) - def setup_external_metadata(self, exec_dir=None, tmp_dir=None, - dataset_files_path=None, config_root=None, - config_file=None, datatypes_config=None, - resolve_metadata_dependencies=False, - set_extension=True, **kwds): + def setup_external_metadata( + self, + exec_dir=None, + tmp_dir=None, + dataset_files_path=None, + config_root=None, + config_file=None, + datatypes_config=None, + resolve_metadata_dependencies=False, + set_extension=True, + **kwds, + ): # extension could still be 'auto' if this is the upload tool. job = self.get_job() if set_extension: for output_dataset_assoc in job.output_datasets: - if output_dataset_assoc.dataset.ext == 'auto': + if output_dataset_assoc.dataset.ext == "auto": context = self.get_dataset_finish_context(dict(), output_dataset_assoc) - output_dataset_assoc.dataset.extension = context.get('ext', 'data') + output_dataset_assoc.dataset.extension = context.get("ext", "data") self.sa_session.flush() if tmp_dir is None: # this dir should should relative to the exec_dir @@ -2050,38 +2190,42 @@ class JobWrapper(HasResourceParameters): if config_file is None: config_file = self.app.config.config_file if datatypes_config is None: - datatypes_config = os.path.join(self.working_directory, 'metadata', 'registry.xml') - safe_makedirs(os.path.join(self.working_directory, 'metadata')) + datatypes_config = os.path.join(self.working_directory, "metadata", "registry.xml") + safe_makedirs(os.path.join(self.working_directory, "metadata")) self.app.datatypes_registry.to_xml_file(path=datatypes_config) inp_data, out_data, out_collections = job.io_dicts(exclude_implicit_outputs=True) job_metadata = os.path.join(self.tool_working_directory, self.tool.provided_metadata_file) object_store_conf = self.object_store.to_dict() - command = self.external_output_metadata.setup_external_metadata(out_data, - out_collections, - self.sa_session, - exec_dir=exec_dir, - tmp_dir=tmp_dir, - dataset_files_path=dataset_files_path, - config_root=config_root, - config_file=config_file, - datatypes_config=datatypes_config, - job_metadata=job_metadata, - provided_metadata_style=self.tool.provided_metadata_style, - object_store_conf=object_store_conf, - tool=self.tool, - job=job, - max_metadata_value_size=self.app.config.max_metadata_value_size, - max_discovered_files=self.app.config.max_discovered_files, - validate_outputs=self.validate_outputs, - link_data_only=self.__link_file_check(), - **kwds) + command = self.external_output_metadata.setup_external_metadata( + out_data, + out_collections, + self.sa_session, + exec_dir=exec_dir, + tmp_dir=tmp_dir, + dataset_files_path=dataset_files_path, + config_root=config_root, + config_file=config_file, + datatypes_config=datatypes_config, + job_metadata=job_metadata, + provided_metadata_style=self.tool.provided_metadata_style, + object_store_conf=object_store_conf, + tool=self.tool, + job=job, + max_metadata_value_size=self.app.config.max_metadata_value_size, + max_discovered_files=self.app.config.max_discovered_files, + validate_outputs=self.validate_outputs, + link_data_only=self.__link_file_check(), + **kwds, + ) if resolve_metadata_dependencies: metadata_tool = self.app.toolbox.get_tool("__SET_METADATA__") if metadata_tool is not None: # Due to tool shed hacks for migrate and installed tool tests... # see (``setup_shed_tools_for_test`` in test/base/driver_util.py). - dependency_shell_commands = metadata_tool.build_dependency_shell_commands(job_directory=self.working_directory, metadata=True) + dependency_shell_commands = metadata_tool.build_dependency_shell_commands( + job_directory=self.working_directory, metadata=True + ) if dependency_shell_commands: dependency_shell_commands = "; ".join(dependency_shell_commands) command = f"{dependency_shell_commands}; {command}" @@ -2118,10 +2262,14 @@ class JobWrapper(HasResourceParameters): return True def container_monitor_command(self, container, **kwds): - if not container or not self.tool.produces_entry_points or not self.get_destination_configuration("container_monitor", True): + if ( + not container + or not self.tool.produces_entry_points + or not self.get_destination_configuration("container_monitor", True) + ): return None - exec_dir = kwds.get('exec_dir', os.path.abspath(os.getcwd())) + exec_dir = kwds.get("exec_dir", os.path.abspath(os.getcwd())) work_dir = self.working_directory configs_dir = ensure_configs_directory(work_dir) container_config = os.path.join(configs_dir, "container_config.json") @@ -2140,11 +2288,7 @@ class JobWrapper(HasResourceParameters): encoded_job_id = self.app.security.encode_id(job_id) job_key = self.app.security.encode_id(job_id, kind="jobs_files") endpoint_base = "%s/api/jobs/%s/ports?job_key=%s" - callback_url = endpoint_base % ( - galaxy_url, - encoded_job_id, - job_key - ) + callback_url = endpoint_base % (galaxy_url, encoded_job_id, job_key) container_config_dict["callback_url"] = callback_url with open(container_config, "w") as f: @@ -2161,7 +2305,7 @@ class JobWrapper(HasResourceParameters): elif job.galaxy_session is not None: return f"anonymous@{job.galaxy_session.remote_addr.split()[-1]}" else: - return 'anonymous@unknown' + return "anonymous@unknown" def __update_output(self, job, hda, clean_only=False): """Handle writing outputs to the object store. @@ -2186,16 +2330,16 @@ class JobWrapper(HasResourceParameters): pass def __link_file_check(self): - """ outputs_to_working_directory breaks library uploads where data is + """outputs_to_working_directory breaks library uploads where data is linked. This method is a hack that solves that problem, but is specific to the upload tool and relies on an injected job param. This method should be removed ASAP and replaced with some properly generic and stateful way of determining link-only datasets. -nate """ - if self.tool and self.tool.id == 'upload1': + if self.tool and self.tool.id == "upload1": job = self.get_job() param_dict = job.get_param_values(self.app) - return param_dict.get('link_data_only') == 'link_to_files' + return param_dict.get("link_data_only") == "link_to_files" else: # The tool is unavailable, we try to move the outputs. return False @@ -2204,8 +2348,9 @@ class JobWrapper(HasResourceParameters): job = self.get_job() external_chown_script = self.get_destination_configuration("external_chown_script", None) if job.user is not None and external_chown_script: - ret = external_chown(self.working_directory, self.user_system_pwent, - external_chown_script, description="working directory") + ret = external_chown( + self.working_directory, self.user_system_pwent, external_chown_script, description="working directory" + ) if not ret: os.chmod(self.working_directory, RWXRWXRWX) @@ -2213,8 +2358,9 @@ class JobWrapper(HasResourceParameters): job = self.get_job() external_chown_script = self.get_destination_configuration("external_chown_script", None) if job.user is not None and external_chown_script: - external_chown(self.working_directory, self.galaxy_system_pwent, - external_chown_script, description="working directory") + external_chown( + self.working_directory, self.galaxy_system_pwent, external_chown_script, description="working directory" + ) @property def user_system_pwent(self): @@ -2250,7 +2396,12 @@ class JobWrapper(HasResourceParameters): def set_container(self, container): if container: - cont = model.JobContainerAssociation(job=self.get_job(), container_type=container.container_type, container_name=container.container_name, container_info=container.container_info) + cont = model.JobContainerAssociation( + job=self.get_job(), + container_type=container.container_type, + container_name=container.container_name, + container_info=container.container_info, + ) self.sa_session.add(cont) self.sa_session.flush() @@ -2260,6 +2411,7 @@ class TaskWrapper(JobWrapper): Extension of JobWrapper intended for running tasks. Should be refactored into a generalized executable unit wrapper parent, then jobs and tasks. """ + # Abstract this to be more useful for running tasks that *don't* necessarily compose a job. is_task = True @@ -2309,7 +2461,12 @@ class TaskWrapper(JobWrapper): self.sa_session.flush() if not self.remote_command_line: - self.command_line, self.version_command_line, extra_filenames, self.environment_variables = tool_evaluator.build() + ( + self.command_line, + self.version_command_line, + extra_filenames, + self.environment_variables, + ) = tool_evaluator.build() self.extra_filenames.extend(extra_filenames) # Ensure galaxy_lib_dir is set in case there are any later chdirs @@ -2321,12 +2478,12 @@ class TaskWrapper(JobWrapper): self.sa_session.add(task) self.sa_session.flush() - self.status = 'prepared' + self.status = "prepared" return self.extra_filenames def fail(self, message, exception=False): log.error(f"TaskWrapper Failure {message}") - self.status = 'error' + self.status = "error" # How do we want to handle task failure? Fail the job and let it clean up? def change_state(self, state, info=False, flush=True, job=None): @@ -2367,16 +2524,17 @@ class TaskWrapper(JobWrapper): """ # This may have ended too soon - log.debug('task %s for job %d ended; exit code: %d' - % (self.task_id, self.job_id, - tool_exit_code if tool_exit_code is not None else -256)) + log.debug( + "task %s for job %d ended; exit code: %d" + % (self.task_id, self.job_id, tool_exit_code if tool_exit_code is not None else -256) + ) # default post job setup_external_metadata self.sa_session.expunge_all() task = self.get_task() # if the job was deleted, don't finish it if task.state == task.states.DELETED: # Job was deleted by an administrator - delete_files = self.cleanup_job in ('always', 'onsuccess') + delete_files = self.cleanup_job in ("always", "onsuccess") self.cleanup(delete_files=delete_files) return elif task.state == task.states.ERROR: @@ -2420,9 +2578,17 @@ class TaskWrapper(JobWrapper): # Handled at the parent job level. Do nothing here. pass - def setup_external_metadata(self, exec_dir=None, tmp_dir=None, dataset_files_path=None, - config_root=None, config_file=None, datatypes_config=None, - set_extension=True, **kwds): + def setup_external_metadata( + self, + exec_dir=None, + tmp_dir=None, + dataset_files_path=None, + config_root=None, + config_file=None, + datatypes_config=None, + set_extension=True, + **kwds, + ): # There is no metadata setting for tasks. This is handled after the merge, at the job level. return "" diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index 6012fdc45c0..3b34f0bde78 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -22,7 +22,7 @@ try: from pulsar.client.staging import COMMAND_VERSION_FILENAME except ImportError: # Package unit tests - COMMAND_VERSION_FILENAME = 'COMMAND_VERSION' + COMMAND_VERSION_FILENAME = "COMMAND_VERSION" import galaxy.datatypes.registry import galaxy.model.mapping @@ -67,7 +67,7 @@ logging.basicConfig() log = logging.getLogger(__name__) -MAX_STDIO_READ_BYTES = 100 * 10 ** 6 # 100 MB +MAX_STDIO_READ_BYTES = 100 * 10**6 # 100 MB def set_validated_state(dataset_instance): @@ -81,7 +81,9 @@ def set_validated_state(dataset_instance): dataset_instance.metadata.__validated_state_message__ = datatype_validation.message -def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size): +def set_meta_with_tool_provided( + dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size +): # This method is somewhat odd, in that we set the metadata attributes from tool, # then call set_meta, then set metadata attributes from tool again. # This is intentional due to interplay of overwrite kwd, the fact that some metadata @@ -90,7 +92,9 @@ def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, data extension = dataset_instance.extension if extension == "_sniff_": try: - extension = sniff.handle_uploaded_dataset_file(dataset_instance.dataset.external_filename, datatypes_registry) + extension = sniff.handle_uploaded_dataset_file( + dataset_instance.dataset.external_filename, datatypes_registry + ) # We need to both set the extension so it is available to set_meta # and record it in the metadata so it can be reloaded on the server # side and the model updated (see MetadataCollection.{from,to}_JSON_dict) @@ -100,10 +104,10 @@ def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, data except Exception: log.exception("Problem sniffing datatype.") - for metadata_name, metadata_value in file_dict.get('metadata', {}).items(): + for metadata_name, metadata_value in file_dict.get("metadata", {}).items(): setattr(dataset_instance.metadata, metadata_name, metadata_value) dataset_instance.datatype.set_meta(dataset_instance, **set_meta_kwds) - for metadata_name, metadata_value in file_dict.get('metadata', {}).items(): + for metadata_name, metadata_value in file_dict.get("metadata", {}).items(): setattr(dataset_instance.metadata, metadata_name, metadata_value) if max_metadata_value_size: @@ -153,7 +157,9 @@ def set_metadata_portable(): tool_provided_metadata = load_job_metadata(job_metadata, provided_metadata_style) def set_meta(new_dataset_instance, file_dict): - set_meta_with_tool_provided(new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size) + set_meta_with_tool_provided( + new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size + ) try: object_store = get_object_store(tool_job_working_directory=tool_job_working_directory) @@ -178,29 +184,29 @@ def set_metadata_portable(): # TODO: constants... locations = [ - (outputs_directory, 'tool_'), - (tool_job_working_directory, ''), - (outputs_directory, ''), # # Pulsar style output directory? Was this ever used - did this ever work? + (outputs_directory, "tool_"), + (tool_job_working_directory, ""), + (outputs_directory, ""), # # Pulsar style output directory? Was this ever used - did this ever work? ] for directory, prefix in locations: if os.path.exists(os.path.join(directory, f"{prefix}stdout")): - with open(os.path.join(directory, f"{prefix}stdout"), 'rb') as f: + with open(os.path.join(directory, f"{prefix}stdout"), "rb") as f: tool_stdout = f.read(MAX_STDIO_READ_BYTES) - with open(os.path.join(directory, f"{prefix}stderr"), 'rb') as f: + with open(os.path.join(directory, f"{prefix}stderr"), "rb") as f: tool_stderr = f.read(MAX_STDIO_READ_BYTES) break else: - if os.path.exists(os.path.join(tool_job_working_directory, 'task_0')): + if os.path.exists(os.path.join(tool_job_working_directory, "task_0")): # We have a task splitting job - tool_stdout = b'' - tool_stderr = b'' - paths = Path(tool_job_working_directory).glob('task_*') + tool_stdout = b"" + tool_stderr = b"" + paths = Path(tool_job_working_directory).glob("task_*") for path in paths: - with open(path / 'outputs' / 'tool_stdout', 'rb') as f: + with open(path / "outputs" / "tool_stdout", "rb") as f: task_stdout = f.read(MAX_STDIO_READ_BYTES) if task_stdout: tool_stdout = b"%s[%s stdout]\n%s\n" % (tool_stdout, path.name.encode(), task_stdout) - with open(path / 'outputs' / 'tool_stderr', 'rb') as f: + with open(path / "outputs" / "tool_stderr", "rb") as f: task_stderr = f.read(MAX_STDIO_READ_BYTES) if task_stderr: tool_stderr = b"%s[%s stdout]\n%s\n" % (tool_stderr, path.name.encode(), task_stderr) @@ -217,26 +223,34 @@ def set_metadata_portable(): exit_code_file = default_exit_code_file(".", job_id_tag) tool_exit_code = read_exit_code_from(exit_code_file, job_id_tag) - check_output_detected_state, tool_stdout, tool_stderr, job_messages = check_output(stdio_regexes, stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code, job_id_tag) + check_output_detected_state, tool_stdout, tool_stderr, job_messages = check_output( + stdio_regexes, stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code, job_id_tag + ) if check_output_detected_state == DETECTED_JOB_STATE.OK and not tool_provided_metadata.has_failed_outputs(): final_job_state = Job.states.OK else: final_job_state = Job.states.ERROR - version_string_path = os.path.join('outputs', COMMAND_VERSION_FILENAME) + version_string_path = os.path.join("outputs", COMMAND_VERSION_FILENAME) version_string = collect_shrinked_content_from_path(version_string_path) expression_context = ExpressionContext(dict(stdout=tool_stdout[:255], stderr=tool_stderr[:255])) # Load outputs. - export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True, strip_metadata_files=False, serialize_jobs=True) + export_store = store.DirectoryModelExportStore( + "metadata/outputs_populated", + serialize_dataset_objects=True, + for_edit=True, + strip_metadata_files=False, + serialize_jobs=True, + ) try: - import_model_store = store.imported_store_for_metadata('metadata/outputs_new', object_store=object_store) + import_model_store = store.imported_store_for_metadata("metadata/outputs_new", object_store=object_store) except AssertionError: # Remove in 21.09, this should only happen for jobs that started on <= 20.09 and finish now import_model_store = None - tool_script_file = os.path.join(tool_job_working_directory, 'tool_script.sh') + tool_script_file = os.path.join(tool_job_working_directory, "tool_script.sh") job = None if import_model_store and export_store: job = next(iter(import_model_store.sa_session.objects[Job].values())) @@ -257,12 +271,14 @@ def set_metadata_portable(): output_collections = {} for name, output_collection in metadata_params["output_collections"].items(): # TODO: remove HistoryDatasetCollectionAssociation fallback on 22.01, model_class used to not be serialized prior to 21.09 - model_class = output_collection.get('model_class', 'HistoryDatasetCollectionAssociation') - collection = import_model_store.sa_session.query(getattr(galaxy.model, model_class)).find(output_collection["id"]) + model_class = output_collection.get("model_class", "HistoryDatasetCollectionAssociation") + collection = import_model_store.sa_session.query(getattr(galaxy.model, model_class)).find( + output_collection["id"] + ) output_collections[name] = collection output_instances = {} for name, output in metadata_params["outputs"].items(): - klass = getattr(galaxy.model, output.get('model_class', 'HistoryDatasetAssociation')) + klass = getattr(galaxy.model, output.get("model_class", "HistoryDatasetAssociation")) output_instances[name] = import_model_store.sa_session.query(klass).find(output["id"]) input_ext = json.loads(metadata_params["job_params"].get("__input_ext") or '"data"') @@ -283,7 +299,7 @@ def set_metadata_portable(): with open(tool_script_file) as command_fh: command_line_lines = [] for i, line in enumerate(command_fh): - if i == 0 and line.endswith('COMMAND_VERSION 2>&1;'): + if i == 0 and line.endswith("COMMAND_VERSION 2>&1;"): # Don't record version command as part of command line continue command_line_lines.append(line) @@ -295,16 +311,16 @@ def set_metadata_portable(): destination = unnamed_output_dict["destination"] elements = unnamed_output_dict["elements"] destination_type = destination["type"] - if destination_type == 'hdas': + if destination_type == "hdas": for element in elements: - filename = element.get('filename') - object_id = element.get('object_id') + filename = element.get("filename") + object_id = element.get("object_id") if filename and object_id: unnamed_id_to_path[object_id] = os.path.join(job_context.job_working_directory, filename) for output_name, output_dict in outputs.items(): dataset_instance_id = output_dict["id"] - klass = getattr(galaxy.model, output_dict.get('model_class', 'HistoryDatasetAssociation')) + klass = getattr(galaxy.model, output_dict.get("model_class", "HistoryDatasetAssociation")) dataset = None if import_model_store: dataset = import_model_store.sa_session.query(klass).find(dataset_instance_id) @@ -312,7 +328,8 @@ def set_metadata_portable(): # legacy check for jobs that started before 21.01, remove on 21.05 filename_in = os.path.join(f"metadata/metadata_in_{output_name}") import pickle - dataset = pickle.load(open(filename_in, 'rb')) # load DatasetInstance + + dataset = pickle.load(open(filename_in, "rb")) # load DatasetInstance assert dataset is not None filename_kwds = os.path.join(f"metadata/metadata_kwds_{output_name}") @@ -324,15 +341,21 @@ def set_metadata_portable(): legacy_object_store_store_by = metadata_params.get("object_store_store_by", "id") # Same block as below... - set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds))) # load kwds; need to ensure our keywords are not unicode + set_meta_kwds = stringify_dictionary_keys( + json.load(open(filename_kwds)) + ) # load kwds; need to ensure our keywords are not unicode try: external_filename = unnamed_id_to_path.get(dataset_instance_id, dataset_filename_override) if not os.path.exists(external_filename): matches = glob.glob(external_filename) assert len(matches) == 1, f"More than one file matched by output glob '{external_filename}'" external_filename = matches[0] - assert safe_contains(tool_job_working_directory, external_filename), f"Cannot collect output '{external_filename}' from outside of working directory" - created_from_basename = os.path.relpath(external_filename, os.path.join(tool_job_working_directory, 'working')) + assert safe_contains( + tool_job_working_directory, external_filename + ), f"Cannot collect output '{external_filename}' from outside of working directory" + created_from_basename = os.path.relpath( + external_filename, os.path.join(tool_job_working_directory, "working") + ) dataset.dataset.created_from_basename = created_from_basename # override filename if we're dealing with outputs to working directory and dataset is not linked to link_data_only = metadata_params.get("link_data_only") @@ -345,8 +368,8 @@ def set_metadata_portable(): files_path = os.path.abspath(os.path.join(tool_job_working_directory, "working", extra_files_dir_name)) dataset.dataset.external_extra_files_path = files_path file_dict = tool_provided_metadata.get_dataset_meta(output_name, dataset.dataset.id, dataset.dataset.uuid) - if 'ext' in file_dict: - dataset.extension = file_dict['ext'] + if "ext" in file_dict: + dataset.extension = file_dict["ext"] # Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles override_metadata = json.load(open(override_metadata)) for metadata_name, metadata_file_override in override_metadata: @@ -374,20 +397,20 @@ def set_metadata_portable(): context = ExpressionContext(meta, expression_context) else: context = expression_context - dataset.blurb = 'done' - dataset.peek = 'no peek' - dataset.info = (dataset.info or '') - if context['stdout'].strip(): + dataset.blurb = "done" + dataset.peek = "no peek" + dataset.info = dataset.info or "" + if context["stdout"].strip(): # Ensure white space between entries dataset.info = f"{dataset.info.rstrip()}\n{context['stdout'].strip()}" - if context['stderr'].strip(): + if context["stderr"].strip(): # Ensure white space between entries dataset.info = f"{dataset.info.rstrip()}\n{context['stderr'].strip()}" dataset.tool_version = version_string - if 'uuid' in context: - dataset.dataset.uuid = context['uuid'] + if "uuid" in context: + dataset.dataset.uuid = context["uuid"] if not final_job_state == Job.states.ERROR: - line_count = context.get('line_count', None) + line_count = context.get("line_count", None) try: # Certain datatype's set_peek methods contain a line_count argument dataset.set_peek(line_count=line_count) @@ -406,9 +429,13 @@ def set_metadata_portable(): else: dataset.metadata.to_JSON_dict(filename_out) # write out results of set_meta - json.dump((True, 'Metadata has been set successfully'), open(filename_results_code, 'wt+')) # setting metadata has succeeded + json.dump( + (True, "Metadata has been set successfully"), open(filename_results_code, "wt+") + ) # setting metadata has succeeded except Exception: - json.dump((False, traceback.format_exc()), open(filename_results_code, 'wt+')) # setting metadata has failed somehow + json.dump( + (False, traceback.format_exc()), open(filename_results_code, "wt+") + ) # setting metadata has failed somehow if export_store: export_store._finalize() @@ -423,10 +450,18 @@ def validate_and_load_datatypes_config(datatypes_config): datatypes_config = "configs/registry.xml" if not os.path.exists(datatypes_config): - print(f"Metadata setting failed because registry.xml [{datatypes_config}] could not be found. You may retry setting metadata.") + print( + f"Metadata setting failed because registry.xml [{datatypes_config}] could not be found. You may retry setting metadata." + ) sys.exit(1) datatypes_registry = galaxy.datatypes.registry.Registry() - datatypes_registry.load_datatypes(root_dir=galaxy_root, config=datatypes_config, use_build_sites=False, use_converters=False, use_display_applications=False) + datatypes_registry.load_datatypes( + root_dir=galaxy_root, + config=datatypes_config, + use_build_sites=False, + use_converters=False, + use_display_applications=False, + ) galaxy.model.set_datatypes_registry(datatypes_registry) return datatypes_registry @@ -440,12 +475,16 @@ def write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_ filename = file_dict["filename"] new_dataset_filename = os.path.join(tool_job_working_directory, "working", filename) new_dataset = Dataset(id=-i, external_filename=new_dataset_filename) - extra_files = file_dict.get('extra_files', None) + extra_files = file_dict.get("extra_files", None) if extra_files is not None: new_dataset._extra_files_path = os.path.join(tool_job_working_directory, "working", extra_files) new_dataset.state = new_dataset.states.OK - new_dataset_instance = HistoryDatasetAssociation(id=-i, dataset=new_dataset, extension=file_dict.get('ext', 'data')) + new_dataset_instance = HistoryDatasetAssociation( + id=-i, dataset=new_dataset, extension=file_dict.get("ext", "data") + ) set_meta(new_dataset_instance, file_dict) - file_dict['metadata'] = json.loads(new_dataset_instance.metadata.to_JSON_dict()) # storing metadata in external form, need to turn back into dict, then later jsonify + file_dict["metadata"] = json.loads( + new_dataset_instance.metadata.to_JSON_dict() + ) # storing metadata in external form, need to turn back into dict, then later jsonify tool_provided_metadata.rewrite() diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 219af52d45c..f00197a8654 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -17,7 +17,10 @@ import random import string from collections import defaultdict from collections.abc import Callable -from datetime import datetime, timedelta +from datetime import ( + datetime, + timedelta, +) from enum import Enum from string import Template from typing import ( @@ -32,11 +35,20 @@ from typing import ( TYPE_CHECKING, Union, ) -from uuid import UUID, uuid4 +from uuid import ( + UUID, + uuid4, +) import sqlalchemy from boltons.iterutils import remap -from social_core.storage import AssociationMixin, CodeMixin, NonceMixin, PartialMixin, UserMixin +from social_core.storage import ( + AssociationMixin, + CodeMixin, + NonceMixin, + PartialMixin, + UserMixin, +) from sqlalchemy import ( alias, and_, @@ -103,7 +115,10 @@ from galaxy.model.custom_types import ( TrimmedString, UUIDType, ) -from galaxy.model.item_attrs import get_item_annotation_str, UsesAnnotations +from galaxy.model.item_attrs import ( + get_item_annotation_str, + UsesAnnotations, +) from galaxy.model.orm.now import now from galaxy.model.view import HistoryDatasetCollectionJobStateSummary from galaxy.security import get_permitted_actions @@ -116,10 +131,21 @@ from galaxy.util import ( unicodify, unique_id, ) -from galaxy.util.dictifiable import dict_for, Dictifiable -from galaxy.util.form_builder import (AddressField, CheckboxField, HistoryField, - PasswordField, SelectField, TextArea, TextField, WorkflowField, - WorkflowMappingField) +from galaxy.util.dictifiable import ( + dict_for, + Dictifiable, +) +from galaxy.util.form_builder import ( + AddressField, + CheckboxField, + HistoryField, + PasswordField, + SelectField, + TextArea, + TextField, + WorkflowField, + WorkflowMappingField, +) from galaxy.util.hash_util import new_secure_hash from galaxy.util.json import safe_loads from galaxy.util.sanitize_html import sanitize_html @@ -152,6 +178,7 @@ if TYPE_CHECKING: class _HasTable: table: Table __table__: Table + else: _HasTable = object @@ -180,7 +207,7 @@ class RepresentById(_HasTable): def __repr__(self): try: - r = f'' + r = f"" except Exception: r = object.__repr__(self) log.exception("Caught exception attempting to generate repr for: %s", r) @@ -205,7 +232,9 @@ class ConverterDependencyException(Exception): def _get_datatypes_registry(): if _datatypes_registry is None: - raise Exception("galaxy.model.set_datatypes_registry must be called before performing certain DatasetInstance operations.") + raise Exception( + "galaxy.model.set_datatypes_registry must be called before performing certain DatasetInstance operations." + ) return _datatypes_registry @@ -224,7 +253,7 @@ class HasTags: def to_dict(self, *args, **kwargs): rval = super().to_dict(*args, **kwargs) - rval['tags'] = self.make_tag_string_list() + rval["tags"] = self.make_tag_string_list() return rval def make_tag_string_list(self): @@ -249,8 +278,9 @@ class HasTags: class SerializationOptions: - - def __init__(self, for_edit, serialize_dataset_objects=None, serialize_files_handler=None, strip_metadata_files=None): + def __init__( + self, for_edit, serialize_dataset_objects=None, serialize_files_handler=None, strip_metadata_files=None + ): self.for_edit = for_edit if serialize_dataset_objects is None: serialize_dataset_objects = for_edit @@ -266,7 +296,7 @@ class SerializationOptions: if self.for_edit and obj.id: ret_val["id"] = obj.id elif obj.id: - ret_val["encoded_id"] = id_encoder.encode_id(obj.id, kind='model_export') + ret_val["encoded_id"] = id_encoder.encode_id(obj.id, kind="model_export") else: if not hasattr(obj, "temp_id"): obj.temp_id = uuid4().hex @@ -276,7 +306,7 @@ class SerializationOptions: if self.for_edit and obj.id: return obj.id elif obj.id: - return id_encoder.encode_id(obj.id, kind='model_export') + return id_encoder.encode_id(obj.id, kind="model_export") else: if not hasattr(obj, "temp_id"): obj.temp_id = uuid4().hex @@ -286,7 +316,7 @@ class SerializationOptions: if self.for_edit and obj_id: return obj_id elif obj_id: - return id_encoder.encode_id(obj_id, kind='model_export') + return id_encoder.encode_id(obj_id, kind="model_export") else: raise NotImplementedError() @@ -296,13 +326,12 @@ class SerializationOptions: class Serializable(RepresentById): - - def serialize(self, id_encoder: IdEncodingHelper, serialization_options: SerializationOptions, for_link: bool = False) -> Dict[str, Any]: + def serialize( + self, id_encoder: IdEncodingHelper, serialization_options: SerializationOptions, for_link: bool = False + ) -> Dict[str, Any]: """Serialize model for a re-population in (potentially) another Galaxy instance.""" if for_link: - rval = dict_for( - self - ) + rval = dict_for(self) serialization_options.attach_identifier(id_encoder, self, rval) return rval return self._serialize(id_encoder, serialization_options) @@ -313,14 +342,13 @@ class Serializable(RepresentById): class HasName: - def get_display_name(self): """ These objects have a name attribute can be either a string or a unicode object. If string, convert to unicode object assuming 'utf-8' format. """ name = self.name - name = unicodify(name, 'utf-8') + name = unicodify(name, "utf-8") return name @@ -343,10 +371,8 @@ class UsesCreateAndUpdateTime: class WorkerProcess(Base, UsesCreateAndUpdateTime, _HasTable): - __tablename__ = 'worker_process' - __table_args__ = ( - UniqueConstraint('server_name', 'hostname'), - ) + __tablename__ = "worker_process" + __table_args__ = (UniqueConstraint("server_name", "hostname"),) id = Column(Integer, primary_key=True) server_name = Column(String(255), index=True) @@ -382,28 +408,32 @@ def cached_id(galaxy_model_object): class JobLike: - MAX_NUMERIC = 10**(JOB_METRIC_PRECISION - JOB_METRIC_SCALE) - 1 + MAX_NUMERIC = 10 ** (JOB_METRIC_PRECISION - JOB_METRIC_SCALE) - 1 def _init_metrics(self): self.text_metrics = [] self.numeric_metrics = [] def add_metric(self, plugin, metric_name, metric_value): - plugin = unicodify(plugin, 'utf-8') - metric_name = unicodify(metric_name, 'utf-8') + plugin = unicodify(plugin, "utf-8") + metric_name = unicodify(metric_name, "utf-8") number = isinstance(metric_value, numbers.Number) if number and int(metric_value) <= JobLike.MAX_NUMERIC: metric = self._numeric_metric(plugin, metric_name, metric_value) self.numeric_metrics.append(metric) elif number: - log.warning("Cannot store metric due to database column overflow (max: %s): %s: %s", - JobLike.MAX_NUMERIC, metric_name, metric_value) + log.warning( + "Cannot store metric due to database column overflow (max: %s): %s: %s", + JobLike.MAX_NUMERIC, + metric_name, + metric_value, + ) else: - metric_value = unicodify(metric_value, 'utf-8') + metric_value = unicodify(metric_value, "utf-8") if len(metric_value) > (JOB_METRIC_MAX_LENGTH - 1): # Truncate these values - not needed with sqlite # but other backends must need it. - metric_value = metric_value[:(JOB_METRIC_MAX_LENGTH - 1)] + metric_value = metric_value[: (JOB_METRIC_MAX_LENGTH - 1)] metric = self._text_metric(plugin, metric_name, metric_value) self.text_metrics.append(metric) @@ -415,22 +445,24 @@ class JobLike: def set_streams(self, tool_stdout, tool_stderr, job_stdout=None, job_stderr=None, job_messages=None): def shrink_and_unicodify(what, stream): if len(stream) > galaxy.util.DATABASE_MAX_STRING_SIZE: - log.info("%s for %s %d is greater than %s, only a portion will be logged to database", - what, - type(self), - self.id, - galaxy.util.DATABASE_MAX_STRING_SIZE_PRETTY) + log.info( + "%s for %s %d is greater than %s, only a portion will be logged to database", + what, + type(self), + self.id, + galaxy.util.DATABASE_MAX_STRING_SIZE_PRETTY, + ) return galaxy.util.shrink_and_unicodify(stream) - self.tool_stdout = shrink_and_unicodify('tool_stdout', tool_stdout) - self.tool_stderr = shrink_and_unicodify('tool_stderr', tool_stderr) + self.tool_stdout = shrink_and_unicodify("tool_stdout", tool_stdout) + self.tool_stderr = shrink_and_unicodify("tool_stderr", tool_stderr) if job_stdout is not None: - self.job_stdout = shrink_and_unicodify('job_stdout', job_stdout) + self.job_stdout = shrink_and_unicodify("job_stdout", job_stdout) else: self.job_stdout = None if job_stderr is not None: - self.job_stderr = shrink_and_unicodify('job_stderr', job_stderr) + self.job_stderr = shrink_and_unicodify("job_stderr", job_stderr) else: self.job_stderr = None @@ -449,7 +481,7 @@ class JobLike: @property def stdout(self): - stdout = self.tool_stdout or '' + stdout = self.tool_stdout or "" if self.job_stdout: stdout += f"\n{self.job_stdout}" return stdout @@ -460,7 +492,7 @@ class JobLike: @property def stderr(self): - stderr = self.tool_stderr or '' + stderr = self.tool_stderr or "" if self.job_stderr: stderr += f"\n{self.job_stderr}" return stderr @@ -475,11 +507,12 @@ class User(Base, Dictifiable, RepresentById): Data for a Galaxy user or admin and relations to their histories, credentials, and roles. """ + use_pbkdf2 = True bootstrap_admin_user = False # api_keys: 'List[APIKeys]' already declared as relationship() - __tablename__ = 'galaxy_user' + __tablename__ = "galaxy_user" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) @@ -489,7 +522,7 @@ class User(Base, Dictifiable, RepresentById): password = Column(TrimmedString(255), nullable=False) last_password_change = Column(DateTime, default=now) external = Column(Boolean, default=False) - form_values_id = Column(Integer, ForeignKey('form_values.id'), index=True) + form_values_id = Column(Integer, ForeignKey("form_values.id"), index=True) deleted = Column(Boolean, index=True, default=False) purged = Column(Boolean, index=True, default=False) disk_usage = Column(Numeric(15, 0), index=True) @@ -497,60 +530,74 @@ class User(Base, Dictifiable, RepresentById): active = Column(Boolean, index=True, default=True, nullable=False) activation_token = Column(TrimmedString(64), nullable=True, index=True) - addresses = relationship('UserAddress', - back_populates='user', - order_by=lambda: desc(UserAddress.update_time)) - cloudauthz = relationship('CloudAuthz', back_populates='user') - custos_auth = relationship('CustosAuthnzToken', back_populates='user') - default_permissions = relationship('DefaultUserPermissions', back_populates='user') - groups = relationship('UserGroupAssociation', back_populates='user') - histories = relationship('History', - back_populates='user', - order_by=lambda: desc(History.update_time)) # type: ignore[has-type] - active_histories = relationship('History', + addresses = relationship("UserAddress", back_populates="user", order_by=lambda: desc(UserAddress.update_time)) + cloudauthz = relationship("CloudAuthz", back_populates="user") + custos_auth = relationship("CustosAuthnzToken", back_populates="user") + default_permissions = relationship("DefaultUserPermissions", back_populates="user") + groups = relationship("UserGroupAssociation", back_populates="user") + histories = relationship( + "History", back_populates="user", order_by=lambda: desc(History.update_time) # type: ignore[has-type] + ) + active_histories = relationship( + "History", primaryjoin=(lambda: (History.user_id == User.id) & (not_(History.deleted))), # type: ignore[has-type] viewonly=True, - order_by=lambda: desc(History.update_time)) # type: ignore[has-type] - galaxy_sessions = relationship('GalaxySession', - back_populates='user', - order_by=lambda: desc(GalaxySession.update_time)) # type: ignore[has-type] - quotas = relationship('UserQuotaAssociation', back_populates='user') - social_auth = relationship('UserAuthnzToken', back_populates='user') - stored_workflow_menu_entries = relationship('StoredWorkflowMenuEntry', - primaryjoin=(lambda: - (StoredWorkflowMenuEntry.user_id == User.id) + order_by=lambda: desc(History.update_time), # type: ignore[has-type] + ) + galaxy_sessions = relationship( + "GalaxySession", back_populates="user", order_by=lambda: desc(GalaxySession.update_time) # type: ignore[has-type] + ) + quotas = relationship("UserQuotaAssociation", back_populates="user") + social_auth = relationship("UserAuthnzToken", back_populates="user") + stored_workflow_menu_entries = relationship( + "StoredWorkflowMenuEntry", + primaryjoin=( + lambda: (StoredWorkflowMenuEntry.user_id == User.id) & (StoredWorkflowMenuEntry.stored_workflow_id == StoredWorkflow.id) # type: ignore[has-type] & not_(StoredWorkflow.deleted) # type: ignore[has-type] ), - back_populates='user', - cascade='all, delete-orphan', - collection_class=ordering_list('order_index')) - _preferences = relationship('UserPreference', collection_class=attribute_mapped_collection('name')) - values = relationship('FormValues', - primaryjoin=(lambda: User.form_values_id == FormValues.id)) # type: ignore[has-type] + back_populates="user", + cascade="all, delete-orphan", + collection_class=ordering_list("order_index"), + ) + _preferences = relationship("UserPreference", collection_class=attribute_mapped_collection("name")) + values = relationship( + "FormValues", primaryjoin=(lambda: User.form_values_id == FormValues.id) # type: ignore[has-type] + ) # Add type hint (will this work w/SA?) - api_keys: 'List[APIKeys]' = relationship('APIKeys', - back_populates='user', - order_by=lambda: desc(APIKeys.create_time)) - data_manager_histories = relationship('DataManagerHistoryAssociation', back_populates='user') - roles = relationship('UserRoleAssociation', back_populates='user') - stored_workflows = relationship('StoredWorkflow', back_populates='user', - primaryjoin=(lambda: User.id == StoredWorkflow.user_id)) # type: ignore[has-type] + api_keys: "List[APIKeys]" = relationship( + "APIKeys", back_populates="user", order_by=lambda: desc(APIKeys.create_time) + ) + data_manager_histories = relationship("DataManagerHistoryAssociation", back_populates="user") + roles = relationship("UserRoleAssociation", back_populates="user") + stored_workflows = relationship( + "StoredWorkflow", back_populates="user", primaryjoin=(lambda: User.id == StoredWorkflow.user_id) # type: ignore[has-type] + ) non_private_roles = relationship( - 'UserRoleAssociation', + "UserRoleAssociation", viewonly=True, - primaryjoin=(lambda: - (User.id == UserRoleAssociation.user_id) # type: ignore[has-type] + primaryjoin=( + lambda: (User.id == UserRoleAssociation.user_id) # type: ignore[has-type] & (UserRoleAssociation.role_id == Role.id) # type: ignore[has-type] - & not_(Role.name == User.email)) # type: ignore[has-type] + & not_(Role.name == User.email) # type: ignore[has-type] + ), ) preferences: association_proxy # defined at the end of this module # attributes that will be accessed and returned when calling to_dict( view='collection' ) - dict_collection_visible_keys = ['id', 'email', 'username', 'deleted', 'active', 'last_password_change'] + dict_collection_visible_keys = ["id", "email", "username", "deleted", "active", "last_password_change"] # attributes that will be accessed and returned when calling to_dict( view='element' ) - dict_element_visible_keys = ['id', 'email', 'username', 'total_disk_usage', 'nice_total_disk_usage', 'deleted', 'active', 'last_password_change'] + dict_element_visible_keys = [ + "id", + "email", + "username", + "total_disk_usage", + "nice_total_disk_usage", + "deleted", + "active", + "last_password_change", + ] def __init__(self, email=None, password=None, username=None): self.email = email @@ -564,7 +611,7 @@ class User(Base, Dictifiable, RepresentById): @property def extra_preferences(self): data = defaultdict(lambda: None) - extra_user_preferences = self.preferences.get('extra_user_preferences') + extra_user_preferences = self.preferences.get("extra_user_preferences") if extra_user_preferences: try: data.update(json.loads(extra_user_preferences)) @@ -591,7 +638,8 @@ class User(Base, Dictifiable, RepresentById): :return: void """ self.set_password_cleartext( - ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(length))) + "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(length)) + ) def check_password(self, cleartext): """ @@ -604,9 +652,9 @@ class User(Base, Dictifiable, RepresentById): Gives the system user pwent entry based on e-mail or username depending on the value in real_system_username """ - if real_system_username == 'user_email': - username = self.email.split('@')[0] - elif real_system_username == 'username': + if real_system_username == "user_email": + username = self.email.split("@")[0] + elif real_system_username == "username": username = self.username else: username = real_system_username @@ -622,18 +670,19 @@ class User(Base, Dictifiable, RepresentById): """ try: db_session = object_session(self) - user = db_session.query( - User - ).filter_by( # don't use get, it will use session variant. - id=self.id - ).options( - joinedload("roles"), - joinedload("roles.role"), - joinedload("groups"), - joinedload("groups.group"), - joinedload("groups.group.roles"), - joinedload("groups.group.roles.role") - ).one() + user = ( + db_session.query(User) + .filter_by(id=self.id) # don't use get, it will use session variant. + .options( + joinedload("roles"), + joinedload("roles.role"), + joinedload("groups"), + joinedload("groups.group"), + joinedload("groups.group.roles"), + joinedload("groups.group.roles.role"), + ) + .one() + ) except Exception: # If not persistent user, just use models normaly and # skip optimizations... @@ -647,8 +696,7 @@ class User(Base, Dictifiable, RepresentById): return roles def all_roles_exploiting_cache(self): - """ - """ + """ """ roles = [ura.role for ura in self.roles] for group in [uga.group for uga in self.groups]: for role in [gra.role for gra in group.roles]: @@ -727,7 +775,7 @@ class User(Base, Dictifiable, RepresentById): AND library_dataset_dataset_association.id IS NULL """ sa_session = object_session(self) - usage = sa_session.scalar(sql_calc, {'id': self.id}) + usage = sa_session.scalar(sql_calc, {"id": self.id}) if not dryrun: self.set_disk_usage(usage) sa_session.flush() @@ -752,25 +800,24 @@ class User(Base, Dictifiable, RepresentById): 'foo2' """ if user: - user_id = '%d' % user.id + user_id = "%d" % user.id user_email = str(user.email) user_name = str(user.username) else: user = None - user_id = 'Anonymous' - user_email = 'Anonymous' - user_name = 'Anonymous' + user_id = "Anonymous" + user_email = "Anonymous" + user_name = "Anonymous" environment = {} - environment['__user__'] = user - environment['__user_id__'] = environment['userId'] = user_id - environment['__user_email__'] = environment['userEmail'] = user_email - environment['__user_name__'] = user_name + environment["__user__"] = user + environment["__user_id__"] = environment["userId"] = user_id + environment["__user_email__"] = environment["userEmail"] = user_email + environment["__user_name__"] = user_name return environment @staticmethod def expand_user_properties(user, in_string): - """ - """ + """ """ environment = User.user_template_environment(user) return Template(in_string).safe_substitute(environment) @@ -789,7 +836,7 @@ class User(Base, Dictifiable, RepresentById): def attempt_create_private_role(self): session = object_session(self) role_name = self.email - role_desc = f'Private Role for {self.email}' + role_desc = f"Private Role for {self.email}" role_type = Role.types.PRIVATE role = Role(name=role_name, description=role_desc, type=role_type) assoc = UserRoleAssociation(self, role) @@ -798,12 +845,12 @@ class User(Base, Dictifiable, RepresentById): class PasswordResetToken(Base, _HasTable): - __tablename__ = 'password_reset_token' + __tablename__ = "password_reset_token" token = Column(String(32), primary_key=True, unique=True, index=True) expiration_time = Column(DateTime) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) - user = relationship('User') + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) + user = relationship("User") def __init__(self, user, token=None): if token: @@ -815,7 +862,7 @@ class PasswordResetToken(Base, _HasTable): class DynamicTool(Base, Dictifiable, RepresentById): - __tablename__ = 'dynamic_tool' + __tablename__ = "dynamic_tool" id = Column(Integer, primary_key=True) uuid = Column(UUIDType()) @@ -830,14 +877,14 @@ class DynamicTool(Base, Dictifiable, RepresentById): active = Column(Boolean, default=True) value = Column(MutableJSONType) - dict_collection_visible_keys = ('id', 'tool_id', 'tool_format', 'tool_version', 'uuid', 'active', 'hidden') - dict_element_visible_keys = ('id', 'tool_id', 'tool_format', 'tool_version', 'uuid', 'active', 'hidden') + dict_collection_visible_keys = ("id", "tool_id", "tool_format", "tool_version", "uuid", "active", "hidden") + dict_element_visible_keys = ("id", "tool_id", "tool_format", "tool_version", "uuid", "active", "hidden") def __init__(self, active=True, hidden=True, **kwd): super().__init__(**kwd) self.active = active self.hidden = hidden - _uuid = kwd.get('uuid') + _uuid = kwd.get("uuid") self.uuid = get_uuid(_uuid) @@ -852,40 +899,40 @@ class BaseJobMetric(Base): class JobMetricText(BaseJobMetric, RepresentById): - __tablename__ = 'job_metric_text' + __tablename__ = "job_metric_text" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) plugin = Column(Unicode(255)) metric_name = Column(Unicode(255)) metric_value = Column(Unicode(JOB_METRIC_MAX_LENGTH)) class JobMetricNumeric(BaseJobMetric, RepresentById): - __tablename__ = 'job_metric_numeric' + __tablename__ = "job_metric_numeric" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) plugin = Column(Unicode(255)) metric_name = Column(Unicode(255)) metric_value = Column(Numeric(JOB_METRIC_PRECISION, JOB_METRIC_SCALE)) class TaskMetricText(BaseJobMetric, RepresentById): - __tablename__ = 'task_metric_text' + __tablename__ = "task_metric_text" id = Column(Integer, primary_key=True) - task_id = Column(Integer, ForeignKey('task.id'), index=True) + task_id = Column(Integer, ForeignKey("task.id"), index=True) plugin = Column(Unicode(255)) metric_name = Column(Unicode(255)) metric_value = Column(Unicode(JOB_METRIC_MAX_LENGTH)) class TaskMetricNumeric(BaseJobMetric, RepresentById): - __tablename__ = 'task_metric_numeric' + __tablename__ = "task_metric_numeric" id = Column(Integer, primary_key=True) - task_id = Column(Integer, ForeignKey('task.id'), index=True) + task_id = Column(Integer, ForeignKey("task.id"), index=True) plugin = Column(Unicode(255)) metric_name = Column(Unicode(255)) metric_value = Column(Numeric(JOB_METRIC_PRECISION, JOB_METRIC_SCALE)) @@ -896,17 +943,18 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): A job represents a request to run a tool given input datasets, tool parameters, and output datasets. """ - __tablename__ = 'job' + + __tablename__ = "job" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now, index=True) - history_id = Column(Integer, ForeignKey('history.id'), index=True) - library_folder_id = Column(Integer, ForeignKey('library_folder.id'), index=True) + history_id = Column(Integer, ForeignKey("history.id"), index=True) + library_folder_id = Column(Integer, ForeignKey("library_folder.id"), index=True) tool_id = Column(String(255)) - tool_version = Column(TEXT, default='1.0.0') + tool_version = Column(TEXT, default="1.0.0") galaxy_version = Column(String(64), default=None) - dynamic_tool_id = Column(Integer, ForeignKey('dynamic_tool.id'), index=True, nullable=True) + dynamic_tool_id = Column(Integer, ForeignKey("dynamic_tool.id"), index=True, nullable=True) state = Column(String(64), index=True) info = Column(TrimmedString(255)) copied_from_job_id = Column(Integer, nullable=True) @@ -921,8 +969,8 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): tool_stderr = Column(TEXT) exit_code = Column(Integer, nullable=True) traceback = Column(TEXT) - session_id = Column(Integer, ForeignKey('galaxy_session.id'), index=True, nullable=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True, nullable=True) + session_id = Column(Integer, ForeignKey("galaxy_session.id"), index=True, nullable=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True, nullable=True) job_runner_name = Column(String(255)) job_runner_external_id = Column(String(255), index=True) destination_id = Column(String(255), nullable=True) @@ -932,72 +980,71 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): params = Column(TrimmedString(255), index=True) handler = Column(TrimmedString(255), index=True) - user = relationship('User') - galaxy_session = relationship('GalaxySession') - history = relationship('History', back_populates='jobs') - library_folder = relationship('LibraryFolder') - parameters = relationship('JobParameter') - input_datasets = relationship('JobToInputDatasetAssociation', back_populates='job') - input_dataset_collections = relationship('JobToInputDatasetCollectionAssociation', - back_populates='job') - input_dataset_collection_elements = relationship('JobToInputDatasetCollectionElementAssociation', - back_populates='job') - output_dataset_collection_instances = relationship('JobToOutputDatasetCollectionAssociation', - back_populates='job') - output_dataset_collections = relationship('JobToImplicitOutputDatasetCollectionAssociation', - back_populates='job') - post_job_actions = relationship('PostJobActionAssociation', back_populates='job') - input_library_datasets = relationship('JobToInputLibraryDatasetAssociation', - back_populates='job') - output_library_datasets = relationship('JobToOutputLibraryDatasetAssociation', - back_populates='job') - external_output_metadata = relationship('JobExternalOutputMetadata', back_populates='job') - tasks = relationship('Task', back_populates='job') - output_datasets = relationship('JobToOutputDatasetAssociation', back_populates='job') - state_history = relationship('JobStateHistory') - text_metrics = relationship('JobMetricText') - numeric_metrics = relationship('JobMetricNumeric') - interactivetool_entry_points = relationship('InteractiveToolEntryPoint', - back_populates='job', uselist=True) - implicit_collection_jobs_association = relationship('ImplicitCollectionJobsJobAssociation', - back_populates='job', uselist=False) - container = relationship('JobContainerAssociation', back_populates='job', uselist=False) - data_manager_association = relationship('DataManagerJobAssociation', - back_populates='job', uselist=False) - history_dataset_collection_associations = relationship('HistoryDatasetCollectionAssociation', - back_populates='job') - workflow_invocation_step = relationship('WorkflowInvocationStep', - back_populates='job', uselist=False) + user = relationship("User") + galaxy_session = relationship("GalaxySession") + history = relationship("History", back_populates="jobs") + library_folder = relationship("LibraryFolder") + parameters = relationship("JobParameter") + input_datasets = relationship("JobToInputDatasetAssociation", back_populates="job") + input_dataset_collections = relationship("JobToInputDatasetCollectionAssociation", back_populates="job") + input_dataset_collection_elements = relationship( + "JobToInputDatasetCollectionElementAssociation", back_populates="job" + ) + output_dataset_collection_instances = relationship("JobToOutputDatasetCollectionAssociation", back_populates="job") + output_dataset_collections = relationship("JobToImplicitOutputDatasetCollectionAssociation", back_populates="job") + post_job_actions = relationship("PostJobActionAssociation", back_populates="job") + input_library_datasets = relationship("JobToInputLibraryDatasetAssociation", back_populates="job") + output_library_datasets = relationship("JobToOutputLibraryDatasetAssociation", back_populates="job") + external_output_metadata = relationship("JobExternalOutputMetadata", back_populates="job") + tasks = relationship("Task", back_populates="job") + output_datasets = relationship("JobToOutputDatasetAssociation", back_populates="job") + state_history = relationship("JobStateHistory") + text_metrics = relationship("JobMetricText") + numeric_metrics = relationship("JobMetricNumeric") + interactivetool_entry_points = relationship("InteractiveToolEntryPoint", back_populates="job", uselist=True) + implicit_collection_jobs_association = relationship( + "ImplicitCollectionJobsJobAssociation", back_populates="job", uselist=False + ) + container = relationship("JobContainerAssociation", back_populates="job", uselist=False) + data_manager_association = relationship("DataManagerJobAssociation", back_populates="job", uselist=False) + history_dataset_collection_associations = relationship("HistoryDatasetCollectionAssociation", back_populates="job") + workflow_invocation_step = relationship("WorkflowInvocationStep", back_populates="job", uselist=False) any_output_dataset_collection_instances_deleted: column_property # defined at the end of this module any_output_dataset_deleted: column_property # defined at the end of this module - dict_collection_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version'] - dict_element_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version', 'command_version'] + dict_collection_visible_keys = ["id", "state", "exit_code", "update_time", "create_time", "galaxy_version"] + dict_element_visible_keys = [ + "id", + "state", + "exit_code", + "update_time", + "create_time", + "galaxy_version", + "command_version", + ] _numeric_metric = JobMetricNumeric _text_metric = JobMetricText class states(str, Enum): - NEW = 'new' - RESUBMITTED = 'resubmitted' - UPLOAD = 'upload' - WAITING = 'waiting' - QUEUED = 'queued' - RUNNING = 'running' - OK = 'ok' - ERROR = 'error' - FAILED = 'failed' - PAUSED = 'paused' - DELETING = 'deleting' - DELETED = 'deleted' - DELETED_NEW = 'deleted_new' # now DELETING, remove after 21.0 - STOPPING = 'stop' - STOPPED = 'stopped' + NEW = "new" + RESUBMITTED = "resubmitted" + UPLOAD = "upload" + WAITING = "waiting" + QUEUED = "queued" + RUNNING = "running" + OK = "ok" + ERROR = "error" + FAILED = "failed" + PAUSED = "paused" + DELETING = "deleting" + DELETED = "deleted" + DELETED_NEW = "deleted_new" # now DELETING, remove after 21.0 + STOPPING = "stop" + STOPPED = "stopped" - terminal_states = [states.OK, - states.ERROR, - states.DELETED] + terminal_states = [states.OK, states.ERROR, states.DELETED] #: job states where the job hasn't finished and the model may still change non_ready_states = [ states.NEW, @@ -1038,7 +1085,9 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): out_data.update([(da.name, da.dataset) for da in self.output_library_datasets]) if not exclude_implicit_outputs: - out_collections = {obj.name: obj.dataset_collection_instance for obj in self.output_dataset_collection_instances} + out_collections = { + obj.name: obj.dataset_collection_instance for obj in self.output_dataset_collection_instances + } else: out_collections = {} for obj in self.output_dataset_collection_instances: @@ -1229,13 +1278,19 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): self.input_dataset_collections.append(JobToInputDatasetCollectionAssociation(name, dataset_collection)) def add_input_dataset_collection_element(self, name, dataset_collection_element): - self.input_dataset_collection_elements.append(JobToInputDatasetCollectionElementAssociation(name, dataset_collection_element)) + self.input_dataset_collection_elements.append( + JobToInputDatasetCollectionElementAssociation(name, dataset_collection_element) + ) def add_output_dataset_collection(self, name, dataset_collection_instance): - self.output_dataset_collection_instances.append(JobToOutputDatasetCollectionAssociation(name, dataset_collection_instance)) + self.output_dataset_collection_instances.append( + JobToOutputDatasetCollectionAssociation(name, dataset_collection_instance) + ) def add_implicit_output_dataset_collection(self, name, dataset_collection): - self.output_dataset_collections.append(JobToImplicitOutputDatasetCollectionAssociation(name, dataset_collection)) + self.output_dataset_collections.append( + JobToImplicitOutputDatasetCollectionAssociation(name, dataset_collection) + ) def add_input_library_dataset(self, name, dataset): self.input_library_datasets.append(JobToInputLibraryDatasetAssociation(name, dataset)) @@ -1319,9 +1374,9 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): for dataset in dataset.dataset.history_associations: # propagate info across shared datasets dataset.deleted = True - dataset.blurb = 'deleted' - dataset.peek = 'Job deleted' - dataset.info = 'Job output deleted by user before job completed' + dataset.blurb = "deleted" + dataset.peek = "Job deleted" + dataset.info = "Job output deleted by user before job completed" def mark_failed(self, info="Job execution failed", blurb=None, peek=None): """ @@ -1354,21 +1409,21 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): def _serialize(self, id_encoder, serialization_options): job_attrs = dict_for(self) serialization_options.attach_identifier(id_encoder, self, job_attrs) - job_attrs['tool_id'] = self.tool_id - job_attrs['tool_version'] = self.tool_version - job_attrs['galaxy_version'] = self.galaxy_version - job_attrs['state'] = self.state - job_attrs['info'] = self.info - job_attrs['traceback'] = self.traceback - job_attrs['command_line'] = self.command_line - job_attrs['tool_stderr'] = self.tool_stderr - job_attrs['job_stderr'] = self.job_stderr - job_attrs['tool_stdout'] = self.tool_stdout - job_attrs['job_stdout'] = self.job_stdout - job_attrs['exit_code'] = self.exit_code - job_attrs['create_time'] = self.create_time.isoformat() - job_attrs['update_time'] = self.update_time.isoformat() - job_attrs['job_messages'] = self.job_messages + job_attrs["tool_id"] = self.tool_id + job_attrs["tool_version"] = self.tool_version + job_attrs["galaxy_version"] = self.galaxy_version + job_attrs["state"] = self.state + job_attrs["info"] = self.info + job_attrs["traceback"] = self.traceback + job_attrs["command_line"] = self.command_line + job_attrs["tool_stderr"] = self.tool_stderr + job_attrs["job_stderr"] = self.job_stderr + job_attrs["tool_stdout"] = self.tool_stdout + job_attrs["job_stdout"] = self.job_stdout + job_attrs["exit_code"] = self.exit_code + job_attrs["create_time"] = self.create_time.isoformat() + job_attrs["update_time"] = self.update_time.isoformat() + job_attrs["job_messages"] = self.job_messages # Get the job's parameters param_dict = self.raw_param_dict() @@ -1389,87 +1444,100 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): params_dict = {} for name, value in params_objects.items(): params_dict[name] = value - job_attrs['params'] = params_dict + job_attrs["params"] = params_dict return job_attrs - def to_dict(self, view='collection', system_details=False): - if view == 'admin_job_list': - rval = super().to_dict(view='collection') + def to_dict(self, view="collection", system_details=False): + if view == "admin_job_list": + rval = super().to_dict(view="collection") else: rval = super().to_dict(view=view) - rval['tool_id'] = self.tool_id - rval['history_id'] = self.history_id - if system_details or view == 'admin_job_list': + rval["tool_id"] = self.tool_id + rval["history_id"] = self.history_id + if system_details or view == "admin_job_list": # System level details that only admins should have. - rval['external_id'] = self.job_runner_external_id - rval['command_line'] = self.command_line - rval['traceback'] = self.traceback - if view == 'admin_job_list': - rval['user_email'] = self.user.email if self.user else None - rval['handler'] = self.handler - rval['job_runner_name'] = self.job_runner_name - rval['info'] = self.info - rval['session_id'] = self.session_id + rval["external_id"] = self.job_runner_external_id + rval["command_line"] = self.command_line + rval["traceback"] = self.traceback + if view == "admin_job_list": + rval["user_email"] = self.user.email if self.user else None + rval["handler"] = self.handler + rval["job_runner_name"] = self.job_runner_name + rval["info"] = self.info + rval["session_id"] = self.session_id if self.galaxy_session and self.galaxy_session.remote_host: - rval['remote_host'] = self.galaxy_session.remote_host - if view == 'element': + rval["remote_host"] = self.galaxy_session.remote_host + if view == "element": param_dict = {p.name: p.value for p in self.parameters} - rval['params'] = param_dict + rval["params"] = param_dict input_dict = {} for i in self.input_datasets: if i.dataset is not None: input_dict[i.name] = { - "id": i.dataset.id, "src": "hda", - "uuid": str(i.dataset.dataset.uuid) if i.dataset.dataset.uuid is not None else None + "id": i.dataset.id, + "src": "hda", + "uuid": str(i.dataset.dataset.uuid) if i.dataset.dataset.uuid is not None else None, } for i in self.input_library_datasets: if i.dataset is not None: input_dict[i.name] = { - "id": i.dataset.id, "src": "ldda", - "uuid": str(i.dataset.dataset.uuid) if i.dataset.dataset.uuid is not None else None + "id": i.dataset.id, + "src": "ldda", + "uuid": str(i.dataset.dataset.uuid) if i.dataset.dataset.uuid is not None else None, } for k in input_dict: if k in param_dict: del param_dict[k] - rval['inputs'] = input_dict + rval["inputs"] = input_dict output_dict = {} for i in self.output_datasets: if i.dataset is not None: output_dict[i.name] = { - "id": i.dataset.id, "src": "hda", - "uuid": str(i.dataset.dataset.uuid) if i.dataset.dataset.uuid is not None else None + "id": i.dataset.id, + "src": "hda", + "uuid": str(i.dataset.dataset.uuid) if i.dataset.dataset.uuid is not None else None, } for i in self.output_library_datasets: if i.dataset is not None: output_dict[i.name] = { - "id": i.dataset.id, "src": "ldda", - "uuid": str(i.dataset.dataset.uuid) if i.dataset.dataset.uuid is not None else None + "id": i.dataset.id, + "src": "ldda", + "uuid": str(i.dataset.dataset.uuid) if i.dataset.dataset.uuid is not None else None, } - rval['outputs'] = output_dict - rval['output_collections'] = {jtodca.name: {'id': jtodca.dataset_collection_instance.id, 'src': 'hdca'} for jtodca in self.output_dataset_collection_instances} + rval["outputs"] = output_dict + rval["output_collections"] = { + jtodca.name: {"id": jtodca.dataset_collection_instance.id, "src": "hdca"} + for jtodca in self.output_dataset_collection_instances + } return rval def update_hdca_update_time_for_job(self, update_time, sa_session, supports_skip_locked): - subq = sa_session.query(HistoryDatasetCollectionAssociation.id) \ - .join(ImplicitCollectionJobs) \ - .join(ImplicitCollectionJobsJobAssociation) \ + subq = ( + sa_session.query(HistoryDatasetCollectionAssociation.id) + .join(ImplicitCollectionJobs) + .join(ImplicitCollectionJobsJobAssociation) .filter(ImplicitCollectionJobsJobAssociation.job_id == self.id) + ) if supports_skip_locked: subq = subq.with_for_update(skip_locked=True).subquery() - implicit_statement = HistoryDatasetCollectionAssociation.table.update() \ - .where(HistoryDatasetCollectionAssociation.table.c.id.in_(select(subq))) \ + implicit_statement = ( + HistoryDatasetCollectionAssociation.table.update() + .where(HistoryDatasetCollectionAssociation.table.c.id.in_(select(subq))) .values(update_time=update_time) - explicit_statement = HistoryDatasetCollectionAssociation.table.update() \ - .where(HistoryDatasetCollectionAssociation.table.c.job_id == self.id) \ + ) + explicit_statement = ( + HistoryDatasetCollectionAssociation.table.update() + .where(HistoryDatasetCollectionAssociation.table.c.job_id == self.id) .values(update_time=update_time) + ) sa_session.execute(explicit_statement) if supports_skip_locked: sa_session.execute(implicit_statement) else: - conn = sa_session.connection(execution_options={'isolation_level': 'SERIALIZABLE'}) + conn = sa_session.connection(execution_options={"isolation_level": "SERIALIZABLE"}) with conn.begin() as trans: try: conn.execute(implicit_statement) @@ -1477,29 +1545,30 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): except OperationalError as e: # If this is a serialization failure on PostgreSQL, then e.orig is a psycopg2 TransactionRollbackError # and should have attribute `code`. Other engines should just report the message and move on. - if int(getattr(e.orig, 'pgcode', -1)) != 40001: - log.debug(f"Updating implicit collection uptime_time for job {self.id} failed (this is expected for large collections and not a problem): {unicodify(e)}") + if int(getattr(e.orig, "pgcode", -1)) != 40001: + log.debug( + f"Updating implicit collection uptime_time for job {self.id} failed (this is expected for large collections and not a problem): {unicodify(e)}" + ) trans.rollback() def set_final_state(self, final_state, supports_skip_locked): self.set_state(final_state) # TODO: migrate to where-in subqueries? - statement = ''' + statement = """ UPDATE workflow_invocation_step SET update_time = :update_time WHERE job_id = :job_id; - ''' + """ sa_session = object_session(self) update_time = now() - self.update_hdca_update_time_for_job(update_time=update_time, sa_session=sa_session, supports_skip_locked=supports_skip_locked) - params = { - 'job_id': self.id, - 'update_time': update_time - } + self.update_hdca_update_time_for_job( + update_time=update_time, sa_session=sa_session, supports_skip_locked=supports_skip_locked + ) + params = {"job_id": self.id, "update_time": update_time} sa_session.execute(statement, params) def get_destination_configuration(self, dest_params, config, key, default=None): - """ Get a destination parameter that can be defaulted back + """Get a destination parameter that can be defaulted back in specified config if it needs to be applied globally. """ param_unspecified = object() @@ -1520,7 +1589,8 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): def update_output_states(self, supports_skip_locked): # TODO: migrate to where-in subqueries? - statements = [''' + statements = [ + """ UPDATE dataset SET state = :state, @@ -1530,7 +1600,8 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): INNER JOIN job_to_output_dataset jtod ON jtod.dataset_id = hda.id AND jtod.job_id = :job_id ); - ''', ''' + """, + """ UPDATE dataset SET state = :state, @@ -1540,7 +1611,8 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): INNER JOIN job_to_output_library_dataset jtold ON jtold.ldda_id = ldda.id AND jtold.job_id = :job_id ); - ''', ''' + """, + """ UPDATE history_dataset_association SET info = :info, @@ -1550,7 +1622,8 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): FROM job_to_output_dataset jtod WHERE jtod.job_id = :job_id ); - ''', ''' + """, + """ UPDATE library_dataset_dataset_association SET info = :info, @@ -1560,16 +1633,14 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): FROM job_to_output_library_dataset jtold WHERE jtold.job_id = :job_id ); - '''] + """, + ] sa_session = object_session(self) update_time = now() - self.update_hdca_update_time_for_job(update_time=update_time, sa_session=sa_session, supports_skip_locked=supports_skip_locked) - params = { - 'job_id': self.id, - 'state': self.state, - 'info': self.info, - 'update_time': update_time - } + self.update_hdca_update_time_for_job( + update_time=update_time, sa_session=sa_session, supports_skip_locked=supports_skip_locked + ) + params = {"job_id": self.id, "state": self.state, "info": self.info, "update_time": update_time} for statement in statements: sa_session.execute(statement, params) @@ -1584,7 +1655,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): return True if self.output_dataset_collection_instances: # We'll want to replace this item - return 'job_produced_collection_elements' + return "job_produced_collection_elements" except Exception: log.exception(f"Error trying to determine if job {self.id} is remappable") return False @@ -1600,7 +1671,8 @@ class Task(Base, JobLike, RepresentById): """ A task represents a single component of a job. """ - __tablename__ = 'task' + + __tablename__ = "task" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) @@ -1618,26 +1690,26 @@ class Task(Base, JobLike, RepresentById): job_messages = Column(MutableJSONType, nullable=True) info = Column(TrimmedString(255)) traceback = Column(TEXT) - job_id = Column(Integer, ForeignKey('job.id'), index=True, nullable=False) + job_id = Column(Integer, ForeignKey("job.id"), index=True, nullable=False) working_directory = Column(String(1024)) task_runner_name = Column(String(255)) task_runner_external_id = Column(String(255)) prepare_input_files_cmd = Column(TEXT) - job = relationship('Job', back_populates='tasks') - text_metrics = relationship('TaskMetricText') - numeric_metrics = relationship('TaskMetricNumeric') + job = relationship("Job", back_populates="tasks") + text_metrics = relationship("TaskMetricText") + numeric_metrics = relationship("TaskMetricNumeric") _numeric_metric = TaskMetricNumeric _text_metric = TaskMetricText class states(str, Enum): - NEW = 'new' - WAITING = 'waiting' - QUEUED = 'queued' - RUNNING = 'running' - OK = 'ok' - ERROR = 'error' - DELETED = 'deleted' + NEW = "new" + WAITING = "waiting" + QUEUED = "queued" + RUNNING = "running" + OK = "ok" + ERROR = "error" + DELETED = "deleted" # Please include an accessor (get/set pair) for any new columns/members. def __init__(self, job, working_directory, prepare_files_cmd): @@ -1755,8 +1827,7 @@ class Task(Base, JobLike, RepresentById): # This method is available for runners that do not want/need to # differentiate between the kinds of Runnable things (Jobs and Tasks) # that they're using. - log.debug("Task %d: Set external id to %s" - % (self.id, task_runner_external_id)) + log.debug("Task %d: Set external id to %s" % (self.id, task_runner_external_id)) self.task_runner_external_id = task_runner_external_id def set_task_runner_external_id(self, task_runner_external_id): @@ -1770,10 +1841,10 @@ class Task(Base, JobLike, RepresentById): class JobParameter(Base, RepresentById): - __tablename__ = 'job_parameter' + __tablename__ = "job_parameter" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) name = Column(String(255)) value = Column(TEXT) @@ -1786,16 +1857,15 @@ class JobParameter(Base, RepresentById): class JobToInputDatasetAssociation(Base, RepresentById): - __tablename__ = 'job_to_input_dataset' + __tablename__ = "job_to_input_dataset" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) - dataset_id = Column(Integer, - ForeignKey('history_dataset_association.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) + dataset_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) dataset_version = Column(Integer) name = Column(String(255)) - dataset = relationship('HistoryDatasetAssociation', lazy="joined", back_populates='dependent_jobs') - job = relationship('Job', back_populates='input_datasets') + dataset = relationship("HistoryDatasetAssociation", lazy="joined", back_populates="dependent_jobs") + job = relationship("Job", back_populates="input_datasets") def __init__(self, name, dataset): self.name = name @@ -1804,15 +1874,14 @@ class JobToInputDatasetAssociation(Base, RepresentById): class JobToOutputDatasetAssociation(Base, RepresentById): - __tablename__ = 'job_to_output_dataset' + __tablename__ = "job_to_output_dataset" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) - dataset_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) + dataset_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) name = Column(String(255)) - dataset = relationship('HistoryDatasetAssociation', - lazy="joined", back_populates='creating_job_associations') - job = relationship('Job', back_populates='output_datasets') + dataset = relationship("HistoryDatasetAssociation", lazy="joined", back_populates="creating_job_associations") + job = relationship("Job", back_populates="output_datasets") def __init__(self, name, dataset): self.name = name @@ -1824,15 +1893,14 @@ class JobToOutputDatasetAssociation(Base, RepresentById): class JobToInputDatasetCollectionAssociation(Base, RepresentById): - __tablename__ = 'job_to_input_dataset_collection' + __tablename__ = "job_to_input_dataset_collection" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) - dataset_collection_id = Column(Integer, - ForeignKey('history_dataset_collection_association.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) + dataset_collection_id = Column(Integer, ForeignKey("history_dataset_collection_association.id"), index=True) name = Column(String(255)) - dataset_collection = relationship('HistoryDatasetCollectionAssociation', lazy="joined") - job = relationship('Job', back_populates='input_dataset_collections') + dataset_collection = relationship("HistoryDatasetCollectionAssociation", lazy="joined") + job = relationship("Job", back_populates="input_dataset_collections") def __init__(self, name, dataset_collection): self.name = name @@ -1840,15 +1908,14 @@ class JobToInputDatasetCollectionAssociation(Base, RepresentById): class JobToInputDatasetCollectionElementAssociation(Base, RepresentById): - __tablename__ = 'job_to_input_dataset_collection_element' + __tablename__ = "job_to_input_dataset_collection_element" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) - dataset_collection_element_id = Column(Integer, - ForeignKey('dataset_collection_element.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) + dataset_collection_element_id = Column(Integer, ForeignKey("dataset_collection_element.id"), index=True) name = Column(Unicode(255)) - dataset_collection_element = relationship('DatasetCollectionElement', lazy="joined") - job = relationship('Job', back_populates='input_dataset_collection_elements') + dataset_collection_element = relationship("DatasetCollectionElement", lazy="joined") + job = relationship("Job", back_populates="input_dataset_collection_elements") def __init__(self, name, dataset_collection_element): self.name = name @@ -1858,15 +1925,14 @@ class JobToInputDatasetCollectionElementAssociation(Base, RepresentById): # Many jobs may map to one HistoryDatasetCollection using these for a given # tool output (if mapping over an input collection). class JobToOutputDatasetCollectionAssociation(Base, RepresentById): - __tablename__ = 'job_to_output_dataset_collection' + __tablename__ = "job_to_output_dataset_collection" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) - dataset_collection_id = Column(Integer, - ForeignKey('history_dataset_collection_association.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) + dataset_collection_id = Column(Integer, ForeignKey("history_dataset_collection_association.id"), index=True) name = Column(Unicode(255)) - dataset_collection_instance = relationship('HistoryDatasetCollectionAssociation', lazy="joined") - job = relationship('Job', back_populates='output_dataset_collection_instances') + dataset_collection_instance = relationship("HistoryDatasetCollectionAssociation", lazy="joined") + job = relationship("Job", back_populates="output_dataset_collection_instances") def __init__(self, name, dataset_collection_instance): self.name = name @@ -1881,14 +1947,14 @@ class JobToOutputDatasetCollectionAssociation(Base, RepresentById): # using these. (You can think of many of these models as going into the # creation of a JobToOutputDatasetCollectionAssociation.) class JobToImplicitOutputDatasetCollectionAssociation(Base, RepresentById): - __tablename__ = 'job_to_implicit_output_dataset_collection' + __tablename__ = "job_to_implicit_output_dataset_collection" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) - dataset_collection_id = Column(Integer, ForeignKey('dataset_collection.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) + dataset_collection_id = Column(Integer, ForeignKey("dataset_collection.id"), index=True) name = Column(Unicode(255)) - dataset_collection = relationship('DatasetCollection') - job = relationship('Job', back_populates='output_dataset_collections') + dataset_collection = relationship("DatasetCollection") + job = relationship("Job", back_populates="output_dataset_collections") def __init__(self, name, dataset_collection): self.name = name @@ -1896,15 +1962,14 @@ class JobToImplicitOutputDatasetCollectionAssociation(Base, RepresentById): class JobToInputLibraryDatasetAssociation(Base, RepresentById): - __tablename__ = 'job_to_input_library_dataset' + __tablename__ = "job_to_input_library_dataset" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) - ldda_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) + ldda_id = Column(Integer, ForeignKey("library_dataset_dataset_association.id"), index=True) name = Column(Unicode(255)) - job = relationship('Job', back_populates='input_library_datasets') - dataset = relationship( - 'LibraryDatasetDatasetAssociation', lazy="joined", back_populates='dependent_jobs') + job = relationship("Job", back_populates="input_library_datasets") + dataset = relationship("LibraryDatasetDatasetAssociation", lazy="joined", back_populates="dependent_jobs") def __init__(self, name, dataset): self.name = name @@ -1912,15 +1977,16 @@ class JobToInputLibraryDatasetAssociation(Base, RepresentById): class JobToOutputLibraryDatasetAssociation(Base, RepresentById): - __tablename__ = 'job_to_output_library_dataset' + __tablename__ = "job_to_output_library_dataset" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) - ldda_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) + ldda_id = Column(Integer, ForeignKey("library_dataset_dataset_association.id"), index=True) name = Column(Unicode(255)) - job = relationship('Job', back_populates='output_library_datasets') + job = relationship("Job", back_populates="output_library_datasets") dataset = relationship( - 'LibraryDatasetDatasetAssociation', lazy="joined", back_populates='creating_job_associations') + "LibraryDatasetDatasetAssociation", lazy="joined", back_populates="creating_job_associations" + ) def __init__(self, name, dataset): self.name = name @@ -1928,12 +1994,12 @@ class JobToOutputLibraryDatasetAssociation(Base, RepresentById): class JobStateHistory(Base, RepresentById): - __tablename__ = 'job_state_history' + __tablename__ = "job_state_history" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - job_id = Column(Integer, ForeignKey('job.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) state = Column(String(64), index=True) info = Column(TrimmedString(255)) @@ -1944,18 +2010,19 @@ class JobStateHistory(Base, RepresentById): class ImplicitlyCreatedDatasetCollectionInput(Base, RepresentById): - __tablename__ = 'implicitly_created_dataset_collection_inputs' + __tablename__ = "implicitly_created_dataset_collection_inputs" id = Column(Integer, primary_key=True) - dataset_collection_id = Column(Integer, - ForeignKey('history_dataset_collection_association.id'), index=True) - input_dataset_collection_id = Column(Integer, - ForeignKey('history_dataset_collection_association.id'), index=True) + dataset_collection_id = Column(Integer, ForeignKey("history_dataset_collection_association.id"), index=True) + input_dataset_collection_id = Column(Integer, ForeignKey("history_dataset_collection_association.id"), index=True) name = Column(Unicode(255)) - input_dataset_collection = relationship('HistoryDatasetCollectionAssociation', - primaryjoin=(lambda: HistoryDatasetCollectionAssociation.id # type: ignore[has-type] - == ImplicitlyCreatedDatasetCollectionInput.input_dataset_collection_id) # type: ignore[has-type] + input_dataset_collection = relationship( + "HistoryDatasetCollectionAssociation", + primaryjoin=( + lambda: HistoryDatasetCollectionAssociation.id # type: ignore[has-type] + == ImplicitlyCreatedDatasetCollectionInput.input_dataset_collection_id + ), # type: ignore[has-type] ) def __init__(self, name, input_dataset_collection): @@ -1964,17 +2031,16 @@ class ImplicitlyCreatedDatasetCollectionInput(Base, RepresentById): class ImplicitCollectionJobs(Base, Serializable): - __tablename__ = 'implicit_collection_jobs' + __tablename__ = "implicit_collection_jobs" id = Column(Integer, primary_key=True) - populated_state = Column(TrimmedString(64), default='new', nullable=False) - jobs = relationship('ImplicitCollectionJobsJobAssociation', - back_populates='implicit_collection_jobs') + populated_state = Column(TrimmedString(64), default="new", nullable=False) + jobs = relationship("ImplicitCollectionJobsJobAssociation", back_populates="implicit_collection_jobs") class populated_states(str, Enum): - NEW = 'new' # New implicit jobs object, unpopulated job associations - OK = 'ok' # Job associations are set and fixed. - FAILED = 'failed' # There were issues populating job associations, object is in error. + NEW = "new" # New implicit jobs object, unpopulated job associations + OK = "ok" # Job associations are set and fixed. + FAILED = "failed" # There were issues populating job associations, object is in error. def __init__(self, populated_state=None): self.populated_state = populated_state or ImplicitCollectionJobs.populated_states.NEW @@ -1987,34 +2053,35 @@ class ImplicitCollectionJobs(Base, Serializable): rval = dict_for( self, populated_state=self.populated_state, - jobs=[serialization_options.get_identifier(id_encoder, j_a.job) for j_a in self.jobs] + jobs=[serialization_options.get_identifier(id_encoder, j_a.job) for j_a in self.jobs], ) serialization_options.attach_identifier(id_encoder, self, rval) return rval class ImplicitCollectionJobsJobAssociation(Base, RepresentById): - __tablename__ = 'implicit_collection_jobs_job_association' + __tablename__ = "implicit_collection_jobs_job_association" id = Column(Integer, primary_key=True) - implicit_collection_jobs_id = Column(Integer, ForeignKey('implicit_collection_jobs.id'), index=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) # Consider making this nullable... + implicit_collection_jobs_id = Column(Integer, ForeignKey("implicit_collection_jobs.id"), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) # Consider making this nullable... order_index = Column(Integer, nullable=False) - implicit_collection_jobs = relationship('ImplicitCollectionJobs', back_populates='jobs') - job = relationship('Job', back_populates='implicit_collection_jobs_association') + implicit_collection_jobs = relationship("ImplicitCollectionJobs", back_populates="jobs") + job = relationship("Job", back_populates="implicit_collection_jobs_association") class PostJobAction(Base, RepresentById): - __tablename__ = 'post_job_action' + __tablename__ = "post_job_action" id = Column(Integer, primary_key=True) - workflow_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True, nullable=True) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id"), index=True, nullable=True) action_type = Column(String(255), nullable=False) output_name = Column(String(255), nullable=True) action_arguments = Column(MutableJSONType, nullable=True) - workflow_step = relationship('WorkflowStep', - back_populates='post_job_actions', - primaryjoin=(lambda: WorkflowStep.id == PostJobAction.workflow_step_id) # type: ignore[has-type] + workflow_step = relationship( + "WorkflowStep", + back_populates="post_job_actions", + primaryjoin=(lambda: WorkflowStep.id == PostJobAction.workflow_step_id), # type: ignore[has-type] ) def __init__(self, action_type, workflow_step=None, output_name=None, action_arguments=None): @@ -2025,13 +2092,13 @@ class PostJobAction(Base, RepresentById): class PostJobActionAssociation(Base, RepresentById): - __tablename__ = 'post_job_action_association' + __tablename__ = "post_job_action_association" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True, nullable=False) - post_job_action_id = Column(Integer, ForeignKey('post_job_action.id'), index=True, nullable=False) - post_job_action = relationship('PostJobAction') - job = relationship('Job', back_populates='post_job_actions') + job_id = Column(Integer, ForeignKey("job.id"), index=True, nullable=False) + post_job_action_id = Column(Integer, ForeignKey("post_job_action.id"), index=True, nullable=False) + post_job_action = relationship("PostJobAction") + job = relationship("Job", back_populates="post_job_actions") def __init__(self, pja, job=None, job_id=None): if job is not None: @@ -2044,14 +2111,16 @@ class PostJobActionAssociation(Base, RepresentById): class JobExternalOutputMetadata(Base, RepresentById): - __tablename__ = 'job_external_output_metadata' + __tablename__ = "job_external_output_metadata" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) - history_dataset_association_id = Column(Integer, - ForeignKey('history_dataset_association.id'), index=True, nullable=True) - library_dataset_dataset_association_id = Column(Integer, - ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) + history_dataset_association_id = Column( + Integer, ForeignKey("history_dataset_association.id"), index=True, nullable=True + ) + library_dataset_dataset_association_id = Column( + Integer, ForeignKey("library_dataset_dataset_association.id"), index=True, nullable=True + ) is_valid = Column(Boolean, default=True) filename_in = Column(String(255)) filename_out = Column(String(255)) @@ -2059,9 +2128,9 @@ class JobExternalOutputMetadata(Base, RepresentById): filename_kwds = Column(String(255)) filename_override_metadata = Column(String(255)) job_runner_external_pid = Column(String(255)) - history_dataset_association = relationship('HistoryDatasetAssociation', lazy="joined") - library_dataset_dataset_association = relationship('LibraryDatasetDatasetAssociation', lazy="joined") - job = relationship('Job', back_populates='external_output_metadata') + history_dataset_association = relationship("HistoryDatasetAssociation", lazy="joined") + library_dataset_dataset_association = relationship("LibraryDatasetDatasetAssociation", lazy="joined") + job = relationship("Job", back_populates="external_output_metadata") def __init__(self, job=None, dataset=None): self.job = job @@ -2096,19 +2165,19 @@ class FakeDatasetAssociation: class JobExportHistoryArchive(Base, RepresentById): - __tablename__ = 'job_export_history_archive' + __tablename__ = "job_export_history_archive" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) - history_id = Column(Integer, ForeignKey('history.id'), index=True) - dataset_id = Column(Integer, ForeignKey('dataset.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) + history_id = Column(Integer, ForeignKey("history.id"), index=True) + dataset_id = Column(Integer, ForeignKey("dataset.id"), index=True) compressed = Column(Boolean, index=True, default=False) history_attrs_filename = Column(TEXT) - job = relationship('Job') - dataset = relationship('Dataset') - history = relationship('History', back_populates='exports') + job = relationship("Job") + dataset = relationship("Dataset") + history = relationship("History", back_populates="exports") - ATTRS_FILENAME_HISTORY = 'history_attrs.txt' + ATTRS_FILENAME_HISTORY = "history_attrs.txt" def __init__(self, compressed=False, **kwd): super().__init__(**kwd) @@ -2124,12 +2193,11 @@ class JobExportHistoryArchive(Base, RepresentById): @property def up_to_date(self): - """ Return False, if a new export should be generated for corresponding + """Return False, if a new export should be generated for corresponding history. """ job = self.job - return job.state not in [Job.states.ERROR, Job.states.DELETED] \ - and job.update_time > self.history.update_time + return job.state not in [Job.states.ERROR, Job.states.DELETED] and job.update_time > self.history.update_time @property def ready(self): @@ -2156,11 +2224,7 @@ class JobExportHistoryArchive(Base, RepresentById): sa_session.flush() # ensure job.id and archive_dataset.id are available object_store.create(archive_dataset) # set the object store id, create dataset (if applicable) # Add association for keeping track of job, history, archive relationship. - jeha = JobExportHistoryArchive( - job=job, history=history, - dataset=archive_dataset, - compressed=compressed - ) + jeha = JobExportHistoryArchive(job=job, history=history, dataset=archive_dataset, compressed=compressed) sa_session.add(jeha) # @@ -2175,36 +2239,36 @@ class JobExportHistoryArchive(Base, RepresentById): def to_dict(self): return { - 'id': self.id, - 'job_id': self.job.id, - 'ready': self.ready, - 'preparing': self.preparing, - 'up_to_date': self.up_to_date, + "id": self.id, + "job_id": self.job.id, + "ready": self.ready, + "preparing": self.preparing, + "up_to_date": self.up_to_date, } class JobImportHistoryArchive(Base, RepresentById): - __tablename__ = 'job_import_history_archive' + __tablename__ = "job_import_history_archive" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) - history_id = Column(Integer, ForeignKey('history.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) + history_id = Column(Integer, ForeignKey("history.id"), index=True) archive_dir = Column(TEXT) - job = relationship('Job') - history = relationship('History') + job = relationship("Job") + history = relationship("History") class JobContainerAssociation(Base, RepresentById): - __tablename__ = 'job_container_association' + __tablename__ = "job_container_association" id = Column(Integer, primary_key=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) container_type = Column(TEXT) container_name = Column(TEXT) container_info = Column(MutableJSONType, nullable=True) created_time = Column(DateTime, default=now) modified_time = Column(DateTime, default=now, onupdate=now) - job = relationship('Job', back_populates='container') + job = relationship("Job", back_populates="container") def __init__(self, **kwd): super().__init__(**kwd) @@ -2212,7 +2276,7 @@ class JobContainerAssociation(Base, RepresentById): class InteractiveToolEntryPoint(Base, Dictifiable, RepresentById): - __tablename__ = 'interactivetool_entry_point' + __tablename__ = "interactivetool_entry_point" id = Column(Integer, primary_key=True) job_id = Column(Integer, ForeignKey("job.id"), index=True) @@ -2229,10 +2293,10 @@ class InteractiveToolEntryPoint(Base, Dictifiable, RepresentById): deleted = Column(Boolean, default=False) created_time = Column(DateTime, default=now) modified_time = Column(DateTime, default=now, onupdate=now) - job = relationship('Job', back_populates='interactivetool_entry_points', uselist=False) + job = relationship("Job", back_populates="interactivetool_entry_points", uselist=False) - dict_collection_visible_keys = ['id', 'name', 'active', 'created_time', 'modified_time'] - dict_element_visible_keys = ['id', 'name', 'active', 'created_time', 'modified_time'] + dict_collection_visible_keys = ["id", "name", "active", "created_time", "modified_time"] + dict_element_visible_keys = ["id", "name", "active", "created_time", "modified_time"] def __init__(self, requires_domain=True, configured=False, deleted=False, **kwd): super().__init__(**kwd) @@ -2251,35 +2315,35 @@ class InteractiveToolEntryPoint(Base, Dictifiable, RepresentById): class GenomeIndexToolData(Base, RepresentById): # TODO: params arg is lost - __tablename__ = 'genome_index_tool_data' + __tablename__ = "genome_index_tool_data" id = Column(Integer, primary_key=True) job_id = Column(Integer, ForeignKey("job.id"), index=True) - dataset_id = Column(Integer, ForeignKey('dataset.id'), index=True) + dataset_id = Column(Integer, ForeignKey("dataset.id"), index=True) fasta_path = Column(String(255)) created_time = Column(DateTime, default=now) modified_time = Column(DateTime, default=now, onupdate=now) indexer = Column(String(64)) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) - job = relationship('Job') - dataset = relationship('Dataset') - user = relationship('User') + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) + job = relationship("Job") + dataset = relationship("Dataset") + user = relationship("User") class Group(Base, Dictifiable, RepresentById): - __tablename__ = 'galaxy_group' + __tablename__ = "galaxy_group" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) name = Column(String(255), index=True, unique=True) deleted = Column(Boolean, index=True, default=False) - quotas = relationship('GroupQuotaAssociation', back_populates='group') - roles = relationship('GroupRoleAssociation', back_populates='group') - users = relationship('UserGroupAssociation', back_populates='group') + quotas = relationship("GroupQuotaAssociation", back_populates="group") + roles = relationship("GroupRoleAssociation", back_populates="group") + users = relationship("UserGroupAssociation", back_populates="group") - dict_collection_visible_keys = ['id', 'name'] - dict_element_visible_keys = ['id', 'name'] + dict_collection_visible_keys = ["id", "name"] + dict_element_visible_keys = ["id", "name"] def __init__(self, name=None): self.name = name @@ -2287,15 +2351,15 @@ class Group(Base, Dictifiable, RepresentById): class UserGroupAssociation(Base, RepresentById): - __tablename__ = 'user_group_association' + __tablename__ = "user_group_association" id = Column(Integer, primary_key=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) - group_id = Column(Integer, ForeignKey('galaxy_group.id'), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) + group_id = Column(Integer, ForeignKey("galaxy_group.id"), index=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - user = relationship('User', back_populates='groups') - group = relationship('Group', back_populates='users') + user = relationship("User", back_populates="groups") + group = relationship("Group", back_populates="users") def __init__(self, user, group): self.user = user @@ -2307,12 +2371,10 @@ def is_hda(d): class HistoryAudit(Base, RepresentById): - __tablename__ = 'history_audit' - __table_args__ = ( - PrimaryKeyConstraint(sqlite_on_conflict='IGNORE'), - ) + __tablename__ = "history_audit" + __table_args__ = (PrimaryKeyConstraint(sqlite_on_conflict="IGNORE"),) - history_id = Column(Integer, ForeignKey('history.id'), primary_key=True, nullable=False) + history_id = Column(Integer, ForeignKey("history.id"), primary_key=True, nullable=False) update_time = Column(DateTime, default=now, primary_key=True, nullable=False) # This class should never be instantiated. @@ -2321,29 +2383,35 @@ class HistoryAudit(Base, RepresentById): @classmethod def prune(cls, sa_session): - latest_subq = sa_session.query( - cls.history_id, - func.max(cls.update_time).label('max_update_time')).group_by(cls.history_id).subquery() - not_latest_query = sa_session.query( - cls.history_id, cls.update_time - ).select_from(latest_subq).join( - cls, and_( - cls.update_time < latest_subq.columns.max_update_time, - cls.history_id == latest_subq.columns.history_id)).subquery() + latest_subq = ( + sa_session.query(cls.history_id, func.max(cls.update_time).label("max_update_time")) + .group_by(cls.history_id) + .subquery() + ) + not_latest_query = ( + sa_session.query(cls.history_id, cls.update_time) + .select_from(latest_subq) + .join( + cls, + and_( + cls.update_time < latest_subq.columns.max_update_time, + cls.history_id == latest_subq.columns.history_id, + ), + ) + .subquery() + ) q = cls.__table__.delete().where(tuple_(cls.history_id, cls.update_time).in_(select(not_latest_query))) sa_session.execute(q) class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable): - __tablename__ = 'history' - __table_args__ = ( - Index('ix_history_slug', 'slug', mysql_length=200), - ) + __tablename__ = "history" + __table_args__ = (Index("ix_history_slug", "slug", mysql_length=200),) id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) - _update_time = Column('update_time', DateTime, index=True, default=now, onupdate=now) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + _update_time = Column("update_time", DateTime, index=True, default=now, onupdate=now) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) name = Column(TrimmedString(255)) hid_counter = Column(Integer, default=1) deleted = Column(Boolean, index=True, default=False) @@ -2354,59 +2422,79 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable slug = Column(TEXT) published = Column(Boolean, index=True, default=False) - datasets = relationship('HistoryDatasetAssociation', - back_populates='history', - order_by=lambda: asc(HistoryDatasetAssociation.hid)) # type: ignore[has-type] - exports = relationship('JobExportHistoryArchive', - back_populates='history', + datasets = relationship( + "HistoryDatasetAssociation", back_populates="history", order_by=lambda: asc(HistoryDatasetAssociation.hid) # type: ignore[has-type] + ) + exports = relationship( + "JobExportHistoryArchive", + back_populates="history", primaryjoin=lambda: JobExportHistoryArchive.history_id == History.id, - order_by=lambda: desc(JobExportHistoryArchive.id)) - active_datasets = relationship('HistoryDatasetAssociation', + order_by=lambda: desc(JobExportHistoryArchive.id), + ) + active_datasets = relationship( + "HistoryDatasetAssociation", primaryjoin=( - lambda: and_(HistoryDatasetAssociation.history_id # type: ignore[attr-defined] - == History.id, not_(HistoryDatasetAssociation.deleted)) # type: ignore[has-type] + lambda: and_( + HistoryDatasetAssociation.history_id == History.id, # type: ignore[attr-defined] + not_(HistoryDatasetAssociation.deleted), # type: ignore[has-type] + ) ), order_by=lambda: asc(HistoryDatasetAssociation.hid), # type: ignore[has-type] - viewonly=True) - dataset_collections = relationship('HistoryDatasetCollectionAssociation', back_populates='history') - active_dataset_collections = relationship('HistoryDatasetCollectionAssociation', + viewonly=True, + ) + dataset_collections = relationship("HistoryDatasetCollectionAssociation", back_populates="history") + active_dataset_collections = relationship( + "HistoryDatasetCollectionAssociation", primaryjoin=( - lambda: (and_(HistoryDatasetCollectionAssociation.history_id == History.id, # type: ignore[has-type] - not_(HistoryDatasetCollectionAssociation.deleted))) # type: ignore[has-type] + lambda: ( + and_( + HistoryDatasetCollectionAssociation.history_id == History.id, # type: ignore[has-type] + not_(HistoryDatasetCollectionAssociation.deleted), # type: ignore[has-type] + ) + ) ), order_by=lambda: asc(HistoryDatasetCollectionAssociation.hid), # type: ignore[has-type] - viewonly=True) - visible_datasets = relationship('HistoryDatasetAssociation', + viewonly=True, + ) + visible_datasets = relationship( + "HistoryDatasetAssociation", primaryjoin=( - lambda: and_(HistoryDatasetAssociation.history_id == History.id, # type: ignore[attr-defined] - not_(HistoryDatasetAssociation.deleted), HistoryDatasetAssociation.visible) # type: ignore[has-type] + lambda: and_( + HistoryDatasetAssociation.history_id == History.id, # type: ignore[attr-defined] + not_(HistoryDatasetAssociation.deleted), # type: ignore[has-type] + HistoryDatasetAssociation.visible, # type: ignore[has-type] + ) ), order_by=lambda: asc(HistoryDatasetAssociation.hid), # type: ignore[has-type] - viewonly=True) - visible_dataset_collections = relationship('HistoryDatasetCollectionAssociation', + viewonly=True, + ) + visible_dataset_collections = relationship( + "HistoryDatasetCollectionAssociation", primaryjoin=( lambda: and_( HistoryDatasetCollectionAssociation.history_id == History.id, # type: ignore[has-type] not_(HistoryDatasetCollectionAssociation.deleted), # type: ignore[has-type] - HistoryDatasetCollectionAssociation.visible) # type: ignore[has-type] + HistoryDatasetCollectionAssociation.visible, # type: ignore[has-type] + ) ), order_by=lambda: asc(HistoryDatasetCollectionAssociation.hid), # type: ignore[has-type] - viewonly=True) - tags = relationship('HistoryTagAssociation', - order_by=lambda: HistoryTagAssociation.id, - back_populates='history') - annotations = relationship('HistoryAnnotationAssociation', - order_by=lambda: HistoryAnnotationAssociation.id, - back_populates='history') - ratings = relationship('HistoryRatingAssociation', + viewonly=True, + ) + tags = relationship("HistoryTagAssociation", order_by=lambda: HistoryTagAssociation.id, back_populates="history") + annotations = relationship( + "HistoryAnnotationAssociation", order_by=lambda: HistoryAnnotationAssociation.id, back_populates="history" + ) + ratings = relationship( + "HistoryRatingAssociation", order_by=lambda: HistoryRatingAssociation.id, # type: ignore[has-type] - back_populates='history') - default_permissions = relationship('DefaultHistoryPermissions', back_populates='history') - users_shared_with = relationship('HistoryUserShareAssociation', back_populates='history') - galaxy_sessions = relationship('GalaxySessionToHistoryAssociation', back_populates='history') - workflow_invocations = relationship('WorkflowInvocation', back_populates='history') - user = relationship('User', back_populates='histories') - jobs = relationship('Job', back_populates='history') + back_populates="history", + ) + default_permissions = relationship("DefaultHistoryPermissions", back_populates="history") + users_shared_with = relationship("HistoryUserShareAssociation", back_populates="history") + galaxy_sessions = relationship("GalaxySessionToHistoryAssociation", back_populates="history") + workflow_invocations = relationship("WorkflowInvocation", back_populates="history") + user = relationship("User", back_populates="histories") + jobs = relationship("Job", back_populates="history") update_time = column_property( select(func.max(HistoryAudit.update_time)).where(HistoryAudit.history_id == id).scalar_subquery(), @@ -2417,12 +2505,22 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable # Set up proxy so that # History.users_shared_with # returns a list of users that history is shared with. - users_shared_with_dot_users = association_proxy('users_shared_with', 'user') + users_shared_with_dot_users = association_proxy("users_shared_with", "user") - dict_collection_visible_keys = ['id', 'name', 'published', 'deleted'] - dict_element_visible_keys = ['id', 'name', 'genome_build', 'deleted', 'purged', 'update_time', - 'published', 'importable', 'slug', 'empty'] - default_name = 'Unnamed history' + dict_collection_visible_keys = ["id", "name", "published", "deleted"] + dict_element_visible_keys = [ + "id", + "name", + "genome_build", + "deleted", + "purged", + "update_time", + "published", + "importable", + "slug", + "empty", + ] + default_name = "Unnamed history" def __init__(self, id=None, name=None, user=None): self.id = id @@ -2456,7 +2554,9 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable def add_pending_items(self, set_output_hid=True): # These are assumed to be either copies of existing datasets or new, empty datasets, # so we don't need to set the quota. - self.add_datasets(object_session(self), self._pending_additions, set_hid=set_output_hid, quota=False, flush=False) + self.add_datasets( + object_session(self), self._pending_additions, set_hid=set_output_hid, quota=False, flush=False + ) self._pending_additions = [] def _next_hid(self, n=1): @@ -2476,7 +2576,7 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable update_stmt = update(table).where(table.c.id == history_id).values(hid_counter=table.c.hid_counter + n) with engine.begin() as conn: - if engine.name in ['postgres', 'postgresql']: + if engine.name in ["postgres", "postgresql"]: stmt = update_stmt.returning(table.c.hid_counter) updated_hid = conn.execute(stmt).scalar() hid = updated_hid - n @@ -2485,7 +2585,7 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable hid = conn.execute(select_stmt).scalar() conn.execute(update_stmt) - session.expire(self, ['hid_counter']) + session.expire(self, ["hid_counter"]) return hid def add_galaxy_session(self, galaxy_session, association=None): @@ -2500,8 +2600,10 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable object_session(self).add(dataset) object_session(self).flush() elif not isinstance(dataset, (HistoryDatasetAssociation, HistoryDatasetCollectionAssociation)): - raise TypeError("You can only add Dataset and HistoryDatasetAssociation instances to a history" - + f" ( you tried to add {str(dataset)} ).") + raise TypeError( + "You can only add Dataset and HistoryDatasetAssociation instances to a history" + + f" ( you tried to add {str(dataset)} )." + ) is_dataset = is_hda(dataset) if parent_id: for data in self.datasets: @@ -2517,13 +2619,15 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable if quota and is_dataset and self.user: self.user.adjust_total_disk_usage(dataset.quota_amount(self.user)) dataset.history = self - if is_dataset and genome_build not in [None, '?']: + if is_dataset and genome_build not in [None, "?"]: self.genome_build = genome_build dataset.history_id = self.id return dataset - def add_datasets(self, sa_session, datasets, parent_id=None, genome_build=None, set_hid=True, quota=True, flush=False): - """ Optimized version of add_dataset above that minimizes database + def add_datasets( + self, sa_session, datasets, parent_id=None, genome_build=None, set_hid=True, quota=True, flush=False + ): + """Optimized version of add_dataset above that minimizes database interactions when adding many datasets and collections to history at once. """ optimize = len(datasets) > 1 and parent_id is None and set_hid @@ -2543,14 +2647,14 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable sa_session.flush() def __add_datasets_optimized(self, datasets, genome_build=None): - """ Optimized version of add_dataset above that minimizes database + """Optimized version of add_dataset above that minimizes database interactions when adding many datasets to history at once under certain circumstances. """ n = len(datasets) base_hid = self._next_hid(n=n) - set_genome = genome_build not in [None, '?'] + set_genome = genome_build not in [None, "?"] for i, dataset in enumerate(datasets): dataset.hid = base_hid + i dataset.history = self @@ -2651,13 +2755,13 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable serialization_options.attach_identifier(id_encoder, self, history_attrs) return history_attrs - def to_dict(self, view='collection', value_mapper=None): + def to_dict(self, view="collection", value_mapper=None): # Get basic value. rval = super().to_dict(view=view, value_mapper=value_mapper) - if view == 'element': - rval['size'] = int(self.disk_size) + if view == "element": + rval["size"] = int(self.disk_size) return rval @@ -2681,8 +2785,7 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable @property def paused_jobs(self): db_session = object_session(self) - return db_session.query(Job).filter(Job.history_id == self.id, - Job.state == Job.states.PAUSED).all() + return db_session.query(Job).filter(Job.history_id == self.id, Job.state == Job.states.PAUSED).all() @hybrid.hybrid_property def disk_size(self): @@ -2693,12 +2796,18 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable # non-.expression part of hybrid.hybrid_property: called when an instance is the namespace (not the class) db_session = object_session(self) rval = db_session.query( - func.sum(db_session.query(HistoryDatasetAssociation.dataset_id, Dataset.total_size).join(Dataset) - .filter(HistoryDatasetAssociation.table.c.history_id == self.id) - .filter(HistoryDatasetAssociation.purged != true()) - .filter(Dataset.purged != true()) - # unique datasets only - .distinct().subquery().c.total_size)).first()[0] + func.sum( + db_session.query(HistoryDatasetAssociation.dataset_id, Dataset.total_size) + .join(Dataset) + .filter(HistoryDatasetAssociation.table.c.history_id == self.id) + .filter(HistoryDatasetAssociation.purged != true()) + .filter(Dataset.purged != true()) + # unique datasets only + .distinct() + .subquery() + .c.total_size + ) + ).first()[0] if rval is None: rval = 0 return rval @@ -2711,15 +2820,18 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable """ # .expression acts as a column_property and should return a scalar # first, get the distinct datasets within a history that are not purged - hda_to_dataset_join = join(HistoryDatasetAssociation, Dataset, - HistoryDatasetAssociation.table.c.dataset_id == Dataset.table.c.id) + hda_to_dataset_join = join( + HistoryDatasetAssociation, Dataset, HistoryDatasetAssociation.table.c.dataset_id == Dataset.table.c.id + ) distinct_datasets = ( - select([ - # use labels here to better access from the query above - HistoryDatasetAssociation.table.c.history_id.label('history_id'), - Dataset.total_size.label('dataset_size'), - Dataset.id.label('dataset_id') - ]) + select( + [ + # use labels here to better access from the query above + HistoryDatasetAssociation.table.c.history_id.label("history_id"), + Dataset.total_size.label("dataset_size"), + Dataset.id.label("dataset_id"), + ] + ) .where(HistoryDatasetAssociation.table.c.purged != true()) .where(Dataset.table.c.purged != true()) .select_from(hda_to_dataset_join) @@ -2730,14 +2842,12 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable distinct_datasets_alias = aliased(distinct_datasets, name="datasets") # then, bind as property of history using the cls.id size_query = ( - select([ - func.coalesce(func.sum(distinct_datasets_alias.c.dataset_size), 0) - ]) + select([func.coalesce(func.sum(distinct_datasets_alias.c.dataset_size), 0)]) .select_from(distinct_datasets_alias) .where(distinct_datasets_alias.c.history_id == cls.id) ) # label creates a scalar - return size_query.label('disk_size') + return size_query.label("disk_size") @property def disk_nice_size(self): @@ -2747,45 +2857,51 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable @property def active_dataset_and_roles_query(self): db_session = object_session(self) - return (db_session.query(HistoryDatasetAssociation) + return ( + db_session.query(HistoryDatasetAssociation) .filter(HistoryDatasetAssociation.table.c.history_id == self.id) .filter(not_(HistoryDatasetAssociation.deleted)) .order_by(HistoryDatasetAssociation.table.c.hid.asc()) - .options(joinedload("dataset"), - joinedload("dataset.actions"), - joinedload("dataset.actions.role"), - joinedload("tags"))) + .options( + joinedload("dataset"), + joinedload("dataset.actions"), + joinedload("dataset.actions.role"), + joinedload("tags"), + ) + ) @property def active_datasets_and_roles(self): - if not hasattr(self, '_active_datasets_and_roles'): + if not hasattr(self, "_active_datasets_and_roles"): self._active_datasets_and_roles = self.active_dataset_and_roles_query.all() return self._active_datasets_and_roles @property def active_visible_datasets_and_roles(self): - if not hasattr(self, '_active_visible_datasets_and_roles'): - self._active_visible_datasets_and_roles = self.active_dataset_and_roles_query.filter(HistoryDatasetAssociation.visible).all() + if not hasattr(self, "_active_visible_datasets_and_roles"): + self._active_visible_datasets_and_roles = self.active_dataset_and_roles_query.filter( + HistoryDatasetAssociation.visible + ).all() return self._active_visible_datasets_and_roles @property def active_visible_dataset_collections(self): - if not hasattr(self, '_active_visible_dataset_collections'): + if not hasattr(self, "_active_visible_dataset_collections"): db_session = object_session(self) - query = (db_session.query(HistoryDatasetCollectionAssociation) + query = ( + db_session.query(HistoryDatasetCollectionAssociation) .filter(HistoryDatasetCollectionAssociation.table.c.history_id == self.id) .filter(not_(HistoryDatasetCollectionAssociation.deleted)) .filter(HistoryDatasetCollectionAssociation.visible) .order_by(HistoryDatasetCollectionAssociation.table.c.hid.asc()) - .options(joinedload("collection"), - joinedload("tags"))) + .options(joinedload("collection"), joinedload("tags")) + ) self._active_visible_dataset_collections = query.all() return self._active_visible_dataset_collections @property def active_contents(self): - """ Return all active contents ordered by hid. - """ + """Return all active contents ordered by hid.""" return self.contents_iter(types=["dataset", "dataset_collection"], deleted=False, visible=True) def contents_iter(self, **kwds): @@ -2793,13 +2909,13 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable Fetch filtered list of contents of history. """ default_contents_types = [ - 'dataset', + "dataset", ] - types = kwds.get('types', default_contents_types) + types = kwds.get("types", default_contents_types) iters = [] - if 'dataset' in types: + if "dataset" in types: iters.append(self.__dataset_contents_iter(**kwds)) - if 'dataset_collection' in types: + if "dataset_collection" in types: iters.append(self.__collection_contents_iter(**kwds)) return galaxy.util.merge_sorted_iterables(operator.attrgetter("hid"), *iters) @@ -2811,15 +2927,15 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable assert db_session is not None query = db_session.query(content_class).filter(content_class.table.c.history_id == self.id) query = query.order_by(content_class.table.c.hid.asc()) - deleted = galaxy.util.string_as_bool_or_none(kwds.get('deleted', None)) + deleted = galaxy.util.string_as_bool_or_none(kwds.get("deleted", None)) if deleted is not None: query = query.filter(content_class.deleted == deleted) - visible = galaxy.util.string_as_bool_or_none(kwds.get('visible', None)) + visible = galaxy.util.string_as_bool_or_none(kwds.get("visible", None)) if visible is not None: query = query.filter(content_class.visible == visible) - if 'ids' in kwds: - ids = kwds['ids'] - max_in_filter_length = kwds.get('max_in_filter_length', MAX_IN_FILTER_LENGTH) + if "ids" in kwds: + ids = kwds["ids"] + max_in_filter_length = kwds.get("max_in_filter_length", MAX_IN_FILTER_LENGTH) if len(ids) < max_in_filter_length: query = query.filter(content_class.id.in_(ids)) else: @@ -2835,26 +2951,26 @@ class UserShareAssociation(RepresentById): class HistoryUserShareAssociation(Base, UserShareAssociation): - __tablename__ = 'history_user_share_association' + __tablename__ = "history_user_share_association" id = Column(Integer, primary_key=True) - history_id = Column(Integer, ForeignKey('history.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) - user = relationship('User') - history = relationship('History', back_populates='users_shared_with') + history_id = Column(Integer, ForeignKey("history.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) + user = relationship("User") + history = relationship("History", back_populates="users_shared_with") class UserRoleAssociation(Base, RepresentById): - __tablename__ = 'user_role_association' + __tablename__ = "user_role_association" id = Column(Integer, primary_key=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) - role_id = Column(Integer, ForeignKey('role.id'), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) + role_id = Column(Integer, ForeignKey("role.id"), index=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - user = relationship('User', back_populates="roles") - role = relationship('Role', back_populates="users") + user = relationship("User", back_populates="roles") + role = relationship("Role", back_populates="users") def __init__(self, user, role): self.user = user @@ -2862,15 +2978,15 @@ class UserRoleAssociation(Base, RepresentById): class GroupRoleAssociation(Base, RepresentById): - __tablename__ = 'group_role_association' + __tablename__ = "group_role_association" id = Column(Integer, primary_key=True) - group_id = Column(Integer, ForeignKey('galaxy_group.id'), index=True) - role_id = Column(Integer, ForeignKey('role.id'), index=True) + group_id = Column(Integer, ForeignKey("galaxy_group.id"), index=True) + role_id = Column(Integer, ForeignKey("role.id"), index=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - group = relationship('Group', back_populates='roles') - role = relationship('Role', back_populates='groups') + group = relationship("Group", back_populates="roles") + role = relationship("Role", back_populates="groups") def __init__(self, group, role): self.group = group @@ -2878,7 +2994,7 @@ class GroupRoleAssociation(Base, RepresentById): class Role(Base, Dictifiable, RepresentById): - __tablename__ = 'role' + __tablename__ = "role" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) @@ -2887,20 +3003,20 @@ class Role(Base, Dictifiable, RepresentById): description = Column(TEXT) type = Column(String(40), index=True) deleted = Column(Boolean, index=True, default=False) - dataset_actions = relationship('DatasetPermissions', back_populates='role') - groups = relationship('GroupRoleAssociation', back_populates='role') - users = relationship('UserRoleAssociation', back_populates='role') + dataset_actions = relationship("DatasetPermissions", back_populates="role") + groups = relationship("GroupRoleAssociation", back_populates="role") + users = relationship("UserRoleAssociation", back_populates="role") - dict_collection_visible_keys = ['id', 'name'] - dict_element_visible_keys = ['id', 'name', 'description', 'type'] + dict_collection_visible_keys = ["id", "name"] + dict_element_visible_keys = ["id", "name", "description", "type"] private_id = None class types(str, Enum): - PRIVATE = 'private' - SYSTEM = 'system' - USER = 'user' - ADMIN = 'admin' - SHARING = 'sharing' + PRIVATE = "private" + SYSTEM = "system" + USER = "user" + ADMIN = "admin" + SHARING = "sharing" def __init__(self, name=None, description=None, type=types.SYSTEM, deleted=False): self.name = name @@ -2910,17 +3026,17 @@ class Role(Base, Dictifiable, RepresentById): class UserQuotaAssociation(Base, Dictifiable, RepresentById): - __tablename__ = 'user_quota_association' + __tablename__ = "user_quota_association" id = Column(Integer, primary_key=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) - quota_id = Column(Integer, ForeignKey('quota.id'), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) + quota_id = Column(Integer, ForeignKey("quota.id"), index=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - user = relationship('User', back_populates='quotas') - quota = relationship('Quota', back_populates='users') + user = relationship("User", back_populates="quotas") + quota = relationship("Quota", back_populates="users") - dict_element_visible_keys = ['user'] + dict_element_visible_keys = ["user"] def __init__(self, user, quota): self.user = user @@ -2928,17 +3044,17 @@ class UserQuotaAssociation(Base, Dictifiable, RepresentById): class GroupQuotaAssociation(Base, Dictifiable, RepresentById): - __tablename__ = 'group_quota_association' + __tablename__ = "group_quota_association" id = Column(Integer, primary_key=True) group_id = Column(Integer, ForeignKey("galaxy_group.id"), index=True) - quota_id = Column(Integer, ForeignKey('quota.id'), index=True) + quota_id = Column(Integer, ForeignKey("quota.id"), index=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - group = relationship('Group', back_populates='quotas') - quota = relationship('Quota', back_populates='groups') + group = relationship("Group", back_populates="quotas") + quota = relationship("Quota", back_populates="groups") - dict_element_visible_keys = ['group'] + dict_element_visible_keys = ["group"] def __init__(self, group, quota): self.group = group @@ -2946,7 +3062,7 @@ class GroupQuotaAssociation(Base, Dictifiable, RepresentById): class Quota(Base, Dictifiable, RepresentById): - __tablename__ = 'quota' + __tablename__ = "quota" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) @@ -2956,15 +3072,25 @@ class Quota(Base, Dictifiable, RepresentById): bytes = Column(BigInteger) operation = Column(String(8)) deleted = Column(Boolean, index=True, default=False) - default = relationship('DefaultQuotaAssociation', back_populates='quota') - groups = relationship('GroupQuotaAssociation', back_populates='quota') - users = relationship('UserQuotaAssociation', back_populates='quota') + default = relationship("DefaultQuotaAssociation", back_populates="quota") + groups = relationship("GroupQuotaAssociation", back_populates="quota") + users = relationship("UserQuotaAssociation", back_populates="quota") - dict_collection_visible_keys = ['id', 'name'] - dict_element_visible_keys = ['id', 'name', 'description', 'bytes', 'operation', 'display_amount', 'default', 'users', 'groups'] - valid_operations = ('+', '-', '=') + dict_collection_visible_keys = ["id", "name"] + dict_element_visible_keys = [ + "id", + "name", + "description", + "bytes", + "operation", + "display_amount", + "default", + "users", + "groups", + ] + valid_operations = ("+", "-", "=") - def __init__(self, name=None, description=None, amount=0, operation='='): + def __init__(self, name=None, description=None, amount=0, operation="="): self.name = name self.description = description if amount is None: @@ -2983,6 +3109,7 @@ class Quota(Base, Dictifiable, RepresentById): self.bytes = -1 else: self.bytes = amount + amount = property(get_amount, set_amount) @property @@ -2994,38 +3121,38 @@ class Quota(Base, Dictifiable, RepresentById): class DefaultQuotaAssociation(Base, Dictifiable, RepresentById): - __tablename__ = 'default_quota_association' + __tablename__ = "default_quota_association" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) type = Column(String(32), index=True, unique=True) - quota_id = Column(Integer, ForeignKey('quota.id'), index=True) - quota = relationship('Quota', back_populates='default') + quota_id = Column(Integer, ForeignKey("quota.id"), index=True) + quota = relationship("Quota", back_populates="default") - dict_element_visible_keys = ['type'] + dict_element_visible_keys = ["type"] class types(str, Enum): - UNREGISTERED = 'unregistered' - REGISTERED = 'registered' + UNREGISTERED = "unregistered" + REGISTERED = "registered" def __init__(self, type, quota): - assert type in self.types.__members__.values(), 'Invalid type' + assert type in self.types.__members__.values(), "Invalid type" self.type = type self.quota = quota class DatasetPermissions(Base, RepresentById): - __tablename__ = 'dataset_permissions' + __tablename__ = "dataset_permissions" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) action = Column(TEXT) - dataset_id = Column(Integer, ForeignKey('dataset.id'), index=True) + dataset_id = Column(Integer, ForeignKey("dataset.id"), index=True) role_id = Column(Integer, ForeignKey("role.id"), index=True) - dataset = relationship('Dataset', back_populates='actions') - role = relationship('Role', back_populates='dataset_actions') + dataset = relationship("Dataset", back_populates="actions") + role = relationship("Role", back_populates="dataset_actions") def __init__(self, action, dataset, role=None, role_id=None): self.action = action @@ -3037,16 +3164,16 @@ class DatasetPermissions(Base, RepresentById): class LibraryPermissions(Base, RepresentById): - __tablename__ = 'library_permissions' + __tablename__ = "library_permissions" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) action = Column(TEXT) - library_id = Column(Integer, ForeignKey('library.id'), nullable=True, index=True) - role_id = Column(Integer, ForeignKey('role.id'), index=True) - library = relationship('Library', back_populates='actions') - role = relationship('Role') + library_id = Column(Integer, ForeignKey("library.id"), nullable=True, index=True) + role_id = Column(Integer, ForeignKey("role.id"), index=True) + library = relationship("Library", back_populates="actions") + role = relationship("Role") def __init__(self, action, library_item, role): self.action = action @@ -3058,16 +3185,16 @@ class LibraryPermissions(Base, RepresentById): class LibraryFolderPermissions(Base, RepresentById): - __tablename__ = 'library_folder_permissions' + __tablename__ = "library_folder_permissions" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) action = Column(TEXT) - library_folder_id = Column(Integer, ForeignKey('library_folder.id'), nullable=True, index=True) - role_id = Column(Integer, ForeignKey('role.id'), index=True) - folder = relationship('LibraryFolder', back_populates='actions') - role = relationship('Role') + library_folder_id = Column(Integer, ForeignKey("library_folder.id"), nullable=True, index=True) + role_id = Column(Integer, ForeignKey("role.id"), index=True) + folder = relationship("LibraryFolder", back_populates="actions") + role = relationship("Role") def __init__(self, action, library_item, role): self.action = action @@ -3079,16 +3206,16 @@ class LibraryFolderPermissions(Base, RepresentById): class LibraryDatasetPermissions(Base, RepresentById): - __tablename__ = 'library_dataset_permissions' + __tablename__ = "library_dataset_permissions" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) action = Column(TEXT) - library_dataset_id = Column(Integer, ForeignKey('library_dataset.id'), nullable=True, index=True) - role_id = Column(Integer, ForeignKey('role.id'), index=True) - library_dataset = relationship('LibraryDataset', back_populates='actions') - role = relationship('Role') + library_dataset_id = Column(Integer, ForeignKey("library_dataset.id"), nullable=True, index=True) + role_id = Column(Integer, ForeignKey("role.id"), index=True) + library_dataset = relationship("LibraryDataset", back_populates="actions") + role = relationship("Role") def __init__(self, action, library_item, role): self.action = action @@ -3100,18 +3227,18 @@ class LibraryDatasetPermissions(Base, RepresentById): class LibraryDatasetDatasetAssociationPermissions(Base, RepresentById): - __tablename__ = 'library_dataset_dataset_association_permissions' + __tablename__ = "library_dataset_dataset_association_permissions" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) action = Column(TEXT) library_dataset_dataset_association_id = Column( - Integer, ForeignKey('library_dataset_dataset_association.id'), nullable=True, index=True) - role_id = Column(Integer, ForeignKey('role.id'), index=True) - library_dataset_dataset_association = relationship('LibraryDatasetDatasetAssociation', - back_populates='actions') - role = relationship('Role') + Integer, ForeignKey("library_dataset_dataset_association.id"), nullable=True, index=True + ) + role_id = Column(Integer, ForeignKey("role.id"), index=True) + library_dataset_dataset_association = relationship("LibraryDatasetDatasetAssociation", back_populates="actions") + role = relationship("Role") def __init__(self, action, library_item, role): self.action = action @@ -3123,14 +3250,14 @@ class LibraryDatasetDatasetAssociationPermissions(Base, RepresentById): class DefaultUserPermissions(Base, RepresentById): - __tablename__ = 'default_user_permissions' + __tablename__ = "default_user_permissions" id = Column(Integer, primary_key=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) action = Column(TEXT) - role_id = Column(Integer, ForeignKey('role.id'), index=True) - user = relationship('User', back_populates='default_permissions') - role = relationship('Role') + role_id = Column(Integer, ForeignKey("role.id"), index=True) + user = relationship("User", back_populates="default_permissions") + role = relationship("Role") def __init__(self, user, action, role): self.user = user @@ -3139,14 +3266,14 @@ class DefaultUserPermissions(Base, RepresentById): class DefaultHistoryPermissions(Base, RepresentById): - __tablename__ = 'default_history_permissions' + __tablename__ = "default_history_permissions" id = Column(Integer, primary_key=True) - history_id = Column(Integer, ForeignKey('history.id'), index=True) + history_id = Column(Integer, ForeignKey("history.id"), index=True) action = Column(TEXT) - role_id = Column(Integer, ForeignKey('role.id'), index=True) - history = relationship('History', back_populates='default_permissions') - role = relationship('Role') + role_id = Column(Integer, ForeignKey("role.id"), index=True) + history = relationship("History", back_populates="default_permissions") + role = relationship("Role") def __init__(self, history, action, role): self.history = history @@ -3155,7 +3282,6 @@ class DefaultHistoryPermissions(Base, RepresentById): class StorableObject: - def flush(self): sa_session = object_session(self) if sa_session: @@ -3163,36 +3289,28 @@ class StorableObject: class Dataset(StorableObject, Serializable, _HasTable): - class states(str, Enum): - NEW = 'new' - UPLOAD = 'upload' - QUEUED = 'queued' - RUNNING = 'running' - OK = 'ok' - EMPTY = 'empty' - ERROR = 'error' - DISCARDED = 'discarded' - PAUSED = 'paused' - SETTING_METADATA = 'setting_metadata' - FAILED_METADATA = 'failed_metadata' + NEW = "new" + UPLOAD = "upload" + QUEUED = "queued" + RUNNING = "running" + OK = "ok" + EMPTY = "empty" + ERROR = "error" + DISCARDED = "discarded" + PAUSED = "paused" + SETTING_METADATA = "setting_metadata" + FAILED_METADATA = "failed_metadata" @classmethod def values(self): return self.__members__.values() + # failed_metadata is only valid as DatasetInstance state currently - non_ready_states = ( - states.NEW, - states.UPLOAD, - states.QUEUED, - states.RUNNING, - states.SETTING_METADATA - ) + non_ready_states = (states.NEW, states.UPLOAD, states.QUEUED, states.RUNNING, states.SETTING_METADATA) ready_states = tuple(set(states.__members__.values()) - set(non_ready_states)) - valid_input_states = tuple( - set(states.__members__.values()) - {states.ERROR, states.DISCARDED} - ) + valid_input_states = tuple(set(states.__members__.values()) - {states.ERROR, states.DISCARDED}) terminal_states = ( states.OK, states.EMPTY, @@ -3211,12 +3329,21 @@ class Dataset(StorableObject, Serializable, _HasTable): ERROR = "error" OK = "ok" - permitted_actions = get_permitted_actions(filter='DATASET') + permitted_actions = get_permitted_actions(filter="DATASET") file_path = "/tmp/" object_store = None # This get initialized in mapping.py (method init) by app.py engine = None - def __init__(self, id=None, state=None, external_filename=None, extra_files_path=None, file_size=None, purgable=True, uuid=None): + def __init__( + self, + id=None, + state=None, + external_filename=None, + extra_files_path=None, + file_size=None, + purgable=True, + uuid=None, + ): self.id = id self.uuid = get_uuid(uuid) self.state = state @@ -3243,7 +3370,7 @@ class Dataset(StorableObject, Serializable, _HasTable): if self.object_store.exists(self): return self.object_store.get_filename(self) else: - return '' + return "" else: filename = self.external_filename # Make filename absolute @@ -3254,6 +3381,7 @@ class Dataset(StorableObject, Serializable, _HasTable): self.external_filename = None else: self.external_filename = filename + file_name = property(get_file_name, set_file_name) def get_extra_files_path(self): @@ -3263,7 +3391,7 @@ class Dataset(StorableObject, Serializable, _HasTable): if not getattr(self, "external_extra_files_path", None): if self.object_store.exists(self, dir_only=True, extra_dir=self._extra_files_rel_path): return self.object_store.get_filename(self, dir_only=True, extra_dir=self._extra_files_rel_path) - return '' + return "" else: return os.path.abspath(self.external_extra_files_path) @@ -3276,6 +3404,7 @@ class Dataset(StorableObject, Serializable, _HasTable): self.external_extra_files_path = None else: self.external_extra_files_path = extra_files_path + extra_files_path = property(get_extra_files_path, set_extra_files_path) def extra_files_path_exists(self): @@ -3352,7 +3481,11 @@ class Dataset(StorableObject, Serializable, _HasTable): if rel_path is not None: if self.object_store.exists(self, extra_dir=rel_path, dir_only=True): for root, _, files in os.walk(self.extra_files_path): - self.total_size += sum(os.path.getsize(os.path.join(root, file)) for file in files if os.path.exists(os.path.join(root, file))) + self.total_size += sum( + os.path.getsize(os.path.join(root, file)) + for file in files + if os.path.exists(os.path.join(root, file)) + ) return self.total_size def has_data(self): @@ -3369,9 +3502,11 @@ class Dataset(StorableObject, Serializable, _HasTable): @property def user_can_purge(self): - return self.purged is False \ - and not bool(self.library_associations) \ + return ( + self.purged is False + and not bool(self.library_associations) and len(self.history_associations) == len(self.purged_history_associations) + ) def full_delete(self): """Remove the file and extra files, marks deleted and purged""" @@ -3426,7 +3561,7 @@ class Dataset(StorableObject, Serializable, _HasTable): object_store_id=self.object_store_id, total_size=to_int(self.total_size), created_from_basename=self.created_from_basename, - uuid=str(self.uuid or '') or None, + uuid=str(self.uuid or "") or None, hashes=list(map(lambda h: h.serialize(id_encoder, serialization_options), self.hashes)), sources=list(map(lambda s: s.serialize(id_encoder, serialization_options), self.sources)), ) @@ -3435,17 +3570,22 @@ class Dataset(StorableObject, Serializable, _HasTable): class DatasetSource(Base, Dictifiable, Serializable): - __tablename__ = 'dataset_source' + __tablename__ = "dataset_source" id = Column(Integer, primary_key=True) - dataset_id = Column(Integer, ForeignKey('dataset.id'), index=True) + dataset_id = Column(Integer, ForeignKey("dataset.id"), index=True) source_uri = Column(TEXT) extra_files_path = Column(TEXT) transform = Column(MutableJSONType) - dataset = relationship('Dataset', back_populates='sources') - hashes = relationship('DatasetSourceHash', back_populates='source') - dict_collection_visible_keys = ['id', 'source_uri', 'extra_files_path', "transform"] - dict_element_visible_keys = ['id', 'source_uri', 'extra_files_path', 'transform'] # TODO: implement to_dict and add hashes... + dataset = relationship("Dataset", back_populates="sources") + hashes = relationship("DatasetSourceHash", back_populates="source") + dict_collection_visible_keys = ["id", "source_uri", "extra_files_path", "transform"] + dict_element_visible_keys = [ + "id", + "source_uri", + "extra_files_path", + "transform", + ] # TODO: implement to_dict and add hashes... def _serialize(self, id_encoder, serialization_options): rval = dict_for( @@ -3460,13 +3600,13 @@ class DatasetSource(Base, Dictifiable, Serializable): class DatasetSourceHash(Base, Serializable): - __tablename__ = 'dataset_source_hash' + __tablename__ = "dataset_source_hash" id = Column(Integer, primary_key=True) - dataset_source_id = Column(Integer, ForeignKey('dataset_source.id'), index=True) + dataset_source_id = Column(Integer, ForeignKey("dataset_source.id"), index=True) hash_function = Column(TEXT) hash_value = Column(TEXT) - source = relationship('DatasetSource', back_populates='hashes') + source = relationship("DatasetSource", back_populates="hashes") def _serialize(self, id_encoder, serialization_options): rval = dict_for( @@ -3479,16 +3619,16 @@ class DatasetSourceHash(Base, Serializable): class DatasetHash(Base, Dictifiable, Serializable): - __tablename__ = 'dataset_hash' + __tablename__ = "dataset_hash" id = Column(Integer, primary_key=True) - dataset_id = Column(Integer, ForeignKey('dataset.id'), index=True) + dataset_id = Column(Integer, ForeignKey("dataset.id"), index=True) hash_function = Column(TEXT) hash_value = Column(TEXT) extra_files_path = Column(TEXT) - dataset = relationship('Dataset', back_populates='hashes') - dict_collection_visible_keys = ['id', 'hash_function', 'hash_value', 'extra_files_path'] - dict_element_visible_keys = ['id', 'hash_function', 'hash_value', 'extra_files_path'] + dataset = relationship("Dataset", back_populates="hashes") + dict_collection_visible_keys = ["id", "hash_function", "hash_value", "extra_files_path"] + dict_element_visible_keys = ["id", "hash_function", "hash_value", "extra_files_path"] def _serialize(self, id_encoder, serialization_options): rval = dict_for( @@ -3506,31 +3646,53 @@ def datatype_for_extension(extension, datatypes_registry=None) -> "Data": extension = extension.lower() if datatypes_registry is None: datatypes_registry = _get_datatypes_registry() - if not extension or extension == 'auto' or extension == '_sniff_': - extension = 'data' + if not extension or extension == "auto" or extension == "_sniff_": + extension = "data" ret = datatypes_registry.get_datatype_by_extension(extension) if ret is None: log.warning(f"Datatype class not found for extension '{extension}'") - return datatypes_registry.get_datatype_by_extension('data') + return datatypes_registry.get_datatype_by_extension("data") return ret class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): """A base class for all 'dataset instances', HDAs, LDAs, etc""" + states = Dataset.states conversion_messages = Dataset.conversion_messages permitted_actions = Dataset.permitted_actions class validated_states(str, Enum): - UNKNOWN = 'unknown' - INVALID = 'invalid' - OK = 'ok' + UNKNOWN = "unknown" + INVALID = "invalid" + OK = "ok" - def __init__(self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, tool_version=None, - extension=None, dbkey=None, metadata=None, history=None, dataset=None, deleted=False, - designation=None, parent_id=None, validated_state='unknown', validated_state_message=None, - visible=True, create_dataset=False, sa_session=None, extended_metadata=None, flush=True, - creating_job_id=None): + def __init__( + self, + id=None, + hid=None, + name=None, + info=None, + blurb=None, + peek=None, + tool_version=None, + extension=None, + dbkey=None, + metadata=None, + history=None, + dataset=None, + deleted=False, + designation=None, + parent_id=None, + validated_state="unknown", + validated_state_message=None, + visible=True, + create_dataset=False, + sa_session=None, + extended_metadata=None, + flush=True, + creating_job_id=None, + ): self.name = name or "Unnamed dataset" self.id = id self.info = info @@ -3543,8 +3705,10 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): self._metadata = None self.metadata = metadata or dict() self.extended_metadata = extended_metadata - if dbkey: # dbkey is stored in metadata, only set if non-zero, or else we could clobber one supplied by input 'metadata' - self._metadata['dbkey'] = dbkey + if ( + dbkey + ): # dbkey is stored in metadata, only set if non-zero, or else we could clobber one supplied by input 'metadata' + self._metadata["dbkey"] = dbkey self.deleted = deleted self.visible = visible self.validated_state = validated_state @@ -3592,6 +3756,7 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): if sa_session: object_session(self).add(self.dataset) object_session(self).flush() # flush here, because hda.flush() won't flush the Dataset object + state = property(get_dataset_state, set_dataset_state) def get_file_name(self) -> str: @@ -3601,6 +3766,7 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): def set_file_name(self, filename: str): return self.dataset.set_file_name(filename) + file_name = property(get_file_name, set_file_name) def link_to(self, path): @@ -3623,7 +3789,7 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): def get_metadata(self): # using weakref to store parent (to prevent circ ref), # does a Session.clear() cause parent to be invalidated, while still copying over this non-database attribute? - if not hasattr(self, '_metadata_collection') or self._metadata_collection.parent != self: + if not hasattr(self, "_metadata_collection") or self._metadata_collection.parent != self: self._metadata_collection = galaxy.model.metadata.MetadataCollection(self) return self._metadata_collection @@ -3634,6 +3800,7 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): def set_metadata(self, bunch): # Needs to accept a MetadataCollection, a bunch, or a dict self._metadata = self.metadata.make_dict_copy(bunch) + metadata = property(get_metadata, set_metadata) @property @@ -3674,15 +3841,17 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): if "dbkey" in self.datatype.metadata_spec: if not isinstance(value, list): self.metadata.dbkey = [value] + dbkey = property(get_dbkey, set_dbkey) def ok_to_edit_metadata(self): # prevent modifying metadata when dataset is queued or running as input/output # This code could be more efficient, i.e. by using mappers, but to prevent slowing down loading a History panel, we'll leave the code here for now sa_session = object_session(self) - for job_to_dataset_association in sa_session.query( - JobToInputDatasetAssociation).filter_by(dataset_id=self.id).all() \ - + sa_session.query(JobToOutputDatasetAssociation).filter_by(dataset_id=self.id).all(): + for job_to_dataset_association in ( + sa_session.query(JobToInputDatasetAssociation).filter_by(dataset_id=self.id).all() + + sa_session.query(JobToOutputDatasetAssociation).filter_by(dataset_id=self.id).all() + ): if job_to_dataset_association.job.state not in Job.terminal_states: return False return True @@ -3739,7 +3908,7 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): return _get_datatypes_registry().get_mimetype_by_extension(self.extension.lower()) except AttributeError: # extension is None - return 'data' + return "data" def set_peek(self, **kwd): return self.datatype.set_peek(self, **kwd) @@ -3824,10 +3993,25 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): raise NoConverterException(f"A dependency ({dependency}) is missing a converter.") except KeyError: pass # No deps - new_dataset = next(iter(self.datatype.convert_dataset(trans, self, target_ext, return_output=True, visible=False, deps=deps, target_context=target_context, history=history).values())) + new_dataset = next( + iter( + self.datatype.convert_dataset( + trans, + self, + target_ext, + return_output=True, + visible=False, + deps=deps, + target_context=target_context, + history=history, + ).values() + ) + ) new_dataset.name = self.name self.copy_attributes(new_dataset) - assoc = ImplicitlyConvertedDatasetAssociation(parent=self, file_type=target_ext, dataset=new_dataset, metadata_safe=False) + assoc = ImplicitlyConvertedDatasetAssociation( + parent=self, file_type=target_ext, dataset=new_dataset, metadata_safe=False + ) session = trans.sa_session session.add(new_dataset) session.add(assoc) @@ -3847,7 +4031,7 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): for name, value in self.metadata.items(): # HACK: MetadataFile objects do not have a type/ext, so need to use metadata name # to determine type. - if dataset_ext == 'bai' and name == 'bam_index' and isinstance(value, MetadataFile): + if dataset_ext == "bai" and name == "bam_index" and isinstance(value, MetadataFile): # HACK: MetadataFile objects cannot be used by tools, so return # a fake HDA that points to metadata file. fake_dataset = Dataset(state=Dataset.states.OK, external_filename=value.file_name) @@ -3898,9 +4082,13 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): """ Return true if the dataset is neither ready nor in error """ - return self.state in (self.states.NEW, self.states.UPLOAD, - self.states.QUEUED, self.states.RUNNING, - self.states.SETTING_METADATA) + return self.state in ( + self.states.NEW, + self.states.UPLOAD, + self.states.QUEUED, + self.states.RUNNING, + self.states.SETTING_METADATA, + ) @property def source_library_dataset(self): @@ -3917,6 +4105,7 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): if source: return source return (None, None) + return get_source(self) @property @@ -3937,6 +4126,7 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): except Exception as e: log.warning(e) return lst + return _source_dataset_chain(self, []) @property @@ -4006,15 +4196,19 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): except NoConverterException: return self.conversion_messages.NO_CONVERTER except ConverterDependencyException as dep_error: - return {'kind': self.conversion_messages.ERROR, 'message': dep_error.value} + return {"kind": self.conversion_messages.ERROR, "message": dep_error.value} # Check dataset state and return any messages. msg = None if converted_dataset and converted_dataset.state == Dataset.states.ERROR: - job_id = trans.sa_session.query(JobToOutputDatasetAssociation) \ - .filter_by(dataset_id=converted_dataset.id).first().job_id + job_id = ( + trans.sa_session.query(JobToOutputDatasetAssociation) + .filter_by(dataset_id=converted_dataset.id) + .first() + .job_id + ) job = trans.sa_session.query(Job).get(job_id) - msg = {'kind': self.conversion_messages.ERROR, 'message': job.stderr} + msg = {"kind": self.conversion_messages.ERROR, "message": job.stderr} elif not converted_dataset or converted_dataset.state != Dataset.states.OK: msg = self.conversion_messages.PENDING @@ -4062,19 +4256,20 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): rval["file_metadata"] = file_metadata -class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnotations, - HasName, Serializable): +class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable): """ Resource class that creates a relation between a dataset and a user history. """ - def __init__(self, - hid=None, - history=None, - copied_from_history_dataset_association=None, - copied_from_library_dataset_dataset_association=None, - sa_session=None, - **kwd): + def __init__( + self, + hid=None, + history=None, + copied_from_history_dataset_association=None, + copied_from_library_dataset_dataset_association=None, + sa_session=None, + **kwd, + ): """ Create a a new HDA and associate it with the given history. """ @@ -4103,18 +4298,17 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot if self.update_time and self.state == self.states.OK and not self.deleted: # We only record changes to HDAs that exist in the database and have a update_time new_values = {} - new_values['name'] = changes.get('name', self.name) - new_values['dbkey'] = changes.get('dbkey', self.dbkey) - new_values['extension'] = changes.get('extension', self.extension) - new_values['extended_metadata_id'] = changes.get('extended_metadata_id', self.extended_metadata_id) + new_values["name"] = changes.get("name", self.name) + new_values["dbkey"] = changes.get("dbkey", self.dbkey) + new_values["extension"] = changes.get("extension", self.extension) + new_values["extended_metadata_id"] = changes.get("extended_metadata_id", self.extended_metadata_id) for k, v in new_values.items(): if isinstance(v, list): new_values[k] = v[0] - new_values['update_time'] = self.update_time - new_values['version'] = self.version or 1 - new_values['metadata'] = self._metadata - past_hda = HistoryDatasetAssociationHistory(history_dataset_association_id=self.id, - **new_values) + new_values["update_time"] = self.update_time + new_values["version"] = self.version or 1 + new_values["metadata"] = self._metadata + past_hda = HistoryDatasetAssociationHistory(history_dataset_association_id=self.id, **new_values) self.version = self.version + 1 if self.version else 1 session.add(past_hda) @@ -4144,20 +4338,22 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot hid = None if copy_hid: hid = self.hid - hda = HistoryDatasetAssociation(hid=hid, - name=new_name or self.name, - info=self.info, - blurb=self.blurb, - peek=self.peek, - tool_version=self.tool_version, - extension=self.extension, - dbkey=self.dbkey, - dataset=self.dataset, - visible=self.visible, - deleted=self.deleted, - parent_id=parent_id, - copied_from_history_dataset_association=self, - flush=False) + hda = HistoryDatasetAssociation( + hid=hid, + name=new_name or self.name, + info=self.info, + blurb=self.blurb, + peek=self.peek, + tool_version=self.tool_version, + extension=self.extension, + dbkey=self.dbkey, + dataset=self.dataset, + visible=self.visible, + deleted=self.deleted, + parent_id=parent_id, + copied_from_history_dataset_association=self, + flush=False, + ) # update init non-keywords as well hda.purged = self.purged @@ -4183,8 +4379,16 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot def copy_attributes(self, new_dataset): new_dataset.hid = self.hid - def to_library_dataset_dataset_association(self, trans, target_folder, replace_dataset=None, - parent_id=None, roles=None, ldda_message='', element_identifier=None): + def to_library_dataset_dataset_association( + self, + trans, + target_folder, + replace_dataset=None, + parent_id=None, + roles=None, + ldda_message="", + element_identifier=None, + ): """ Copy this HDA to a library optionally replacing an existing LDDA. """ @@ -4198,27 +4402,30 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot # to the associated Dataset. library_dataset = LibraryDataset(folder=target_folder, name=self.name, info=self.info) user = trans.user or self.history.user - ldda = LibraryDatasetDatasetAssociation(name=element_identifier or self.name, - info=self.info, - blurb=self.blurb, - peek=self.peek, - tool_version=self.tool_version, - extension=self.extension, - dbkey=self.dbkey, - dataset=self.dataset, - library_dataset=library_dataset, - visible=self.visible, - deleted=self.deleted, - parent_id=parent_id, - copied_from_history_dataset_association=self, - user=user) + ldda = LibraryDatasetDatasetAssociation( + name=element_identifier or self.name, + info=self.info, + blurb=self.blurb, + peek=self.peek, + tool_version=self.tool_version, + extension=self.extension, + dbkey=self.dbkey, + dataset=self.dataset, + library_dataset=library_dataset, + visible=self.visible, + deleted=self.deleted, + parent_id=parent_id, + copied_from_history_dataset_association=self, + user=user, + ) library_dataset.library_dataset_dataset_association = ldda object_session(self).add(library_dataset) # If roles were selected on the upload form, restrict access to the Dataset to those roles roles = roles or [] for role in roles: - dp = trans.model.DatasetPermissions(trans.app.security_agent.permitted_actions.DATASET_ACCESS.action, - ldda.dataset, role) + dp = trans.model.DatasetPermissions( + trans.app.security_agent.permitted_actions.DATASET_ACCESS.action, ldda.dataset, role + ) trans.sa_session.add(dp) # Must set metadata after ldda flushed, as MetadataFiles require ldda.id flushed = False @@ -4242,8 +4449,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot return ldda def clear_associated_files(self, metadata_safe=False, purge=False): - """ - """ + """ """ # metadata_safe = True means to only clear when assoc.metadata_safe == False for assoc in self.implicitly_converted_datasets: if not assoc.deleted and (not metadata_safe or not assoc.metadata_safe): @@ -4258,8 +4464,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot return self.dataset.get_access_roles(security_agent) def purge_usage_from_quota(self, user): - """Remove this HDA's quota_amount from user's quota. - """ + """Remove this HDA's quota_amount from user's quota.""" if user: user.adjust_total_disk_usage(-self.quota_amount(user)) @@ -4288,11 +4493,11 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot def _serialize(self, id_encoder, serialization_options): rval = super()._serialize(id_encoder, serialization_options) - rval['state'] = self.state + rval["state"] = self.state rval["hid"] = self.hid - rval["annotation"] = unicodify(getattr(self, 'annotation', '')) + rval["annotation"] = unicodify(getattr(self, "annotation", "")) rval["tags"] = self.make_tag_string_list() - rval['tool_version'] = self.tool_version + rval["tool_version"] = self.tool_version if self.history: rval["history_encoded_id"] = serialization_options.get_identifier(id_encoder, self.history) @@ -4301,12 +4506,14 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot src_hda = self while src_hda.copied_from_history_dataset_association: src_hda = src_hda.copied_from_history_dataset_association - copied_from_history_dataset_association_chain.append(serialization_options.get_identifier(id_encoder, src_hda)) + copied_from_history_dataset_association_chain.append( + serialization_options.get_identifier(id_encoder, src_hda) + ) rval["copied_from_history_dataset_association_id_chain"] = copied_from_history_dataset_association_chain self._handle_serialize_files(id_encoder, serialization_options, rval) return rval - def to_dict(self, view='collection', expose_dataset_path=False): + def to_dict(self, view="collection", expose_dataset_path=False): """ Return attributes of this HDA that are exposed using the API. """ @@ -4315,39 +4522,41 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot # other model classes. original_rval = super().to_dict(view=view) hda = self - rval = dict(id=hda.id, - hda_ldda='hda', - uuid=(lambda uuid: str(uuid) if uuid else None)(hda.dataset.uuid), - hid=hda.hid, - file_ext=hda.ext, - peek=unicodify(hda.display_peek()) if hda.peek and hda.peek != 'no peek' else None, - model_class=self.__class__.__name__, - name=hda.name, - deleted=hda.deleted, - purged=hda.purged, - visible=hda.visible, - state=hda.state, - history_content_type=hda.history_content_type, - file_size=int(hda.get_size()), - create_time=hda.create_time.isoformat(), - update_time=hda.update_time.isoformat(), - data_type=f"{hda.datatype.__class__.__module__}.{hda.datatype.__class__.__name__}", - genome_build=hda.dbkey, - validated_state=hda.validated_state, - validated_state_message=hda.validated_state_message, - misc_info=hda.info.strip() if isinstance(hda.info, str) else hda.info, - misc_blurb=hda.blurb) + rval = dict( + id=hda.id, + hda_ldda="hda", + uuid=(lambda uuid: str(uuid) if uuid else None)(hda.dataset.uuid), + hid=hda.hid, + file_ext=hda.ext, + peek=unicodify(hda.display_peek()) if hda.peek and hda.peek != "no peek" else None, + model_class=self.__class__.__name__, + name=hda.name, + deleted=hda.deleted, + purged=hda.purged, + visible=hda.visible, + state=hda.state, + history_content_type=hda.history_content_type, + file_size=int(hda.get_size()), + create_time=hda.create_time.isoformat(), + update_time=hda.update_time.isoformat(), + data_type=f"{hda.datatype.__class__.__module__}.{hda.datatype.__class__.__name__}", + genome_build=hda.dbkey, + validated_state=hda.validated_state, + validated_state_message=hda.validated_state_message, + misc_info=hda.info.strip() if isinstance(hda.info, str) else hda.info, + misc_blurb=hda.blurb, + ) rval.update(original_rval) if hda.copied_from_library_dataset_dataset_association is not None: - rval['copied_from_ldda_id'] = hda.copied_from_library_dataset_dataset_association.id + rval["copied_from_ldda_id"] = hda.copied_from_library_dataset_dataset_association.id if hda.history is not None: - rval['history_id'] = hda.history.id + rval["history_id"] = hda.history.id if hda.extended_metadata is not None: - rval['extended_metadata'] = hda.extended_metadata.data + rval["extended_metadata"] = hda.extended_metadata.data for name in hda.metadata.spec.keys(): val = hda.metadata.get(name) @@ -4371,9 +4580,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot if jtida.job not in jobs_to_unpause: jobs_to_unpause.add(jtida.job) for jtoda in jtida.job.output_datasets: - jobs_to_unpause.update( - jtoda.dataset.unpause_dependent_jobs(jobs=jobs_to_unpause) - ) + jobs_to_unpause.update(jtoda.dataset.unpause_dependent_jobs(jobs=jobs_to_unpause)) return jobs_to_unpause @property @@ -4381,24 +4588,22 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot return "dataset" # TODO: down into DatasetInstance - content_type = 'dataset' + content_type = "dataset" @hybrid.hybrid_property def type_id(self): - return '-'.join((self.content_type, str(self.id))) + return "-".join((self.content_type, str(self.id))) @type_id.expression # type: ignore[no-redef] def type_id(cls): - return ((type_coerce(cls.content_type, Unicode) + '-' - + type_coerce(cls.id, Unicode)).label('type_id')) + return (type_coerce(cls.content_type, Unicode) + "-" + type_coerce(cls.id, Unicode)).label("type_id") class HistoryDatasetAssociationHistory(Base, Serializable): - __tablename__ = 'history_dataset_association_history' + __tablename__ = "history_dataset_association_history" id = Column(Integer, primary_key=True) - history_dataset_association_id = Column(Integer, - ForeignKey("history_dataset_association.id"), index=True) + history_dataset_association_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) update_time = Column(DateTime, default=now) version = Column(Integer) name = Column(TrimmedString(255)) @@ -4406,16 +4611,17 @@ class HistoryDatasetAssociationHistory(Base, Serializable): _metadata = Column("metadata", MetadataType) extended_metadata_id = Column(Integer, ForeignKey("extended_metadata.id"), index=True) - def __init__(self, - history_dataset_association_id, - name, - dbkey, - update_time, - version, - extension, - extended_metadata_id, - metadata, - ): + def __init__( + self, + history_dataset_association_id, + name, + dbkey, + update_time, + version, + extension, + extended_metadata_id, + metadata, + ): self.history_dataset_association_id = history_dataset_association_id self.name = name self.dbkey = dbkey @@ -4428,17 +4634,16 @@ class HistoryDatasetAssociationHistory(Base, Serializable): # hda read access permission given by a user to a specific site (gen. for external display applications) class HistoryDatasetAssociationDisplayAtAuthorization(Base, RepresentById): - __tablename__ = 'history_dataset_association_display_at_authorization' + __tablename__ = "history_dataset_association_display_at_authorization" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, index=True, default=now, onupdate=now) - history_dataset_association_id = Column(Integer, - ForeignKey('history_dataset_association.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + history_dataset_association_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) site = Column(TrimmedString(255)) - history_dataset_association = relationship('HistoryDatasetAssociation') - user = relationship('User') + history_dataset_association = relationship("HistoryDatasetAssociation") + user = relationship("User") def __init__(self, hda=None, user=None, site=None): self.history_dataset_association = hda @@ -4447,21 +4652,26 @@ class HistoryDatasetAssociationDisplayAtAuthorization(Base, RepresentById): class HistoryDatasetAssociationSubset(Base, RepresentById): - __tablename__ = 'history_dataset_association_subset' + __tablename__ = "history_dataset_association_subset" id = Column(Integer, primary_key=True) - history_dataset_association_id = Column(Integer, - ForeignKey('history_dataset_association.id'), index=True) - history_dataset_association_subset_id = Column(Integer, - ForeignKey('history_dataset_association.id'), index=True) + history_dataset_association_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) + history_dataset_association_subset_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) location = Column(Unicode(255), index=True) - hda = relationship('HistoryDatasetAssociation', - primaryjoin=(lambda: HistoryDatasetAssociationSubset.history_dataset_association_id - == HistoryDatasetAssociation.id)) - subset = relationship('HistoryDatasetAssociation', - primaryjoin=(lambda: HistoryDatasetAssociationSubset.history_dataset_association_subset_id - == HistoryDatasetAssociation.id)) + hda = relationship( + "HistoryDatasetAssociation", + primaryjoin=( + lambda: HistoryDatasetAssociationSubset.history_dataset_association_id == HistoryDatasetAssociation.id + ), + ) + subset = relationship( + "HistoryDatasetAssociation", + primaryjoin=( + lambda: HistoryDatasetAssociationSubset.history_dataset_association_subset_id + == HistoryDatasetAssociation.id + ), + ) def __init__(self, hda, subset, location): self.hda = hda @@ -4470,7 +4680,7 @@ class HistoryDatasetAssociationSubset(Base, RepresentById): class Library(Base, Dictifiable, HasName, Serializable): - __tablename__ = 'library' + __tablename__ = "library" id = Column(Integer, primary_key=True) root_folder_id = Column(Integer, ForeignKey("library_folder.id"), index=True) @@ -4481,12 +4691,12 @@ class Library(Base, Dictifiable, HasName, Serializable): purged = Column(Boolean, index=True, default=False) description = Column(TEXT) synopsis = Column(TEXT) - root_folder = relationship('LibraryFolder', back_populates='library_root') - actions = relationship('LibraryPermissions', back_populates='library') + root_folder = relationship("LibraryFolder", back_populates="library_root") + actions = relationship("LibraryPermissions", back_populates="library") - permitted_actions = get_permitted_actions(filter='LIBRARY') - dict_collection_visible_keys = ['id', 'name'] - dict_element_visible_keys = ['id', 'deleted', 'name', 'description', 'synopsis', 'root_folder_id', 'create_time'] + permitted_actions = get_permitted_actions(filter="LIBRARY") + dict_collection_visible_keys = ["id", "name"] + dict_element_visible_keys = ["id", "deleted", "name", "description", "synopsis", "root_folder_id", "create_time"] def __init__(self, name=None, description=None, synopsis=None, root_folder=None): self.name = name or "Unnamed library" @@ -4507,13 +4717,13 @@ class Library(Base, Dictifiable, HasName, Serializable): serialization_options.attach_identifier(id_encoder, self, rval) return rval - def to_dict(self, view='collection', value_mapper=None): + def to_dict(self, view="collection", value_mapper=None): """ We prepend an F to folders. """ rval = super().to_dict(view=view, value_mapper=value_mapper) - if 'root_folder_id' in rval: - rval['root_folder_id'] = f"F{str(rval['root_folder_id'])}" + if "root_folder_id" in rval: + rval["root_folder_id"] = f"F{str(rval['root_folder_id'])}" return rval def get_active_folders(self, folder, folders=None): @@ -4533,11 +4743,12 @@ class Library(Base, Dictifiable, HasName, Serializable): intermed = [(getattr(v, attr), i, v) for i, v in enumerate(seq)] intermed.sort() return [_[-1] for _ in intermed] + if folders is None: active_folders = [folder] for active_folder in folder.active_folders: active_folders.extend(self.get_active_folders(active_folder, folders)) - return sort_by_attr(active_folders, 'id') + return sort_by_attr(active_folders, "id") def get_access_roles(self, security_agent): roles = [] @@ -4548,13 +4759,11 @@ class Library(Base, Dictifiable, HasName, Serializable): class LibraryFolder(Base, Dictifiable, HasName, Serializable): - __tablename__ = 'library_folder' - __table_args__ = ( - Index('ix_library_folder_name', 'name', mysql_length=200), - ) + __tablename__ = "library_folder" + __table_args__ = (Index("ix_library_folder_name", "name", mysql_length=200),) id = Column(Integer, primary_key=True) - parent_id = Column(Integer, ForeignKey('library_folder.id'), nullable=True, index=True) + parent_id = Column(Integer, ForeignKey("library_folder.id"), nullable=True, index=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) name = Column(TEXT) @@ -4565,37 +4774,57 @@ class LibraryFolder(Base, Dictifiable, HasName, Serializable): purged = Column(Boolean, index=True, default=False) genome_build = Column(TrimmedString(40)) - folders = relationship('LibraryFolder', + folders = relationship( + "LibraryFolder", primaryjoin=(lambda: LibraryFolder.id == LibraryFolder.parent_id), order_by=asc(name), - back_populates='parent') - parent = relationship('LibraryFolder', back_populates='folders', remote_side=[id]) + back_populates="parent", + ) + parent = relationship("LibraryFolder", back_populates="folders", remote_side=[id]) - active_folders = relationship('LibraryFolder', - primaryjoin=( - 'and_(LibraryFolder.parent_id == LibraryFolder.id, not_(LibraryFolder.deleted))'), + active_folders = relationship( + "LibraryFolder", + primaryjoin=("and_(LibraryFolder.parent_id == LibraryFolder.id, not_(LibraryFolder.deleted))"), order_by=asc(name), # """sqlalchemy.exc.ArgumentError: Error creating eager relationship 'active_folders' # on parent class '' to child class '': # Cant use eager loading on a self referential relationship.""" # TODO: This is no longer the case. Fix this: https://docs.sqlalchemy.org/en/14/orm/self_referential.html#configuring-self-referential-eager-loading - viewonly=True) + viewonly=True, + ) - datasets = relationship('LibraryDataset', - primaryjoin=(lambda: LibraryDataset.folder_id == LibraryFolder.id and LibraryDataset.library_dataset_dataset_association_id.isnot(None)), - order_by=(lambda: asc(LibraryDataset._name)), - viewonly=True) - - active_datasets = relationship('LibraryDataset', + datasets = relationship( + "LibraryDataset", primaryjoin=( - 'and_(LibraryDataset.folder_id == LibraryFolder.id, not_(LibraryDataset.deleted), LibraryDataset.library_dataset_dataset_association_id.isnot(None))'), + lambda: LibraryDataset.folder_id == LibraryFolder.id + and LibraryDataset.library_dataset_dataset_association_id.isnot(None) + ), order_by=(lambda: asc(LibraryDataset._name)), - viewonly=True) + viewonly=True, + ) - library_root = relationship('Library', back_populates='root_folder') - actions = relationship('LibraryFolderPermissions', back_populates='folder') + active_datasets = relationship( + "LibraryDataset", + primaryjoin=( + "and_(LibraryDataset.folder_id == LibraryFolder.id, not_(LibraryDataset.deleted), LibraryDataset.library_dataset_dataset_association_id.isnot(None))" + ), + order_by=(lambda: asc(LibraryDataset._name)), + viewonly=True, + ) - dict_element_visible_keys = ['id', 'parent_id', 'name', 'description', 'item_count', 'genome_build', 'update_time', 'deleted'] + library_root = relationship("Library", back_populates="root_folder") + actions = relationship("LibraryFolderPermissions", back_populates="folder") + + dict_element_visible_keys = [ + "id", + "parent_id", + "name", + "description", + "item_count", + "genome_build", + "update_time", + "deleted", + ] def __init__(self, name=None, description=None, item_count=0, order_id=None, genome_build=None): self.name = name or "Unnamed folder" @@ -4608,7 +4837,7 @@ class LibraryFolder(Base, Dictifiable, HasName, Serializable): library_dataset.folder_id = self.id library_dataset.order_id = self.item_count self.item_count += 1 - if genome_build not in [None, '?']: + if genome_build not in [None, "?"]: self.genome_build = genome_build def add_folder(self, folder): @@ -4619,7 +4848,11 @@ class LibraryFolder(Base, Dictifiable, HasName, Serializable): @property def activatable_library_datasets(self): # This needs to be a list - return [ld for ld in self.datasets if ld.library_dataset_dataset_association and not ld.library_dataset_dataset_association.dataset.deleted] + return [ + ld + for ld in self.datasets + if ld.library_dataset_dataset_association and not ld.library_dataset_dataset_association.dataset.deleted + ] def _serialize(self, id_encoder, serialization_options): rval = dict_for( @@ -4640,14 +4873,14 @@ class LibraryFolder(Base, Dictifiable, HasName, Serializable): datasets = [] for dataset in self.datasets: datasets.append(dataset.serialize(id_encoder, serialization_options)) - rval['datasets'] = datasets + rval["datasets"] = datasets serialization_options.attach_identifier(id_encoder, self, rval) return rval - def to_dict(self, view='collection', value_mapper=None): + def to_dict(self, view="collection", value_mapper=None): rval = super().to_dict(view=view, value_mapper=value_mapper) - rval['library_path'] = self.library_path - rval['parent_library_id'] = self.parent_library.id + rval["library_path"] = self.library_path + rval["parent_library_id"] = self.parent_library.id return rval @property @@ -4668,44 +4901,52 @@ class LibraryFolder(Base, Dictifiable, HasName, Serializable): class LibraryDataset(Base, Serializable): - __tablename__ = 'library_dataset' + __tablename__ = "library_dataset" id = Column(Integer, primary_key=True) # current version of dataset, if null, there is not a current version selected - library_dataset_dataset_association_id = Column(Integer, - ForeignKey('library_dataset_dataset_association.id', - use_alter=True, name='library_dataset_dataset_association_id_fk'), - nullable=True, index=True) - folder_id = Column(Integer, ForeignKey('library_folder.id'), index=True) + library_dataset_dataset_association_id = Column( + Integer, + ForeignKey( + "library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fk" + ), + nullable=True, + index=True, + ) + folder_id = Column(Integer, ForeignKey("library_folder.id"), index=True) # not currently being used, but for possible future use order_id = Column(Integer) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) # when not None/null this will supercede display in library (but not when imported into user's history?) - _name = Column('name', TrimmedString(255), index=True) + _name = Column("name", TrimmedString(255), index=True) # when not None/null this will supercede display in library (but not when imported into user's history?) - _info = Column('info', TrimmedString(255)) + _info = Column("info", TrimmedString(255)) deleted = Column(Boolean, index=True, default=False) purged = Column(Boolean, index=True, default=False) - folder = relationship('LibraryFolder') - library_dataset_dataset_association = relationship('LibraryDatasetDatasetAssociation', - foreign_keys=library_dataset_dataset_association_id, - post_update=True) - expired_datasets = relationship('LibraryDatasetDatasetAssociation', + folder = relationship("LibraryFolder") + library_dataset_dataset_association = relationship( + "LibraryDatasetDatasetAssociation", foreign_keys=library_dataset_dataset_association_id, post_update=True + ) + expired_datasets = relationship( + "LibraryDatasetDatasetAssociation", foreign_keys=[id, library_dataset_dataset_association_id], primaryjoin=( - 'and_(LibraryDataset.id == LibraryDatasetDatasetAssociation.library_dataset_id, \ - not_(LibraryDataset.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.id))' + "and_(LibraryDataset.id == LibraryDatasetDatasetAssociation.library_dataset_id, \ + not_(LibraryDataset.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.id))" ), viewonly=True, - uselist=True) - actions = relationship('LibraryDatasetPermissions', back_populates='library_dataset') + uselist=True, + ) + actions = relationship("LibraryDatasetPermissions", back_populates="library_dataset") # This class acts as a proxy to the currently selected LDDA - upload_options = [('upload_file', 'Upload files'), - ('upload_directory', 'Upload directory of files'), - ('upload_paths', 'Upload files from filesystem paths'), - ('import_from_history', 'Import datasets from your current history')] + upload_options = [ + ("upload_file", "Upload files"), + ("upload_directory", "Upload directory of files"), + ("upload_paths", "Upload files from filesystem paths"), + ("import_from_history", "Import datasets from your current history"), + ] def get_info(self): if self.library_dataset_dataset_association: @@ -4713,10 +4954,11 @@ class LibraryDataset(Base, Serializable): elif self._info: return self._info else: - return 'no info' + return "no info" def set_info(self, info): self._info = info + info = property(get_info, set_info) def get_name(self): @@ -4725,10 +4967,11 @@ class LibraryDataset(Base, Serializable): elif self._name: return self._name else: - return 'Unnamed dataset' + return "Unnamed dataset" def set_name(self, name): self._name = name + name = property(get_name, set_name) def display_name(self): @@ -4745,54 +4988,57 @@ class LibraryDataset(Base, Serializable): serialization_options.attach_identifier(id_encoder, self, rval) return rval - def to_dict(self, view='collection'): + def to_dict(self, view="collection"): # Since this class is a proxy to rather complex attributes we want to # display in other objects, we can't use the simpler method used by # other model classes. ldda = self.library_dataset_dataset_association - rval = dict(id=self.id, - ldda_id=ldda.id, - parent_library_id=self.folder.parent_library.id, - folder_id=self.folder_id, - model_class=self.__class__.__name__, - state=ldda.state, - name=ldda.name, - file_name=ldda.file_name, - created_from_basename=ldda.created_from_basename, - uploaded_by=ldda.user and ldda.user.email, - message=ldda.message, - date_uploaded=ldda.create_time.isoformat(), - update_time=ldda.update_time.isoformat(), - file_size=int(ldda.get_size()), - file_ext=ldda.ext, - data_type=f"{ldda.datatype.__class__.__module__}.{ldda.datatype.__class__.__name__}", - genome_build=ldda.dbkey, - misc_info=ldda.info, - misc_blurb=ldda.blurb, - peek=(lambda ldda: ldda.display_peek() if ldda.peek and ldda.peek != 'no peek' else None)(ldda)) + rval = dict( + id=self.id, + ldda_id=ldda.id, + parent_library_id=self.folder.parent_library.id, + folder_id=self.folder_id, + model_class=self.__class__.__name__, + state=ldda.state, + name=ldda.name, + file_name=ldda.file_name, + created_from_basename=ldda.created_from_basename, + uploaded_by=ldda.user and ldda.user.email, + message=ldda.message, + date_uploaded=ldda.create_time.isoformat(), + update_time=ldda.update_time.isoformat(), + file_size=int(ldda.get_size()), + file_ext=ldda.ext, + data_type=f"{ldda.datatype.__class__.__module__}.{ldda.datatype.__class__.__name__}", + genome_build=ldda.dbkey, + misc_info=ldda.info, + misc_blurb=ldda.blurb, + peek=(lambda ldda: ldda.display_peek() if ldda.peek and ldda.peek != "no peek" else None)(ldda), + ) if ldda.dataset.uuid is None: - rval['uuid'] = None + rval["uuid"] = None else: - rval['uuid'] = str(ldda.dataset.uuid) + rval["uuid"] = str(ldda.dataset.uuid) for name in ldda.metadata.spec.keys(): val = ldda.metadata.get(name) if isinstance(val, MetadataFile): val = val.file_name elif isinstance(val, list): - val = ', '.join(str(v) for v in val) + val = ", ".join(str(v) for v in val) rval[f"metadata_{name}"] = val return rval class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, Serializable): - - def __init__(self, - copied_from_history_dataset_association=None, - copied_from_library_dataset_dataset_association=None, - library_dataset=None, - user=None, - sa_session=None, - **kwd): + def __init__( + self, + copied_from_history_dataset_association=None, + copied_from_library_dataset_dataset_association=None, + library_dataset=None, + user=None, + sa_session=None, + **kwd, + ): # FIXME: sa_session is must be passed to DataSetInstance if the create_dataset # parameter in kwd is True so that the new object can be flushed. Is there a better way? DatasetInstance.__init__(self, sa_session=sa_session, **kwd) @@ -4805,19 +5051,21 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, Serializable): def to_history_dataset_association(self, target_history, parent_id=None, add_to_history=False, visible=None): sa_session = object_session(self) - hda = HistoryDatasetAssociation(name=self.name, - info=self.info, - blurb=self.blurb, - peek=self.peek, - tool_version=self.tool_version, - extension=self.extension, - dbkey=self.dbkey, - dataset=self.dataset, - visible=visible if visible is not None else self.visible, - deleted=self.deleted, - parent_id=parent_id, - copied_from_library_dataset_dataset_association=self, - history=target_history) + hda = HistoryDatasetAssociation( + name=self.name, + info=self.info, + blurb=self.blurb, + peek=self.peek, + tool_version=self.tool_version, + extension=self.extension, + dbkey=self.dbkey, + dataset=self.dataset, + visible=visible if visible is not None else self.visible, + deleted=self.deleted, + parent_id=parent_id, + copied_from_library_dataset_dataset_association=self, + history=target_history, + ) tag_manager = galaxy.model.tags.GalaxyTagHandler(sa_session) src_ldda_tags = tag_manager.get_tags_str(self.tags) @@ -4835,19 +5083,21 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, Serializable): def copy(self, parent_id=None, target_folder=None): sa_session = object_session(self) - ldda = LibraryDatasetDatasetAssociation(name=self.name, - info=self.info, - blurb=self.blurb, - peek=self.peek, - tool_version=self.tool_version, - extension=self.extension, - dbkey=self.dbkey, - dataset=self.dataset, - visible=self.visible, - deleted=self.deleted, - parent_id=parent_id, - copied_from_library_dataset_dataset_association=self, - folder=target_folder) + ldda = LibraryDatasetDatasetAssociation( + name=self.name, + info=self.info, + blurb=self.blurb, + peek=self.peek, + tool_version=self.tool_version, + extension=self.extension, + dbkey=self.dbkey, + dataset=self.dataset, + visible=self.visible, + deleted=self.deleted, + parent_id=parent_id, + copied_from_library_dataset_dataset_association=self, + folder=target_folder, + ) tag_manager = galaxy.model.tags.GalaxyTagHandler(sa_session) src_ldda_tags = tag_manager.get_tags_str(self.tags) @@ -4880,7 +5130,7 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, Serializable): self._handle_serialize_files(id_encoder, serialization_options, rval) return rval - def to_dict(self, view='collection'): + def to_dict(self, view="collection"): # Since this class is a proxy to rather complex attributes we want to # display in other objects, we can't use the simpler method used by # other model classes. @@ -4891,30 +5141,32 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, Serializable): file_size = 0 # TODO: render tags here - rval = dict(id=ldda.id, - hda_ldda='ldda', - model_class=self.__class__.__name__, - name=ldda.name, - deleted=ldda.deleted, - visible=ldda.visible, - state=ldda.state, - library_dataset_id=ldda.library_dataset_id, - file_size=file_size, - file_name=ldda.file_name, - update_time=ldda.update_time.isoformat(), - file_ext=ldda.ext, - data_type=f"{ldda.datatype.__class__.__module__}.{ldda.datatype.__class__.__name__}", - genome_build=ldda.dbkey, - misc_info=ldda.info, - misc_blurb=ldda.blurb, - created_from_basename=ldda.created_from_basename) + rval = dict( + id=ldda.id, + hda_ldda="ldda", + model_class=self.__class__.__name__, + name=ldda.name, + deleted=ldda.deleted, + visible=ldda.visible, + state=ldda.state, + library_dataset_id=ldda.library_dataset_id, + file_size=file_size, + file_name=ldda.file_name, + update_time=ldda.update_time.isoformat(), + file_ext=ldda.ext, + data_type=f"{ldda.datatype.__class__.__module__}.{ldda.datatype.__class__.__name__}", + genome_build=ldda.dbkey, + misc_info=ldda.info, + misc_blurb=ldda.blurb, + created_from_basename=ldda.created_from_basename, + ) if ldda.dataset.uuid is None: - rval['uuid'] = None + rval["uuid"] = None else: - rval['uuid'] = str(ldda.dataset.uuid) - rval['parent_library_id'] = ldda.library_dataset.folder.parent_library.id + rval["uuid"] = str(ldda.dataset.uuid) + rval["parent_library_id"] = ldda.library_dataset.folder.parent_library.id if ldda.extended_metadata is not None: - rval['extended_metadata'] = ldda.extended_metadata.data + rval["extended_metadata"] = ldda.extended_metadata.data for name in ldda.metadata.spec.keys(): val = ldda.metadata.get(name) if isinstance(val, MetadataFile): @@ -4930,7 +5182,7 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, Serializable): ldda = self sql = text( - ''' + """ WITH RECURSIVE parent_folders_of(folder_id) AS (SELECT folder_id FROM library_dataset @@ -4946,32 +5198,34 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, Serializable): WHERE id = :ldda_id) WHERE exists (SELECT 1 FROM parent_folders_of WHERE library_folder.id = parent_folders_of.folder_id) - ''').execution_options(autocommit=True) - ret = object_session(self).execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) + """ + ).execution_options(autocommit=True) + ret = object_session(self).execute(sql, {"library_dataset_id": ldda.library_dataset_id, "ldda_id": ldda.id}) if ret.rowcount < 1: - log.warning(f'Attempt to updated parent folder times failed: {ret.rowcount} records updated.') + log.warning(f"Attempt to updated parent folder times failed: {ret.rowcount} records updated.") class ExtendedMetadata(Base, RepresentById): - __tablename__ = 'extended_metadata' + __tablename__ = "extended_metadata" id = Column(Integer, primary_key=True) data = Column(MutableJSONType) - children = relationship('ExtendedMetadataIndex', back_populates='extended_metadata') + children = relationship("ExtendedMetadataIndex", back_populates="extended_metadata") def __init__(self, data): self.data = data class ExtendedMetadataIndex(Base, RepresentById): - __tablename__ = 'extended_metadata_index' + __tablename__ = "extended_metadata_index" id = Column(Integer, primary_key=True) - extended_metadata_id = Column(Integer, - ForeignKey('extended_metadata.id', onupdate='CASCADE', ondelete='CASCADE'), index=True) + extended_metadata_id = Column( + Integer, ForeignKey("extended_metadata.id", onupdate="CASCADE", ondelete="CASCADE"), index=True + ) path = Column(String(255)) value = Column(TEXT) - extended_metadata = relationship('ExtendedMetadata', back_populates='children') + extended_metadata = relationship("ExtendedMetadata", back_populates="children") def __init__(self, extended_metadata, path, value): self.extended_metadata = extended_metadata @@ -4980,25 +5234,27 @@ class ExtendedMetadataIndex(Base, RepresentById): class LibraryInfoAssociation(Base, RepresentById): - __tablename__ = 'library_info_association' + __tablename__ = "library_info_association" id = Column(Integer, primary_key=True) - library_id = Column(Integer, ForeignKey('library.id'), index=True) - form_definition_id = Column(Integer, ForeignKey('form_definition.id'), index=True) - form_values_id = Column(Integer, ForeignKey('form_values.id'), index=True) + library_id = Column(Integer, ForeignKey("library.id"), index=True) + form_definition_id = Column(Integer, ForeignKey("form_definition.id"), index=True) + form_values_id = Column(Integer, ForeignKey("form_values.id"), index=True) inheritable = Column(Boolean, index=True, default=False) deleted = Column(Boolean, index=True, default=False) - library = relationship('Library', + library = relationship( + "Library", primaryjoin=( - lambda: and_( - LibraryInfoAssociation.library_id == Library.id, - not_(LibraryInfoAssociation.deleted)) - )) - template = relationship('FormDefinition', - primaryjoin=lambda: LibraryInfoAssociation.form_definition_id == FormDefinition.id) - info = relationship('FormValues', - primaryjoin=lambda: LibraryInfoAssociation.form_values_id == FormValues.id) # type: ignore[has-type] + lambda: and_(LibraryInfoAssociation.library_id == Library.id, not_(LibraryInfoAssociation.deleted)) + ), + ) + template = relationship( + "FormDefinition", primaryjoin=lambda: LibraryInfoAssociation.form_definition_id == FormDefinition.id + ) + info = relationship( + "FormValues", primaryjoin=lambda: LibraryInfoAssociation.form_values_id == FormValues.id # type: ignore[has-type] + ) def __init__(self, library, form_definition, info, inheritable=False): self.library = library @@ -5008,23 +5264,28 @@ class LibraryInfoAssociation(Base, RepresentById): class LibraryFolderInfoAssociation(Base, RepresentById): - __tablename__ = 'library_folder_info_association' + __tablename__ = "library_folder_info_association" id = Column(Integer, primary_key=True) - library_folder_id = Column(Integer, ForeignKey('library_folder.id'), nullable=True, index=True) - form_definition_id = Column(Integer, ForeignKey('form_definition.id'), index=True) - form_values_id = Column(Integer, ForeignKey('form_values.id'), index=True) + library_folder_id = Column(Integer, ForeignKey("library_folder.id"), nullable=True, index=True) + form_definition_id = Column(Integer, ForeignKey("form_definition.id"), index=True) + form_values_id = Column(Integer, ForeignKey("form_values.id"), index=True) inheritable = Column(Boolean, index=True, default=False) deleted = Column(Boolean, index=True, default=False) - folder = relationship('LibraryFolder', - primaryjoin=(lambda: - (LibraryFolderInfoAssociation.library_folder_id == LibraryFolder.id) - & (not_(LibraryFolderInfoAssociation.deleted)))) - template = relationship('FormDefinition', - primaryjoin=(lambda: LibraryFolderInfoAssociation.form_definition_id == FormDefinition.id)) - info = relationship('FormValues', - primaryjoin=(lambda: LibraryFolderInfoAssociation.form_values_id == FormValues.id)) # type: ignore[has-type] + folder = relationship( + "LibraryFolder", + primaryjoin=( + lambda: (LibraryFolderInfoAssociation.library_folder_id == LibraryFolder.id) + & (not_(LibraryFolderInfoAssociation.deleted)) + ), + ) + template = relationship( + "FormDefinition", primaryjoin=(lambda: LibraryFolderInfoAssociation.form_definition_id == FormDefinition.id) + ) + info = relationship( + "FormValues", primaryjoin=(lambda: LibraryFolderInfoAssociation.form_values_id == FormValues.id) # type: ignore[has-type] + ) def __init__(self, folder, form_definition, info, inheritable=False): self.folder = folder @@ -5034,27 +5295,33 @@ class LibraryFolderInfoAssociation(Base, RepresentById): class LibraryDatasetDatasetInfoAssociation(Base, RepresentById): - __tablename__ = 'library_dataset_dataset_info_association' + __tablename__ = "library_dataset_dataset_info_association" id = Column(Integer, primary_key=True) - library_dataset_dataset_association_id = Column(Integer, - ForeignKey('library_dataset_dataset_association.id'), nullable=True, index=True) - form_definition_id = Column(Integer, ForeignKey('form_definition.id'), index=True) - form_values_id = Column(Integer, ForeignKey('form_values.id'), index=True) + library_dataset_dataset_association_id = Column( + Integer, ForeignKey("library_dataset_dataset_association.id"), nullable=True, index=True + ) + form_definition_id = Column(Integer, ForeignKey("form_definition.id"), index=True) + form_values_id = Column(Integer, ForeignKey("form_values.id"), index=True) deleted = Column(Boolean, index=True, default=False) - library_dataset_dataset_association = relationship('LibraryDatasetDatasetAssociation', - primaryjoin=(lambda: - (LibraryDatasetDatasetInfoAssociation.library_dataset_dataset_association_id - == LibraryDatasetDatasetAssociation.id) + library_dataset_dataset_association = relationship( + "LibraryDatasetDatasetAssociation", + primaryjoin=( + lambda: ( + LibraryDatasetDatasetInfoAssociation.library_dataset_dataset_association_id + == LibraryDatasetDatasetAssociation.id + ) & (not_(LibraryDatasetDatasetInfoAssociation.deleted)) - )) - template = relationship('FormDefinition', - primaryjoin=(lambda: - LibraryDatasetDatasetInfoAssociation.form_definition_id == FormDefinition.id)) - info = relationship('FormValues', - primaryjoin=(lambda: - LibraryDatasetDatasetInfoAssociation.form_values_id == FormValues.id)) # type: ignore[has-type] + ), + ) + template = relationship( + "FormDefinition", + primaryjoin=(lambda: LibraryDatasetDatasetInfoAssociation.form_definition_id == FormDefinition.id), + ) + info = relationship( + "FormValues", primaryjoin=(lambda: LibraryDatasetDatasetInfoAssociation.form_values_id == FormValues.id) # type: ignore[has-type] + ) def __init__(self, library_dataset_dataset_association, form_definition, info): # TODO: need to figure out if this should be inheritable to the associated LibraryDataset @@ -5068,51 +5335,58 @@ class LibraryDatasetDatasetInfoAssociation(Base, RepresentById): class ImplicitlyConvertedDatasetAssociation(Base, RepresentById): - __tablename__ = 'implicitly_converted_dataset_association' + __tablename__ = "implicitly_converted_dataset_association" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - hda_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True, nullable=True) - ldda_id = Column(Integer, - ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) - hda_parent_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) - ldda_parent_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) + hda_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True, nullable=True) + ldda_id = Column(Integer, ForeignKey("library_dataset_dataset_association.id"), index=True, nullable=True) + hda_parent_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) + ldda_parent_id = Column(Integer, ForeignKey("library_dataset_dataset_association.id"), index=True) deleted = Column(Boolean, index=True, default=False) metadata_safe = Column(Boolean, index=True, default=True) type = Column(TrimmedString(255)) - parent_hda = relationship('HistoryDatasetAssociation', - primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_parent_id - == HistoryDatasetAssociation.id), - back_populates='implicitly_converted_datasets') - dataset_ldda = relationship('LibraryDatasetDatasetAssociation', - primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.ldda_id - == LibraryDatasetDatasetAssociation.id), - back_populates='implicitly_converted_parent_datasets') - dataset = relationship('HistoryDatasetAssociation', - primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_id - == HistoryDatasetAssociation.id), - back_populates='implicitly_converted_parent_datasets') - parent_ldda = relationship('LibraryDatasetDatasetAssociation', - primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.ldda_parent_id - == LibraryDatasetDatasetAssociation.table.c.id), - back_populates='implicitly_converted_datasets') + parent_hda = relationship( + "HistoryDatasetAssociation", + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_parent_id == HistoryDatasetAssociation.id), + back_populates="implicitly_converted_datasets", + ) + dataset_ldda = relationship( + "LibraryDatasetDatasetAssociation", + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.ldda_id == LibraryDatasetDatasetAssociation.id), + back_populates="implicitly_converted_parent_datasets", + ) + dataset = relationship( + "HistoryDatasetAssociation", + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_id == HistoryDatasetAssociation.id), + back_populates="implicitly_converted_parent_datasets", + ) + parent_ldda = relationship( + "LibraryDatasetDatasetAssociation", + primaryjoin=( + lambda: ImplicitlyConvertedDatasetAssociation.ldda_parent_id == LibraryDatasetDatasetAssociation.table.c.id + ), + back_populates="implicitly_converted_datasets", + ) - def __init__(self, id=None, parent=None, dataset=None, file_type=None, deleted=False, purged=False, metadata_safe=True): + def __init__( + self, id=None, parent=None, dataset=None, file_type=None, deleted=False, purged=False, metadata_safe=True + ): self.id = id if isinstance(dataset, HistoryDatasetAssociation): self.dataset = dataset elif isinstance(dataset, LibraryDatasetDatasetAssociation): self.dataset_ldda = dataset else: - raise AttributeError(f'Unknown dataset type provided for dataset: {type(dataset)}') + raise AttributeError(f"Unknown dataset type provided for dataset: {type(dataset)}") if isinstance(parent, HistoryDatasetAssociation): self.parent_hda = parent elif isinstance(parent, LibraryDatasetDatasetAssociation): self.parent_ldda = parent else: - raise AttributeError(f'Unknown dataset type provided for parent: {type(parent)}') + raise AttributeError(f"Unknown dataset type provided for parent: {type(parent)}") self.type = file_type self.deleted = deleted self.purged = purged @@ -5146,36 +5420,32 @@ class InnerCollectionFilter(NamedTuple): class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): - __tablename__ = 'dataset_collection' + __tablename__ = "dataset_collection" id = Column(Integer, primary_key=True) collection_type = Column(Unicode(255), nullable=False) - populated_state = Column(TrimmedString(64), default='ok', nullable=False) + populated_state = Column(TrimmedString(64), default="ok", nullable=False) populated_state_message = Column(TEXT) element_count = Column(Integer, nullable=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - elements = relationship('DatasetCollectionElement', + elements = relationship( + "DatasetCollectionElement", primaryjoin=(lambda: DatasetCollection.id == DatasetCollectionElement.dataset_collection_id), # type: ignore[has-type] - back_populates='collection', - order_by=lambda: DatasetCollectionElement.element_index) # type: ignore[has-type] + back_populates="collection", + order_by=lambda: DatasetCollectionElement.element_index, # type: ignore[has-type] + ) - dict_collection_visible_keys = ['id', 'collection_type'] - dict_element_visible_keys = ['id', 'collection_type'] + dict_collection_visible_keys = ["id", "collection_type"] + dict_element_visible_keys = ["id", "collection_type"] class populated_states(str, Enum): - NEW = 'new' # New dataset collection, unpopulated elements - OK = 'ok' # Collection elements populated (HDAs may or may not have errors) - FAILED = 'failed' # some problem populating state, won't be populated + NEW = "new" # New dataset collection, unpopulated elements + OK = "ok" # Collection elements populated (HDAs may or may not have errors) + FAILED = "failed" # some problem populating state, won't be populated - def __init__( - self, - id=None, - collection_type=None, - populated=True, - element_count=None - ): + def __init__(self, id=None, collection_type=None, populated=True, element_count=None): self.id = id self.collection_type = collection_type if not populated: @@ -5189,8 +5459,18 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): hda_attributes: Optional[Iterable[str]] = None, dataset_attributes: Optional[Iterable[str]] = None, dataset_permission_attributes: Optional[Iterable[str]] = None, - return_entities: Optional[Iterable[Union[Type[HistoryDatasetAssociation], Type[Dataset], Type[DatasetPermissions], Type['DatasetCollection'], Type['DatasetCollectionElement']]]] = None, - inner_filter: Optional[InnerCollectionFilter] = None + return_entities: Optional[ + Iterable[ + Union[ + Type[HistoryDatasetAssociation], + Type[Dataset], + Type[DatasetPermissions], + Type["DatasetCollection"], + Type["DatasetCollectionElement"], + ] + ] + ] = None, + inner_filter: Optional[InnerCollectionFilter] = None, ): collection_attributes = collection_attributes or () element_attributes = element_attributes or () @@ -5211,22 +5491,21 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): label_fragment = f"_{nesting_level}" if nesting_level is not None else "" return [getattr(column_collection, a).label(f"{a}{label_fragment}") for a in attributes] - q = db_session.query( - *attribute_columns(dce.c, element_attributes, nesting_level), - *attribute_columns(dc.c, collection_attributes, nesting_level), - ).select_from( - dce, dc - ).join( - dce, dce.c.dataset_collection_id == dc.c.id - ).filter(dc.c.id == dataset_collection.id) - while ':' in depth_collection_type: + q = ( + db_session.query( + *attribute_columns(dce.c, element_attributes, nesting_level), + *attribute_columns(dc.c, collection_attributes, nesting_level), + ) + .select_from(dce, dc) + .join(dce, dce.c.dataset_collection_id == dc.c.id) + .filter(dc.c.id == dataset_collection.id) + ) + while ":" in depth_collection_type: nesting_level += 1 inner_dc = alias(DatasetCollection) inner_dce = alias(DatasetCollectionElement) order_by_columns.append(inner_dce.c.element_index) - q = q.join( - inner_dc, inner_dc.c.id == dce.c.child_collection_id - ).outerjoin( + q = q.join(inner_dc, inner_dc.c.id == dce.c.child_collection_id).outerjoin( inner_dce, inner_dce.c.dataset_collection_id == inner_dc.c.id ) q = q.add_columns( @@ -5239,16 +5518,20 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): if inner_filter: q = q.filter(inner_filter.produce_filter(dc.c)) - if hda_attributes or dataset_attributes or dataset_permission_attributes or return_entities and not return_entities == (DatasetCollectionElement,): + if ( + hda_attributes + or dataset_attributes + or dataset_permission_attributes + or return_entities + and not return_entities == (DatasetCollectionElement,) + ): q = q.join(HistoryDatasetAssociation).join(Dataset) if dataset_permission_attributes: q = q.join(DatasetPermissions) - q = q.add_columns( - *attribute_columns(HistoryDatasetAssociation, hda_attributes) - ).add_columns( - *attribute_columns(Dataset, dataset_attributes) - ).add_columns( - *attribute_columns(DatasetPermissions, dataset_permission_attributes) + q = ( + q.add_columns(*attribute_columns(HistoryDatasetAssociation, hda_attributes)) + .add_columns(*attribute_columns(Dataset, dataset_attributes)) + .add_columns(*attribute_columns(DatasetPermissions, dataset_permission_attributes)) ) for entity in return_entities: q = q.add_entity(entity) @@ -5258,11 +5541,8 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): @property def dataset_states_and_extensions_summary(self): - if not hasattr(self, '_dataset_states_and_extensions_summary'): - q = self._get_nested_collection_attributes( - hda_attributes=('extension',), - dataset_attributes=('state',) - ) + if not hasattr(self, "_dataset_states_and_extensions_summary"): + q = self._get_nested_collection_attributes(hda_attributes=("extension",), dataset_attributes=("state",)) extensions = set() states = set() for extension, state in q: @@ -5275,14 +5555,16 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): @property def populated_optimized(self): - if not hasattr(self, '_populated_optimized'): + if not hasattr(self, "_populated_optimized"): _populated_optimized = True if ":" not in self.collection_type: _populated_optimized = self.populated_state == DatasetCollection.populated_states.OK else: q = self._get_nested_collection_attributes( - collection_attributes=('populated_state',), - inner_filter=InnerCollectionFilter('populated_state', operator.__ne__, DatasetCollection.populated_states.OK) + collection_attributes=("populated_state",), + inner_filter=InnerCollectionFilter( + "populated_state", operator.__ne__, DatasetCollection.populated_states.OK + ), ) _populated_optimized = q.session.query(~exists(q.subquery())).scalar() @@ -5299,10 +5581,8 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): @property def dataset_action_tuples(self): - if not hasattr(self, '_dataset_action_tuples'): - q = self._get_nested_collection_attributes( - dataset_permission_attributes=('action', 'role_id') - ) + if not hasattr(self, "_dataset_action_tuples"): + q = self._get_nested_collection_attributes(dataset_permission_attributes=("action", "role_id")) _dataset_action_tuples = [] for _dataset_action_tuple in q: if _dataset_action_tuple[0] is None: @@ -5316,9 +5596,7 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): @property def element_identifiers_extensions_and_paths(self): q = self._get_nested_collection_attributes( - element_attributes=('element_identifier',), - hda_attributes=('extension',), - return_entities=(Dataset,) + element_attributes=("element_identifier",), hda_attributes=("extension",), return_entities=(Dataset,) ) return [(row[:-2], row.extension, row.Dataset.file_name) for row in q] @@ -5329,9 +5607,9 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): results = [] if object_session(self): q = self._get_nested_collection_attributes( - element_attributes=('element_identifier',), - hda_attributes=('extension',), - return_entities=(HistoryDatasetAssociation, Dataset) + element_attributes=("element_identifier",), + hda_attributes=("extension",), + return_entities=(HistoryDatasetAssociation, Dataset), ) # element_identifiers, extension, path for row in q: @@ -5343,7 +5621,14 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): # This will be in a remote tool evaluation context, so can't query database for dataset_element in self.dataset_elements_and_identifiers(): # Let's pretend name is element identifier - results.append([dataset_element._identifiers, dataset_element.hda.extension, dataset_element.hda.file_name, dataset_element.hda.get_metadata_file_paths_and_extensions()]) + results.append( + [ + dataset_element._identifiers, + dataset_element.hda.extension, + dataset_element.hda.file_name, + dataset_element.hda.get_metadata_file_paths_and_extensions(), + ] + ) return results @property @@ -5426,7 +5711,7 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): @property def state(self): # TODO: DatasetCollection state handling... - return 'ok' + return "ok" def validate(self): if self.collection_type is None: @@ -5447,11 +5732,15 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): error_message = f"Dataset collection has no {get_by_attribute} with key {key}." raise KeyError(error_message) - def copy(self, destination=None, element_destination=None, dataset_instance_attributes=None, flush=True, minimize_copies=False): - new_collection = DatasetCollection( - collection_type=self.collection_type, - element_count=self.element_count - ) + def copy( + self, + destination=None, + element_destination=None, + dataset_instance_attributes=None, + flush=True, + minimize_copies=False, + ): + new_collection = DatasetCollection(collection_type=self.collection_type, element_count=self.element_count) for element in self.elements: element.copy_to_collection( new_collection, @@ -5467,7 +5756,9 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): return new_collection def replace_failed_elements(self, replacements): - hda_id_to_element = dict(self._get_nested_collection_attributes(return_entities=[DatasetCollectionElement], hda_attributes=['id'])) + hda_id_to_element = dict( + self._get_nested_collection_attributes(return_entities=[DatasetCollectionElement], hda_attributes=["id"]) + ) for failed, replacement in replacements.items(): element = hda_id_to_element.get(failed.id) if element: @@ -5487,14 +5778,13 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): type=self.collection_type, populated_state=self.populated_state, populated_state_message=self.populated_state_message, - elements=list(map(lambda e: e.serialize(id_encoder, serialization_options), self.elements)) + elements=list(map(lambda e: e.serialize(id_encoder, serialization_options), self.elements)), ) serialization_options.attach_identifier(id_encoder, self, rval) return rval class DatasetCollectionInstance(HasName, UsesCreateAndUpdateTime): - @property def state(self): return self.collection.state @@ -5553,74 +5843,81 @@ class HistoryDatasetCollectionAssociation( UsesAnnotations, Serializable, ): - """ Associates a DatasetCollection with a History. """ - __tablename__ = 'history_dataset_collection_association' + """Associates a DatasetCollection with a History.""" + + __tablename__ = "history_dataset_collection_association" id = Column(Integer, primary_key=True) - collection_id = Column(Integer, ForeignKey('dataset_collection.id'), index=True) - history_id = Column(Integer, ForeignKey('history.id'), index=True) + collection_id = Column(Integer, ForeignKey("dataset_collection.id"), index=True) + history_id = Column(Integer, ForeignKey("history.id"), index=True) name = Column(TrimmedString(255)) hid = Column(Integer) visible = Column(Boolean) deleted = Column(Boolean, default=False) - copied_from_history_dataset_collection_association_id = Column(Integer, - ForeignKey('history_dataset_collection_association.id'), nullable=True) + copied_from_history_dataset_collection_association_id = Column( + Integer, ForeignKey("history_dataset_collection_association.id"), nullable=True + ) implicit_output_name = Column(Unicode(255), nullable=True) - job_id = Column(ForeignKey('job.id'), index=True, nullable=True) - implicit_collection_jobs_id = Column( - ForeignKey('implicit_collection_jobs.id'), index=True, nullable=True) + job_id = Column(ForeignKey("job.id"), index=True, nullable=True) + implicit_collection_jobs_id = Column(ForeignKey("implicit_collection_jobs.id"), index=True, nullable=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now, index=True) - collection = relationship('DatasetCollection') - history = relationship('History', back_populates='dataset_collections') + collection = relationship("DatasetCollection") + history = relationship("History", back_populates="dataset_collections") copied_from_history_dataset_collection_association = relationship( - 'HistoryDatasetCollectionAssociation', + "HistoryDatasetCollectionAssociation", primaryjoin=copied_from_history_dataset_collection_association_id == id, remote_side=[id], uselist=False, - back_populates='copied_to_history_dataset_collection_association', + back_populates="copied_to_history_dataset_collection_association", ) copied_to_history_dataset_collection_association = relationship( - 'HistoryDatasetCollectionAssociation', - back_populates='copied_from_history_dataset_collection_association', + "HistoryDatasetCollectionAssociation", + back_populates="copied_from_history_dataset_collection_association", ) - implicit_input_collections = relationship('ImplicitlyCreatedDatasetCollectionInput', - primaryjoin=(lambda: HistoryDatasetCollectionAssociation.id - == ImplicitlyCreatedDatasetCollectionInput.dataset_collection_id) + implicit_input_collections = relationship( + "ImplicitlyCreatedDatasetCollectionInput", + primaryjoin=( + lambda: HistoryDatasetCollectionAssociation.id + == ImplicitlyCreatedDatasetCollectionInput.dataset_collection_id + ), ) - implicit_collection_jobs = relationship('ImplicitCollectionJobs', uselist=False) + implicit_collection_jobs = relationship("ImplicitCollectionJobs", uselist=False) job = relationship( - 'Job', - back_populates='history_dataset_collection_associations', + "Job", + back_populates="history_dataset_collection_associations", uselist=False, ) - job_state_summary = relationship(HistoryDatasetCollectionJobStateSummary, - primaryjoin=(lambda: HistoryDatasetCollectionAssociation.id - == HistoryDatasetCollectionJobStateSummary.__table__.c.hdca_id), + job_state_summary = relationship( + HistoryDatasetCollectionJobStateSummary, + primaryjoin=( + lambda: HistoryDatasetCollectionAssociation.id + == HistoryDatasetCollectionJobStateSummary.__table__.c.hdca_id + ), foreign_keys=HistoryDatasetCollectionJobStateSummary.__table__.c.hdca_id, uselist=False, ) tags = relationship( - 'HistoryDatasetCollectionTagAssociation', + "HistoryDatasetCollectionTagAssociation", order_by=lambda: HistoryDatasetCollectionTagAssociation.id, - back_populates='dataset_collection', + back_populates="dataset_collection", ) annotations = relationship( - 'HistoryDatasetCollectionAssociationAnnotationAssociation', + "HistoryDatasetCollectionAssociationAnnotationAssociation", order_by=lambda: HistoryDatasetCollectionAssociationAnnotationAssociation.id, - back_populates='history_dataset_collection', + back_populates="history_dataset_collection", ) ratings = relationship( - 'HistoryDatasetCollectionRatingAssociation', + "HistoryDatasetCollectionRatingAssociation", order_by=lambda: HistoryDatasetCollectionRatingAssociation.id, # type: ignore[has-type] - back_populates='dataset_collection', + back_populates="dataset_collection", ) - creating_job_associations = relationship('JobToOutputDatasetCollectionAssociation', viewonly=True) + creating_job_associations = relationship("JobToOutputDatasetCollectionAssociation", viewonly=True) - dict_dbkeysandextensions_visible_keys = ['dbkeys', 'extensions'] - editable_keys = ('name', 'deleted', 'visible') + dict_dbkeysandextensions_visible_keys = ["dbkeys", "extensions"] + editable_keys = ("name", "deleted", "visible") def __init__(self, deleted=False, visible=True, **kwd): super().__init__(**kwd) @@ -5636,16 +5933,15 @@ class HistoryDatasetCollectionAssociation( return "dataset_collection" # TODO: down into DatasetCollectionInstance - content_type = 'dataset_collection' + content_type = "dataset_collection" @hybrid.hybrid_property def type_id(self): - return '-'.join((self.content_type, str(self.id))) + return "-".join((self.content_type, str(self.id))) @type_id.expression # type: ignore[no-redef] def type_id(cls): - return ((type_coerce(cls.content_type, Unicode) + '-' - + type_coerce(cls.id, Unicode)).label('type_id')) + return (type_coerce(cls.content_type, Unicode) + "-" + type_coerce(cls.id, Unicode)).label("type_id") @property def job_source_type(self): @@ -5658,13 +5954,13 @@ class HistoryDatasetCollectionAssociation( @property def dataset_dbkeys_and_extensions_summary(self): - if not hasattr(self, '_dataset_dbkeys_and_extensions_summary'): - rows = self.collection._get_nested_collection_attributes(hda_attributes=('_metadata', 'extension')) + if not hasattr(self, "_dataset_dbkeys_and_extensions_summary"): + rows = self.collection._get_nested_collection_attributes(hda_attributes=("_metadata", "extension")) extensions = set() dbkeys = set() for row in rows: if row is not None: - dbkey_field = row._metadata.get('dbkey') + dbkey_field = row._metadata.get("dbkey") if isinstance(dbkey_field, list): for dbkey in dbkey_field: dbkeys.add(dbkey) @@ -5702,10 +5998,12 @@ class HistoryDatasetCollectionAssociation( implicit_input_collections = [] for implicit_input_collection in self.implicit_input_collections: input_hdca = implicit_input_collection.input_dataset_collection - implicit_input_collections.append({ - "name": implicit_input_collection.name, - "input_dataset_collection": serialization_options.get_identifier(id_encoder, input_hdca) - }) + implicit_input_collections.append( + { + "name": implicit_input_collection.name, + "input_dataset_collection": serialization_options.get_identifier(id_encoder, input_hdca), + } + ) if implicit_input_collections: rval["implicit_input_collections"] = implicit_input_collections @@ -5714,18 +6012,22 @@ class HistoryDatasetCollectionAssociation( src_hdca = self while src_hdca.copied_from_history_dataset_collection_association: src_hdca = src_hdca.copied_from_history_dataset_collection_association - copied_from_history_dataset_collection_association_chain.append(serialization_options.get_identifier(id_encoder, src_hdca)) - rval["copied_from_history_dataset_collection_association_id_chain"] = copied_from_history_dataset_collection_association_chain + copied_from_history_dataset_collection_association_chain.append( + serialization_options.get_identifier(id_encoder, src_hdca) + ) + rval[ + "copied_from_history_dataset_collection_association_id_chain" + ] = copied_from_history_dataset_collection_association_chain serialization_options.attach_identifier(id_encoder, self, rval) return rval - def to_dict(self, view='collection'): + def to_dict(self, view="collection"): original_dict_value = super().to_dict(view=view) - if (view == 'dbkeysandextensions'): + if view == "dbkeysandextensions": (dbkeys, extensions) = self.dataset_dbkeys_and_extensions_summary dict_value = dict( dbkey=dbkeys.pop() if len(dbkeys) == 1 else "?", - extension=extensions.pop() if len(extensions) == 1 else "auto" + extension=extensions.pop() if len(extensions) == 1 else "auto", ) else: dict_value = dict( @@ -5738,7 +6040,7 @@ class HistoryDatasetCollectionAssociation( job_source_type=self.job_source_type, create_time=self.create_time.isoformat(), update_time=self.update_time.isoformat(), - **self._base_to_dict(view=view) + **self._base_to_dict(view=view), ) dict_value.update(original_dict_value) @@ -5746,7 +6048,9 @@ class HistoryDatasetCollectionAssociation( return dict_value def add_implicit_input_collection(self, name, history_dataset_collection): - self.implicit_input_collections.append(ImplicitlyCreatedDatasetCollectionInput(name, history_dataset_collection)) + self.implicit_input_collections.append( + ImplicitlyCreatedDatasetCollectionInput(name, history_dataset_collection) + ) def find_implicit_input_collection(self, name): matching_collection = None @@ -5756,7 +6060,14 @@ class HistoryDatasetCollectionAssociation( break return matching_collection - def copy(self, element_destination=None, dataset_instance_attributes=None, flush=True, set_hid=True, minimize_copies=False): + def copy( + self, + element_destination=None, + dataset_instance_attributes=None, + flush=True, + set_hid=True, + minimize_copies=False, + ): """ Create a copy of this history dataset collection association. Copy underlying collection. @@ -5809,15 +6120,18 @@ class HistoryDatasetCollectionAssociation( HDCA = HistoryDatasetCollectionAssociation # non-recursive part of the cte (starting point) - parents_cte = Query(DCE.dataset_collection_id) \ - .filter(or_(DCE.child_collection_id == collection_id, DCE.dataset_collection_id == collection_id)) \ + parents_cte = ( + Query(DCE.dataset_collection_id) + .filter(or_(DCE.child_collection_id == collection_id, DCE.dataset_collection_id == collection_id)) .cte(name="element_parents", recursive="True") + ) ep = aliased(parents_cte, name="ep") # add the recursive part of the cte expression dce = aliased(DCE, name="dce") - rec = Query(dce.dataset_collection_id.label('dataset_collection_id')) \ - .filter(dce.child_collection_id == ep.c.dataset_collection_id) + rec = Query(dce.dataset_collection_id.label("dataset_collection_id")).filter( + dce.child_collection_id == ep.c.dataset_collection_id + ) parents_cte = parents_cte.union(rec) # join parents to hdca, look for matching hdca_id @@ -5830,29 +6144,36 @@ class HistoryDatasetCollectionAssociation( class LibraryDatasetCollectionAssociation(Base, DatasetCollectionInstance, RepresentById): - """ Associates a DatasetCollection with a library folder. """ - __tablename__ = 'library_dataset_collection_association' + """Associates a DatasetCollection with a library folder.""" + + __tablename__ = "library_dataset_collection_association" id = Column(Integer, primary_key=True) - collection_id = Column(Integer, ForeignKey('dataset_collection.id'), index=True) - folder_id = Column(Integer, ForeignKey('library_folder.id'), index=True) + collection_id = Column(Integer, ForeignKey("dataset_collection.id"), index=True) + folder_id = Column(Integer, ForeignKey("library_folder.id"), index=True) name = Column(TrimmedString(255)) deleted = Column(Boolean, default=False) - collection = relationship('DatasetCollection') - folder = relationship('LibraryFolder') + collection = relationship("DatasetCollection") + folder = relationship("LibraryFolder") - tags = relationship('LibraryDatasetCollectionTagAssociation', + tags = relationship( + "LibraryDatasetCollectionTagAssociation", order_by=lambda: LibraryDatasetCollectionTagAssociation.id, - back_populates='dataset_collection') - annotations = relationship('LibraryDatasetCollectionAnnotationAssociation', + back_populates="dataset_collection", + ) + annotations = relationship( + "LibraryDatasetCollectionAnnotationAssociation", order_by=lambda: LibraryDatasetCollectionAnnotationAssociation.id, - back_populates="dataset_collection") - ratings = relationship('LibraryDatasetCollectionRatingAssociation', + back_populates="dataset_collection", + ) + ratings = relationship( + "LibraryDatasetCollectionRatingAssociation", order_by=lambda: LibraryDatasetCollectionRatingAssociation.id, # type: ignore[has-type] - back_populates="dataset_collection") + back_populates="dataset_collection", + ) - editable_keys = ('name', 'deleted') + editable_keys = ("name", "deleted") def __init__(self, deleted=False, **kwd): super().__init__(**kwd) @@ -5861,47 +6182,47 @@ class LibraryDatasetCollectionAssociation(Base, DatasetCollectionInstance, Repre # it is on instance instead of collection. self.deleted = deleted - def to_dict(self, view='collection'): - dict_value = dict( - folder_id=self.folder.id, - **self._base_to_dict(view=view) - ) + def to_dict(self, view="collection"): + dict_value = dict(folder_id=self.folder.id, **self._base_to_dict(view=view)) return dict_value class DatasetCollectionElement(Base, Dictifiable, Serializable): - """ Associates a DatasetInstance (hda or ldda) with a DatasetCollection. """ - __tablename__ = 'dataset_collection_element' + """Associates a DatasetInstance (hda or ldda) with a DatasetCollection.""" + + __tablename__ = "dataset_collection_element" id = Column(Integer, primary_key=True) # Parent collection id describing what collection this element belongs to. - dataset_collection_id = Column(Integer, - ForeignKey('dataset_collection.id'), index=True, nullable=False) + dataset_collection_id = Column(Integer, ForeignKey("dataset_collection.id"), index=True, nullable=False) # Child defined by this association - HDA, LDDA, or another dataset association... - hda_id = Column(Integer, - ForeignKey('history_dataset_association.id'), index=True, nullable=True) - ldda_id = Column(Integer, - ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) - child_collection_id = Column(Integer, - ForeignKey('dataset_collection.id'), index=True, nullable=True) + hda_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True, nullable=True) + ldda_id = Column(Integer, ForeignKey("library_dataset_dataset_association.id"), index=True, nullable=True) + child_collection_id = Column(Integer, ForeignKey("dataset_collection.id"), index=True, nullable=True) # Element index and identifier to define this parent-child relationship. element_index = Column(Integer) element_identifier = Column(Unicode(255)) - hda = relationship('HistoryDatasetAssociation', - primaryjoin=(lambda: DatasetCollectionElement.hda_id == HistoryDatasetAssociation.id)) - ldda = relationship('LibraryDatasetDatasetAssociation', - primaryjoin=(lambda: DatasetCollectionElement.ldda_id == LibraryDatasetDatasetAssociation.id)) - child_collection = relationship('DatasetCollection', - primaryjoin=(lambda: DatasetCollectionElement.child_collection_id == DatasetCollection.id)) - collection = relationship('DatasetCollection', + hda = relationship( + "HistoryDatasetAssociation", + primaryjoin=(lambda: DatasetCollectionElement.hda_id == HistoryDatasetAssociation.id), + ) + ldda = relationship( + "LibraryDatasetDatasetAssociation", + primaryjoin=(lambda: DatasetCollectionElement.ldda_id == LibraryDatasetDatasetAssociation.id), + ) + child_collection = relationship( + "DatasetCollection", primaryjoin=(lambda: DatasetCollectionElement.child_collection_id == DatasetCollection.id) + ) + collection = relationship( + "DatasetCollection", primaryjoin=(lambda: DatasetCollection.id == DatasetCollectionElement.dataset_collection_id), - back_populates='elements', + back_populates="elements", ) # actionable dataset id needs to be available via API... - dict_collection_visible_keys = ['id', 'element_type', 'element_index', 'element_identifier'] - dict_element_visible_keys = ['id', 'element_type', 'element_index', 'element_identifier'] + dict_collection_visible_keys = ["id", "element_type", "element_index", "element_identifier"] + dict_element_visible_keys = ["id", "element_type", "element_index", "element_identifier"] UNINITIALIZED_ELEMENT = object() @@ -5920,7 +6241,7 @@ class DatasetCollectionElement(Base, Dictifiable, Serializable): elif isinstance(element, DatasetCollection): self.child_collection = element elif element != self.UNINITIALIZED_ELEMENT: - raise AttributeError(f'Unknown element type provided: {type(element)}') + raise AttributeError(f"Unknown element type provided: {type(element)}") self.id = id self.collection = collection @@ -5980,7 +6301,15 @@ class DatasetCollectionElement(Base, Dictifiable, Serializable): else: return [element_object] - def copy_to_collection(self, collection, destination=None, element_destination=None, dataset_instance_attributes=None, flush=True, minimize_copies=False): + def copy_to_collection( + self, + collection, + destination=None, + element_destination=None, + dataset_instance_attributes=None, + flush=True, + minimize_copies=False, + ): dataset_instance_attributes = dataset_instance_attributes or {} element_object = self.element_object if element_destination: @@ -5996,7 +6325,11 @@ class DatasetCollectionElement(Base, Dictifiable, Serializable): new_element_object = None if minimize_copies: new_element_object = element_destination.get_dataset_by_hid(element_object.hid) - if new_element_object and new_element_object.dataset and new_element_object.dataset.id == element_object.dataset_id: + if ( + new_element_object + and new_element_object.dataset + and new_element_object.dataset.id == element_object.dataset_id + ): element_object = new_element_object else: new_element_object = element_object.copy(flush=flush, copy_tags=element_object.tags) @@ -6025,7 +6358,7 @@ class DatasetCollectionElement(Base, Dictifiable, Serializable): self, element_type=self.element_type, element_index=self.element_index, - element_identifier=self.element_identifier + element_identifier=self.element_identifier, ) serialization_options.attach_identifier(id_encoder, self, rval) element_obj = self.element_object @@ -6037,33 +6370,33 @@ class DatasetCollectionElement(Base, Dictifiable, Serializable): class Event(Base, RepresentById): - __tablename__ = 'event' + __tablename__ = "event" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - history_id = Column(Integer, ForeignKey('history.id'), index=True, nullable=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True, nullable=True) + history_id = Column(Integer, ForeignKey("history.id"), index=True, nullable=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True, nullable=True) message = Column(TrimmedString(1024)) - session_id = Column(Integer, ForeignKey('galaxy_session.id'), index=True, nullable=True) + session_id = Column(Integer, ForeignKey("galaxy_session.id"), index=True, nullable=True) tool_id = Column(String(255)) - history = relationship('History') - user = relationship('User') - galaxy_session = relationship('GalaxySession') + history = relationship("History") + user = relationship("User") + galaxy_session = relationship("GalaxySession") class GalaxySession(Base, RepresentById): - __tablename__ = 'galaxy_session' + __tablename__ = "galaxy_session" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True, nullable=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True, nullable=True) remote_host = Column(String(255)) remote_addr = Column(String(255)) referer = Column(TEXT) - current_history_id = Column(Integer, ForeignKey('history.id'), nullable=True) + current_history_id = Column(Integer, ForeignKey("history.id"), nullable=True) # unique 128 bit random number coerced to a string session_key = Column(TrimmedString(255), index=True, unique=True) is_valid = Column(Boolean, default=False) @@ -6071,9 +6404,9 @@ class GalaxySession(Base, RepresentById): prev_session_id = Column(Integer) disk_usage = Column(Numeric(15, 0), index=True) last_action = Column(DateTime) - current_history = relationship('History') - histories = relationship('GalaxySessionToHistoryAssociation', back_populates='galaxy_session') - user = relationship('User', back_populates='galaxy_sessions') + current_history = relationship("History") + histories = relationship("GalaxySessionToHistoryAssociation", back_populates="galaxy_session") + user = relationship("User", back_populates="galaxy_sessions") def __init__(self, is_valid=False, **kwd): super().__init__(**kwd) @@ -6093,18 +6426,19 @@ class GalaxySession(Base, RepresentById): def set_disk_usage(self, bytes): self.disk_usage = bytes + total_disk_usage = property(get_disk_usage, set_disk_usage) class GalaxySessionToHistoryAssociation(Base, RepresentById): - __tablename__ = 'galaxy_session_to_history' + __tablename__ = "galaxy_session_to_history" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) - session_id = Column(Integer, ForeignKey('galaxy_session.id'), index=True) - history_id = Column(Integer, ForeignKey('history.id'), index=True) - galaxy_session = relationship('GalaxySession', back_populates='histories') - history = relationship('History', back_populates='galaxy_sessions') + session_id = Column(Integer, ForeignKey("galaxy_session.id"), index=True) + history_id = Column(Integer, ForeignKey("history.id"), index=True) + galaxy_session = relationship("GalaxySession", back_populates="histories") + history = relationship("History", back_populates="galaxy_sessions") def __init__(self, galaxy_session, history): self.galaxy_session = galaxy_session @@ -6125,18 +6459,17 @@ class StoredWorkflow(Base, HasTags, Dictifiable, RepresentById): Each time a workflow is modified a revision is created, represented by a new :class:`galaxy.model.Workflow` instance. See :class:`galaxy.model.Workflow` for more information """ - __tablename__ = 'stored_workflow' - __table_args__ = ( - Index('ix_stored_workflow_slug', 'slug', mysql_length=200), - ) + + __tablename__ = "stored_workflow" + __table_args__ = (Index("ix_stored_workflow_slug", "slug", mysql_length=200),) id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now, index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True, nullable=False) - latest_workflow_id = Column(Integer, - ForeignKey('workflow.id', use_alter=True, name='stored_workflow_latest_workflow_id_fk'), - index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True, nullable=False) + latest_workflow_id = Column( + Integer, ForeignKey("workflow.id", use_alter=True, name="stored_workflow_latest_workflow_id_fk"), index=True + ) name = Column(TEXT) deleted = Column(Boolean, default=False) hidden = Column(Boolean, default=False) @@ -6145,48 +6478,72 @@ class StoredWorkflow(Base, HasTags, Dictifiable, RepresentById): from_path = Column(TEXT) published = Column(Boolean, index=True, default=False) - user = relationship('User', - primaryjoin=(lambda: User.id == StoredWorkflow.user_id), - back_populates='stored_workflows') - workflows = relationship('Workflow', - back_populates='stored_workflow', + user = relationship( + "User", primaryjoin=(lambda: User.id == StoredWorkflow.user_id), back_populates="stored_workflows" + ) + workflows = relationship( + "Workflow", + back_populates="stored_workflow", cascade="all, delete-orphan", primaryjoin=(lambda: StoredWorkflow.id == Workflow.stored_workflow_id), # type: ignore[has-type] - order_by=lambda: -Workflow.id) # type: ignore[has-type] - latest_workflow = relationship('Workflow', + order_by=lambda: -Workflow.id, # type: ignore[has-type] + ) + latest_workflow = relationship( + "Workflow", post_update=True, primaryjoin=(lambda: StoredWorkflow.latest_workflow_id == Workflow.id), # type: ignore[has-type] - lazy=False) - tags = relationship('StoredWorkflowTagAssociation', + lazy=False, + ) + tags = relationship( + "StoredWorkflowTagAssociation", order_by=lambda: StoredWorkflowTagAssociation.id, - back_populates="stored_workflow") - owner_tags = relationship('StoredWorkflowTagAssociation', - primaryjoin=(lambda: - and_(StoredWorkflow.id == StoredWorkflowTagAssociation.stored_workflow_id, - StoredWorkflow.user_id == StoredWorkflowTagAssociation.user_id) + back_populates="stored_workflow", + ) + owner_tags = relationship( + "StoredWorkflowTagAssociation", + primaryjoin=( + lambda: and_( + StoredWorkflow.id == StoredWorkflowTagAssociation.stored_workflow_id, + StoredWorkflow.user_id == StoredWorkflowTagAssociation.user_id, + ) ), viewonly=True, - order_by=lambda: StoredWorkflowTagAssociation.id) - annotations = relationship('StoredWorkflowAnnotationAssociation', + order_by=lambda: StoredWorkflowTagAssociation.id, + ) + annotations = relationship( + "StoredWorkflowAnnotationAssociation", order_by=lambda: StoredWorkflowAnnotationAssociation.id, - back_populates="stored_workflow") - ratings = relationship('StoredWorkflowRatingAssociation', + back_populates="stored_workflow", + ) + ratings = relationship( + "StoredWorkflowRatingAssociation", order_by=lambda: StoredWorkflowRatingAssociation.id, # type: ignore[has-type] - back_populates="stored_workflow") - users_shared_with = relationship('StoredWorkflowUserShareAssociation', - back_populates='stored_workflow') + back_populates="stored_workflow", + ) + users_shared_with = relationship("StoredWorkflowUserShareAssociation", back_populates="stored_workflow") average_rating: column_property # Set up proxy so that # StoredWorkflow.users_shared_with # returns a list of users that workflow is shared with. - users_shared_with_dot_users = association_proxy('users_shared_with', 'user') + users_shared_with_dot_users = association_proxy("users_shared_with", "user") - dict_collection_visible_keys = ['id', 'name', 'create_time', 'update_time', 'published', 'deleted', 'hidden'] - dict_element_visible_keys = ['id', 'name', 'create_time', 'update_time', 'published', 'deleted', 'hidden'] + dict_collection_visible_keys = ["id", "name", "create_time", "update_time", "published", "deleted", "hidden"] + dict_element_visible_keys = ["id", "name", "create_time", "update_time", "published", "deleted", "hidden"] - def __init__(self, user=None, name=None, slug=None, create_time=None, update_time=None, published=False, latest_workflow_id=None, workflow=None, hidden=False): + def __init__( + self, + user=None, + name=None, + slug=None, + create_time=None, + update_time=None, + published=False, + latest_workflow_id=None, + workflow=None, + hidden=False, + ): self.user = user self.name = name self.slug = slug @@ -6206,10 +6563,14 @@ class StoredWorkflow(Base, HasTags, Dictifiable, RepresentById): def show_in_tool_panel(self, user_id): sa_session = object_session(self) - return bool(sa_session.query(StoredWorkflowMenuEntry).filter( - StoredWorkflowMenuEntry.stored_workflow_id == self.id, - StoredWorkflowMenuEntry.user_id == user_id, - ).count()) + return bool( + sa_session.query(StoredWorkflowMenuEntry) + .filter( + StoredWorkflowMenuEntry.stored_workflow_id == self.id, + StoredWorkflowMenuEntry.user_id == user_id, + ) + .count() + ) def copy_tags_from(self, target_user, source_workflow): # Override to only copy owner tags. @@ -6218,9 +6579,11 @@ class StoredWorkflow(Base, HasTags, Dictifiable, RepresentById): new_swta.user = target_user self.tags.append(new_swta) - def to_dict(self, view='collection', value_mapper=None): + def to_dict(self, view="collection", value_mapper=None): rval = super().to_dict(view=view, value_mapper=value_mapper) - rval['latest_workflow_uuid'] = (lambda uuid: str(uuid) if self.latest_workflow.uuid else None)(self.latest_workflow.uuid) + rval["latest_workflow_uuid"] = (lambda uuid: str(uuid) if self.latest_workflow.uuid else None)( + self.latest_workflow.uuid + ) return rval @@ -6231,14 +6594,15 @@ class Workflow(Base, Dictifiable, RepresentById): See :class:`galaxy.model.WorkflowStep` for more information """ - __tablename__ = 'workflow' + + __tablename__ = "workflow" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) # workflows will belong to either a stored workflow or a parent/nesting workflow. - stored_workflow_id = Column(Integer, ForeignKey('stored_workflow.id'), index=True, nullable=True) - parent_workflow_id = Column(Integer, ForeignKey('workflow.id'), index=True, nullable=True) + stored_workflow_id = Column(Integer, ForeignKey("stored_workflow.id"), index=True, nullable=True) + parent_workflow_id = Column(Integer, ForeignKey("workflow.id"), index=True, nullable=True) name = Column(TEXT) has_cycles = Column(Boolean) has_errors = Column(Boolean) @@ -6247,25 +6611,30 @@ class Workflow(Base, Dictifiable, RepresentById): license = Column(TEXT) uuid = Column(UUIDType, nullable=True) - steps = relationship('WorkflowStep', - back_populates='workflow', + steps = relationship( + "WorkflowStep", + back_populates="workflow", primaryjoin=(lambda: Workflow.id == WorkflowStep.workflow_id), # type: ignore[has-type] order_by=lambda: asc(WorkflowStep.order_index), # type: ignore[has-type] cascade="all, delete-orphan", - lazy=False) + lazy=False, + ) parent_workflow_steps = relationship( - 'WorkflowStep', + "WorkflowStep", primaryjoin=(lambda: Workflow.id == WorkflowStep.subworkflow_id), # type: ignore[has-type] - back_populates='subworkflow') - stored_workflow = relationship('StoredWorkflow', + back_populates="subworkflow", + ) + stored_workflow = relationship( + "StoredWorkflow", primaryjoin=(lambda: StoredWorkflow.id == Workflow.stored_workflow_id), - back_populates='workflows') + back_populates="workflows", + ) step_count: column_property - dict_collection_visible_keys = ['name', 'has_cycles', 'has_errors'] - dict_element_visible_keys = ['name', 'has_cycles', 'has_errors'] - input_step_types = ['data_input', 'data_collection_input', 'parameter_input'] + dict_collection_visible_keys = ["name", "has_cycles", "has_errors"] + dict_element_visible_keys = ["name", "has_cycles", "has_errors"] + input_step_types = ["data_input", "data_collection_input", "parameter_input"] def __init__(self, uuid=None): self.user = None @@ -6280,9 +6649,9 @@ class Workflow(Base, Dictifiable, RepresentById): return True return False - def to_dict(self, view='collection', value_mapper=None): + def to_dict(self, view="collection", value_mapper=None): rval = super().to_dict(view=view, value_mapper=value_mapper) - rval['uuid'] = (lambda uuid: str(uuid) if uuid else None)(self.uuid) + rval["uuid"] = (lambda uuid: str(uuid) if uuid else None)(self.uuid) return rval @property @@ -6333,7 +6702,7 @@ class Workflow(Base, Dictifiable, RepresentById): @property def top_level_workflow(self): - """ If this workflow is not attached to stored workflow directly, + """If this workflow is not attached to stored workflow directly, recursively grab its parents until it is the top level workflow which must have a stored workflow associated with it. """ @@ -6346,7 +6715,7 @@ class Workflow(Base, Dictifiable, RepresentById): @property def top_level_stored_workflow(self): - """ If this workflow is not attached to stored workflow directly, + """If this workflow is not attached to stored workflow directly, recursively grab its parents until it is the top level workflow which must have a stored workflow associated with it and then grab that stored workflow. @@ -6354,7 +6723,7 @@ class Workflow(Base, Dictifiable, RepresentById): return self.top_level_workflow.stored_workflow def copy(self, user=None): - """ Copy a workflow for a new StoredWorkflow object. + """Copy a workflow for a new StoredWorkflow object. Pass user if user-specific information needed. """ @@ -6395,14 +6764,15 @@ class WorkflowStep(Base, RepresentById): See :class:`galaxy.model.WorkflowStepInput` and :class:`galaxy.model.WorkflowStepConnection` for more information. """ - __tablename__ = 'workflow_step' + + __tablename__ = "workflow_step" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - workflow_id = Column(Integer, ForeignKey('workflow.id'), index=True, nullable=False) - subworkflow_id = Column(Integer, ForeignKey('workflow.id'), index=True, nullable=True) - dynamic_tool_id = Column(Integer, ForeignKey('dynamic_tool.id'), index=True, nullable=True) + workflow_id = Column(Integer, ForeignKey("workflow.id"), index=True, nullable=False) + subworkflow_id = Column(Integer, ForeignKey("workflow.id"), index=True, nullable=True) + dynamic_tool_id = Column(Integer, ForeignKey("dynamic_tool.id"), index=True, nullable=True) type = Column(String(64)) tool_id = Column(TEXT) tool_version = Column(TEXT) @@ -6415,26 +6785,28 @@ class WorkflowStep(Base, RepresentById): label = Column(Unicode(255)) temp_input_connections: Optional[InputConnDictType] - subworkflow = relationship('Workflow', + subworkflow = relationship( + "Workflow", primaryjoin=(lambda: Workflow.id == WorkflowStep.subworkflow_id), - back_populates='parent_workflow_steps') - dynamic_tool = relationship('DynamicTool', - primaryjoin=(lambda: DynamicTool.id == WorkflowStep.dynamic_tool_id)) - tags = relationship('WorkflowStepTagAssociation', - order_by=lambda: WorkflowStepTagAssociation.id, - back_populates='workflow_step') - annotations = relationship('WorkflowStepAnnotationAssociation', - order_by=lambda: WorkflowStepAnnotationAssociation.id, - back_populates="workflow_step") - post_job_actions = relationship('PostJobAction', back_populates='workflow_step') - inputs = relationship('WorkflowStepInput', back_populates='workflow_step') - workflow_outputs = relationship('WorkflowOutput', back_populates='workflow_step') - output_connections = relationship('WorkflowStepConnection', - primaryjoin=(lambda: WorkflowStepConnection.output_step_id == WorkflowStep.id) + back_populates="parent_workflow_steps", ) - workflow = relationship('Workflow', - primaryjoin=(lambda: Workflow.id == WorkflowStep.workflow_id), - back_populates='steps' + dynamic_tool = relationship("DynamicTool", primaryjoin=(lambda: DynamicTool.id == WorkflowStep.dynamic_tool_id)) + tags = relationship( + "WorkflowStepTagAssociation", order_by=lambda: WorkflowStepTagAssociation.id, back_populates="workflow_step" + ) + annotations = relationship( + "WorkflowStepAnnotationAssociation", + order_by=lambda: WorkflowStepAnnotationAssociation.id, + back_populates="workflow_step", + ) + post_job_actions = relationship("PostJobAction", back_populates="workflow_step") + inputs = relationship("WorkflowStepInput", back_populates="workflow_step") + workflow_outputs = relationship("WorkflowOutput", back_populates="workflow_step") + output_connections = relationship( + "WorkflowStepConnection", primaryjoin=(lambda: WorkflowStepConnection.output_step_id == WorkflowStep.id) + ) + workflow = relationship( + "Workflow", primaryjoin=(lambda: Workflow.id == WorkflowStep.workflow_id), back_populates="steps" ) STEP_TYPE_TO_INPUT_TYPE = { @@ -6454,7 +6826,9 @@ class WorkflowStep(Base, RepresentById): @property def input_type(self): - assert self.type and self.type in self.STEP_TYPE_TO_INPUT_TYPE, "step.input_type can only be called on input step types" + assert ( + self.type and self.type in self.STEP_TYPE_TO_INPUT_TYPE + ), "step.input_type can only be called on input step types" return self.STEP_TYPE_TO_INPUT_TYPE[self.type] @property @@ -6598,10 +6972,12 @@ class WorkflowStep(Base, RepresentById): if old_conn.input_subworkflow_step_id: new_conn.input_subworkflow_step = subworkflow_step_mapping[old_conn.input_subworkflow_step_id] for orig_pja in self.post_job_actions: - PostJobAction(orig_pja.action_type, - copied_step, - output_name=orig_pja.output_name, - action_arguments=orig_pja.action_arguments) + PostJobAction( + orig_pja.action_type, + copied_step, + output_name=orig_pja.output_name, + action_arguments=orig_pja.action_arguments, + ) copied_step.workflow_outputs = copy_list(self.workflow_outputs, copied_step) def log_str(self): @@ -6618,14 +6994,19 @@ class WorkflowStep(Base, RepresentById): class WorkflowStepInput(Base, RepresentById): - __tablename__ = 'workflow_step_input' + __tablename__ = "workflow_step_input" __table_args__ = ( - Index('ix_workflow_step_input_workflow_step_id_name_unique', - 'workflow_step_id', 'name', unique=True, mysql_length={'name': 200}), + Index( + "ix_workflow_step_input_workflow_step_id_name_unique", + "workflow_step_id", + "name", + unique=True, + mysql_length={"name": 200}, + ), ) id = Column(Integer, primary_key=True) - workflow_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id"), index=True) name = Column(TEXT) merge_type = Column(TEXT) scatter_type = Column(TEXT) @@ -6635,14 +7016,17 @@ class WorkflowStepInput(Base, RepresentById): default_value_set = Column(Boolean, default=False) runtime_value = Column(Boolean, default=False) - workflow_step = relationship('WorkflowStep', - back_populates='inputs', - cascade='all', - primaryjoin=(lambda: WorkflowStepInput.workflow_step_id == WorkflowStep.id)) + workflow_step = relationship( + "WorkflowStep", + back_populates="inputs", + cascade="all", + primaryjoin=(lambda: WorkflowStepInput.workflow_step_id == WorkflowStep.id), + ) connections = relationship( - 'WorkflowStepConnection', - back_populates='input_step_input', - primaryjoin=(lambda: WorkflowStepConnection.input_step_input_id == WorkflowStepInput.id)) + "WorkflowStepConnection", + back_populates="input_step_input", + primaryjoin=(lambda: WorkflowStepConnection.input_step_input_id == WorkflowStepInput.id), + ) def __init__(self, workflow_step): self.workflow_step = workflow_step @@ -6661,24 +7045,29 @@ class WorkflowStepInput(Base, RepresentById): class WorkflowStepConnection(Base, RepresentById): - __tablename__ = 'workflow_step_connection' + __tablename__ = "workflow_step_connection" id = Column(Integer, primary_key=True) - output_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True) - input_step_input_id = Column(Integer, ForeignKey('workflow_step_input.id'), index=True) + output_step_id = Column(Integer, ForeignKey("workflow_step.id"), index=True) + input_step_input_id = Column(Integer, ForeignKey("workflow_step_input.id"), index=True) output_name = Column(TEXT) - input_subworkflow_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True) + input_subworkflow_step_id = Column(Integer, ForeignKey("workflow_step.id"), index=True) - input_step_input = relationship('WorkflowStepInput', - back_populates='connections', - cascade='all', - primaryjoin=(lambda: WorkflowStepConnection.input_step_input_id == WorkflowStepInput.id)) - input_subworkflow_step = relationship('WorkflowStep', - primaryjoin=(lambda: WorkflowStepConnection.input_subworkflow_step_id == WorkflowStep.id)) - output_step = relationship('WorkflowStep', - back_populates='output_connections', - cascade='all', - primaryjoin=(lambda: WorkflowStepConnection.output_step_id == WorkflowStep.id)) + input_step_input = relationship( + "WorkflowStepInput", + back_populates="connections", + cascade="all", + primaryjoin=(lambda: WorkflowStepConnection.input_step_input_id == WorkflowStepInput.id), + ) + input_subworkflow_step = relationship( + "WorkflowStep", primaryjoin=(lambda: WorkflowStepConnection.input_subworkflow_step_id == WorkflowStep.id) + ) + output_step = relationship( + "WorkflowStep", + back_populates="output_connections", + cascade="all", + primaryjoin=(lambda: WorkflowStepConnection.output_step_id == WorkflowStep.id), + ) # Constant used in lieu of output_name and input_name to indicate an # implicit connection between two steps that is not dependent on a dataset @@ -6689,7 +7078,7 @@ class WorkflowStepConnection(Base, RepresentById): @property def non_data_connection(self): - return (self.output_name == self.input_name == WorkflowStepConnection.NON_DATA_CONNECTION) + return self.output_name == self.input_name == WorkflowStepConnection.NON_DATA_CONNECTION @property def input_name(self): @@ -6712,16 +7101,18 @@ class WorkflowStepConnection(Base, RepresentById): class WorkflowOutput(Base, RepresentById): - __tablename__ = 'workflow_output' + __tablename__ = "workflow_output" id = Column(Integer, primary_key=True) - workflow_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True, nullable=False) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id"), index=True, nullable=False) output_name = Column(String(255), nullable=True) label = Column(Unicode(255)) uuid = Column(UUIDType) - workflow_step = relationship('WorkflowStep', - back_populates='workflow_outputs', - primaryjoin=(lambda: WorkflowStep.id == WorkflowOutput.workflow_step_id)) + workflow_step = relationship( + "WorkflowStep", + back_populates="workflow_outputs", + primaryjoin=(lambda: WorkflowStep.id == WorkflowOutput.workflow_step_id), + ) def __init__(self, workflow_step, output_name=None, label=None, uuid=None): self.workflow_step = workflow_step @@ -6737,79 +7128,81 @@ class WorkflowOutput(Base, RepresentById): class StoredWorkflowUserShareAssociation(Base, UserShareAssociation): - __tablename__ = 'stored_workflow_user_share_connection' + __tablename__ = "stored_workflow_user_share_connection" id = Column(Integer, primary_key=True) - stored_workflow_id = Column(Integer, ForeignKey('stored_workflow.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) - user = relationship('User') - stored_workflow = relationship('StoredWorkflow', back_populates='users_shared_with') + stored_workflow_id = Column(Integer, ForeignKey("stored_workflow.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) + user = relationship("User") + stored_workflow = relationship("StoredWorkflow", back_populates="users_shared_with") class StoredWorkflowMenuEntry(Base, RepresentById): - __tablename__ = 'stored_workflow_menu_entry' + __tablename__ = "stored_workflow_menu_entry" id = Column(Integer, primary_key=True) - stored_workflow_id = Column(Integer, ForeignKey('stored_workflow.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + stored_workflow_id = Column(Integer, ForeignKey("stored_workflow.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) order_index = Column(Integer) - stored_workflow = relationship('StoredWorkflow') - user = relationship('User', back_populates='stored_workflow_menu_entries', - primaryjoin=(lambda: - (StoredWorkflowMenuEntry.user_id == User.id) + stored_workflow = relationship("StoredWorkflow") + user = relationship( + "User", + back_populates="stored_workflow_menu_entries", + primaryjoin=( + lambda: (StoredWorkflowMenuEntry.user_id == User.id) & (StoredWorkflowMenuEntry.stored_workflow_id == StoredWorkflow.id) - & not_(StoredWorkflow.deleted)) + & not_(StoredWorkflow.deleted) + ), ) class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentById): - __tablename__ = 'workflow_invocation' + __tablename__ = "workflow_invocation" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now, index=True) - workflow_id = Column(Integer, ForeignKey('workflow.id'), index=True, nullable=False) + workflow_id = Column(Integer, ForeignKey("workflow.id"), index=True, nullable=False) state = Column(TrimmedString(64), index=True) scheduler = Column(TrimmedString(255), index=True) handler = Column(TrimmedString(255), index=True) uuid = Column(UUIDType()) - history_id = Column(Integer, ForeignKey('history.id'), index=True) + history_id = Column(Integer, ForeignKey("history.id"), index=True) - history = relationship('History', back_populates='workflow_invocations') - input_parameters = relationship('WorkflowRequestInputParameter', - back_populates='workflow_invocation') - step_states = relationship('WorkflowRequestStepState', back_populates='workflow_invocation') - input_step_parameters = relationship('WorkflowRequestInputStepParameter', - back_populates='workflow_invocation') - input_datasets = relationship('WorkflowRequestToInputDatasetAssociation', - back_populates='workflow_invocation') - input_dataset_collections = relationship('WorkflowRequestToInputDatasetCollectionAssociation', - back_populates='workflow_invocation') - subworkflow_invocations = relationship('WorkflowInvocationToSubworkflowInvocationAssociation', - primaryjoin=(lambda: - WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id - == WorkflowInvocation.id), - back_populates='parent_workflow_invocation', + history = relationship("History", back_populates="workflow_invocations") + input_parameters = relationship("WorkflowRequestInputParameter", back_populates="workflow_invocation") + step_states = relationship("WorkflowRequestStepState", back_populates="workflow_invocation") + input_step_parameters = relationship("WorkflowRequestInputStepParameter", back_populates="workflow_invocation") + input_datasets = relationship("WorkflowRequestToInputDatasetAssociation", back_populates="workflow_invocation") + input_dataset_collections = relationship( + "WorkflowRequestToInputDatasetCollectionAssociation", back_populates="workflow_invocation" + ) + subworkflow_invocations = relationship( + "WorkflowInvocationToSubworkflowInvocationAssociation", + primaryjoin=( + lambda: WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id == WorkflowInvocation.id + ), + back_populates="parent_workflow_invocation", uselist=True, ) - steps = relationship('WorkflowInvocationStep', back_populates='workflow_invocation') - workflow = relationship('Workflow') - output_dataset_collections = relationship('WorkflowInvocationOutputDatasetCollectionAssociation', - back_populates='workflow_invocation') - output_datasets = relationship('WorkflowInvocationOutputDatasetAssociation', - back_populates='workflow_invocation') - output_values = relationship('WorkflowInvocationOutputValue', back_populates='workflow_invocation') + steps = relationship("WorkflowInvocationStep", back_populates="workflow_invocation") + workflow = relationship("Workflow") + output_dataset_collections = relationship( + "WorkflowInvocationOutputDatasetCollectionAssociation", back_populates="workflow_invocation" + ) + output_datasets = relationship("WorkflowInvocationOutputDatasetAssociation", back_populates="workflow_invocation") + output_values = relationship("WorkflowInvocationOutputValue", back_populates="workflow_invocation") - dict_collection_visible_keys = ['id', 'update_time', 'create_time', 'workflow_id', 'history_id', 'uuid', 'state'] - dict_element_visible_keys = ['id', 'update_time', 'create_time', 'workflow_id', 'history_id', 'uuid', 'state'] + dict_collection_visible_keys = ["id", "update_time", "create_time", "workflow_id", "history_id", "uuid", "state"] + dict_element_visible_keys = ["id", "update_time", "create_time", "workflow_id", "history_id", "uuid", "state"] class states(str, Enum): - NEW = 'new' # Brand new workflow invocation... maybe this should be same as READY - READY = 'ready' # Workflow ready for another iteration of scheduling. - SCHEDULED = 'scheduled' # Workflow has been scheduled. - CANCELLED = 'cancelled' - FAILED = 'failed' + NEW = "new" # Brand new workflow invocation... maybe this should be same as READY + READY = "ready" # Workflow ready for another iteration of scheduling. + SCHEDULED = "scheduled" # Workflow has been scheduled. + CANCELLED = "cancelled" + FAILED = "failed" non_terminal_states = [states.NEW, states.READY] @@ -6845,7 +7238,7 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentBy @property def active(self): - """ Indicates the workflow invocation is somehow active - and in + """Indicates the workflow invocation is somehow active - and in particular valid actions may be performed on its WorkflowInvocationSteps. """ @@ -6895,23 +7288,21 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentBy def poll_unhandled_workflow_ids(sa_session): and_conditions = [ WorkflowInvocation.state == WorkflowInvocation.states.NEW, - WorkflowInvocation.handler.is_(None) + WorkflowInvocation.handler.is_(None), ] - query = sa_session.query( - WorkflowInvocation.id - ).filter(and_(*and_conditions)).order_by(WorkflowInvocation.table.c.id.asc()) + query = ( + sa_session.query(WorkflowInvocation.id) + .filter(and_(*and_conditions)) + .order_by(WorkflowInvocation.table.c.id.asc()) + ) return [wid for wid in query.all()] @staticmethod - def poll_active_workflow_ids( - sa_session, - scheduler=None, - handler=None - ): + def poll_active_workflow_ids(sa_session, scheduler=None, handler=None): and_conditions = [ or_( WorkflowInvocation.state == WorkflowInvocation.states.NEW, - WorkflowInvocation.state == WorkflowInvocation.states.READY + WorkflowInvocation.state == WorkflowInvocation.states.READY, ), ] if scheduler is not None: @@ -6919,9 +7310,11 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentBy if handler is not None: and_conditions.append(WorkflowInvocation.handler == handler) - query = sa_session.query( - WorkflowInvocation.id - ).filter(and_(*and_conditions)).order_by(WorkflowInvocation.table.c.id.asc()) + query = ( + sa_session.query(WorkflowInvocation.id) + .filter(and_(*and_conditions)) + .order_by(WorkflowInvocation.table.c.id.asc()) + ) # Immediately just load all ids into memory so time slicing logic # is relatively intutitive. return [wid for wid in query.all()] @@ -6964,9 +7357,13 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentBy # That probably isn't good. workflow_output = self.workflow.workflow_output_for(label) if workflow_output: - raise Exception(f"Failed to find workflow output named [{label}], one was defined but none registered during execution.") + raise Exception( + f"Failed to find workflow output named [{label}], one was defined but none registered during execution." + ) else: - raise Exception(f"Failed to find workflow output named [{label}], workflow doesn't define output by that name - valid names are {self.workflow.workflow_output_labels}.") + raise Exception( + f"Failed to find workflow output named [{label}], workflow doesn't define output by that name - valid names are {self.workflow.workflow_output_labels}." + ) def get_input_object(self, label): for input_dataset_assoc in self.input_datasets: @@ -6995,15 +7392,15 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentBy inputs.append(input_dataset_collection_assoc) return inputs - def to_dict(self, view='collection', value_mapper=None, step_details=False, legacy_job_state=False): + def to_dict(self, view="collection", value_mapper=None, step_details=False, legacy_job_state=False): rval = super().to_dict(view=view, value_mapper=value_mapper) - if view == 'element': + if view == "element": steps = [] for step in self.steps: if step_details: - v = step.to_dict(view='element') + v = step.to_dict(view="element") else: - v = step.to_dict(view='collection') + v = step.to_dict(view="collection") if legacy_job_state: step_jobs = step.jobs if step_jobs: @@ -7017,28 +7414,28 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentBy steps.append(v) else: steps.append(v) - rval['steps'] = steps + rval["steps"] = steps inputs = {} for input_item_association in self.input_datasets + self.input_dataset_collections: - if input_item_association.history_content_type == 'dataset': - src = 'hda' + if input_item_association.history_content_type == "dataset": + src = "hda" item = input_item_association.dataset - elif input_item_association.history_content_type == 'dataset_collection': - src = 'hdca' + elif input_item_association.history_content_type == "dataset_collection": + src = "hdca" item = input_item_association.dataset_collection else: # TODO: LDDAs are not implemented in workflow_request_to_input_dataset table raise Exception(f"Unknown history content type '{input_item_association.history_content_type}'") # Should this maybe also be by label ? Would break backwards compatibility though inputs[str(input_item_association.workflow_step.order_index)] = { - 'id': item.id, - 'src': src, - 'label': input_item_association.workflow_step.label, - 'workflow_step_id': input_item_association.workflow_step_id, + "id": item.id, + "src": src, + "label": input_item_association.workflow_step.label, + "workflow_step_id": input_item_association.workflow_step_id, } - rval['inputs'] = inputs + rval["inputs"] = inputs input_parameters = {} for input_step_parameter in self.input_step_parameters: @@ -7046,11 +7443,11 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentBy if not label: continue input_parameters[label] = { - 'parameter_value': input_step_parameter.parameter_value, - 'label': label, - 'workflow_step_id': input_step_parameter.workflow_step_id, + "parameter_value": input_step_parameter.parameter_value, + "label": label, + "workflow_step_id": input_step_parameter.workflow_step_id, } - rval['input_step_parameters'] = input_parameters + rval["input_step_parameters"] = input_parameters outputs = {} for output_assoc in self.output_datasets: @@ -7060,9 +7457,9 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentBy continue outputs[label] = { - 'src': 'hda', - 'id': output_assoc.dataset_id, - 'workflow_step_id': output_assoc.workflow_step_id, + "src": "hda", + "id": output_assoc.dataset_id, + "workflow_step_id": output_assoc.workflow_step_id, } output_collections = {} @@ -7072,13 +7469,13 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentBy continue output_collections[label] = { - 'src': 'hdca', - 'id': output_assoc.dataset_collection_id, - 'workflow_step_id': output_assoc.workflow_step_id, + "src": "hdca", + "id": output_assoc.dataset_collection_id, + "workflow_step_id": output_assoc.workflow_step_id, } - rval['outputs'] = outputs - rval['output_collections'] = output_collections + rval["outputs"] = outputs + rval["output_collections"] = output_collections output_values = {} for output_param in self.output_values: @@ -7086,7 +7483,7 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentBy if not label: continue output_values[label] = output_param.value - rval['output_values'] = output_values + rval["output_values"] = output_values return rval @@ -7149,78 +7546,95 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentBy class WorkflowInvocationToSubworkflowInvocationAssociation(Base, Dictifiable, RepresentById): - __tablename__ = 'workflow_invocation_to_subworkflow_invocation_association' + __tablename__ = "workflow_invocation_to_subworkflow_invocation_association" id = Column(Integer, primary_key=True) - workflow_invocation_id = Column(Integer, - ForeignKey('workflow_invocation.id', name='fk_wfi_swi_wfi'), index=True) - subworkflow_invocation_id = Column(Integer, - ForeignKey('workflow_invocation.id', name='fk_wfi_swi_swi'), index=True) - workflow_step_id = Column(Integer, ForeignKey('workflow_step.id', name='fk_wfi_swi_ws')) + workflow_invocation_id = Column(Integer, ForeignKey("workflow_invocation.id", name="fk_wfi_swi_wfi"), index=True) + subworkflow_invocation_id = Column(Integer, ForeignKey("workflow_invocation.id", name="fk_wfi_swi_swi"), index=True) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id", name="fk_wfi_swi_ws")) - subworkflow_invocation = relationship('WorkflowInvocation', - primaryjoin=(lambda: - WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id - == WorkflowInvocation.id), + subworkflow_invocation = relationship( + "WorkflowInvocation", + primaryjoin=( + lambda: WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id + == WorkflowInvocation.id + ), uselist=False, ) - workflow_step = relationship('WorkflowStep') - parent_workflow_invocation = relationship('WorkflowInvocation', - primaryjoin=(lambda: - WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id - == WorkflowInvocation.id), - back_populates='subworkflow_invocations', + workflow_step = relationship("WorkflowStep") + parent_workflow_invocation = relationship( + "WorkflowInvocation", + primaryjoin=( + lambda: WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id == WorkflowInvocation.id + ), + back_populates="subworkflow_invocations", uselist=False, ) - dict_collection_visible_keys = ['id', 'workflow_step_id', 'workflow_invocation_id', 'subworkflow_invocation_id'] - dict_element_visible_keys = ['id', 'workflow_step_id', 'workflow_invocation_id', 'subworkflow_invocation_id'] + dict_collection_visible_keys = ["id", "workflow_step_id", "workflow_invocation_id", "subworkflow_invocation_id"] + dict_element_visible_keys = ["id", "workflow_step_id", "workflow_invocation_id", "subworkflow_invocation_id"] class WorkflowInvocationStep(Base, Dictifiable, RepresentById): - __tablename__ = 'workflow_invocation_step' + __tablename__ = "workflow_invocation_step" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - workflow_invocation_id = Column(Integer, - ForeignKey('workflow_invocation.id'), index=True, nullable=False) - workflow_step_id = Column(Integer, - ForeignKey('workflow_step.id'), index=True, nullable=False) + workflow_invocation_id = Column(Integer, ForeignKey("workflow_invocation.id"), index=True, nullable=False) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id"), index=True, nullable=False) state = Column(TrimmedString(64), index=True) - job_id = Column(Integer, ForeignKey('job.id'), index=True, nullable=True) - implicit_collection_jobs_id = Column(Integer, - ForeignKey('implicit_collection_jobs.id'), index=True, nullable=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True, nullable=True) + implicit_collection_jobs_id = Column(Integer, ForeignKey("implicit_collection_jobs.id"), index=True, nullable=True) action = Column(MutableJSONType, nullable=True) - workflow_step = relationship('WorkflowStep') - job = relationship('Job', back_populates='workflow_invocation_step', uselist=False) - implicit_collection_jobs = relationship('ImplicitCollectionJobs', uselist=False) + workflow_step = relationship("WorkflowStep") + job = relationship("Job", back_populates="workflow_invocation_step", uselist=False) + implicit_collection_jobs = relationship("ImplicitCollectionJobs", uselist=False) output_dataset_collections = relationship( - 'WorkflowInvocationStepOutputDatasetCollectionAssociation', - back_populates='workflow_invocation_step') - output_datasets = relationship('WorkflowInvocationStepOutputDatasetAssociation', - back_populates='workflow_invocation_step') - workflow_invocation = relationship('WorkflowInvocation', back_populates='steps') - output_value = relationship('WorkflowInvocationOutputValue', - foreign_keys='[WorkflowInvocationStep.workflow_invocation_id, WorkflowInvocationStep.workflow_step_id]', - primaryjoin=(lambda: and_( - WorkflowInvocationStep.workflow_invocation_id - == WorkflowInvocationOutputValue.workflow_invocation_id, - WorkflowInvocationStep.workflow_step_id == WorkflowInvocationOutputValue.workflow_step_id, - )), - back_populates='workflow_invocation_step', - viewonly=True + "WorkflowInvocationStepOutputDatasetCollectionAssociation", back_populates="workflow_invocation_step" + ) + output_datasets = relationship( + "WorkflowInvocationStepOutputDatasetAssociation", back_populates="workflow_invocation_step" + ) + workflow_invocation = relationship("WorkflowInvocation", back_populates="steps") + output_value = relationship( + "WorkflowInvocationOutputValue", + foreign_keys="[WorkflowInvocationStep.workflow_invocation_id, WorkflowInvocationStep.workflow_step_id]", + primaryjoin=( + lambda: and_( + WorkflowInvocationStep.workflow_invocation_id == WorkflowInvocationOutputValue.workflow_invocation_id, + WorkflowInvocationStep.workflow_step_id == WorkflowInvocationOutputValue.workflow_step_id, + ) + ), + back_populates="workflow_invocation_step", + viewonly=True, ) subworkflow_invocation_id: column_property - dict_collection_visible_keys = ['id', 'update_time', 'job_id', 'workflow_step_id', 'subworkflow_invocation_id', 'state', 'action'] - dict_element_visible_keys = ['id', 'update_time', 'job_id', 'workflow_step_id', 'subworkflow_invocation_id', 'state', 'action'] + dict_collection_visible_keys = [ + "id", + "update_time", + "job_id", + "workflow_step_id", + "subworkflow_invocation_id", + "state", + "action", + ] + dict_element_visible_keys = [ + "id", + "update_time", + "job_id", + "workflow_step_id", + "subworkflow_invocation_id", + "state", + "action", + ] class states(str, Enum): - NEW = 'new' # Brand new workflow invocation step - READY = 'ready' # Workflow invocation step ready for another iteration of scheduling. - SCHEDULED = 'scheduled' # Workflow invocation step has been scheduled. + NEW = "new" # Brand new workflow invocation step + READY = "ready" # Workflow invocation step ready for another iteration of scheduling. + SCHEDULED = "scheduled" # Workflow invocation step has been scheduled. # CANCELLED = 'cancelled', TODO: implement and expose # FAILED = 'failed', TODO: implement and expose @@ -7253,14 +7667,14 @@ class WorkflowInvocationStep(Base, Dictifiable, RepresentById): else: return [] - def to_dict(self, view='collection', value_mapper=None): + def to_dict(self, view="collection", value_mapper=None): rval = super().to_dict(view=view, value_mapper=value_mapper) - rval['order_index'] = self.workflow_step.order_index - rval['workflow_step_label'] = self.workflow_step.label - rval['workflow_step_uuid'] = str(self.workflow_step.uuid) + rval["order_index"] = self.workflow_step.order_index + rval["workflow_step_label"] = self.workflow_step.label + rval["workflow_step_uuid"] = str(self.workflow_step.uuid) # Following no longer makes sense... # rval['state'] = self.job.state if self.job is not None else None - if view == 'element': + if view == "element": jobs = [] for job in self.jobs: jobs.append(job.to_dict()) @@ -7269,67 +7683,71 @@ class WorkflowInvocationStep(Base, Dictifiable, RepresentById): for output_assoc in self.output_datasets: name = output_assoc.output_name outputs[name] = { - 'src': 'hda', - 'id': output_assoc.dataset.id, - 'uuid': str(output_assoc.dataset.dataset.uuid) if output_assoc.dataset.dataset.uuid is not None else None + "src": "hda", + "id": output_assoc.dataset.id, + "uuid": str(output_assoc.dataset.dataset.uuid) + if output_assoc.dataset.dataset.uuid is not None + else None, } output_collections = {} for output_assoc in self.output_dataset_collections: name = output_assoc.output_name output_collections[name] = { - 'src': 'hdca', - 'id': output_assoc.dataset_collection.id, + "src": "hdca", + "id": output_assoc.dataset_collection.id, } - rval['outputs'] = outputs - rval['output_collections'] = output_collections - rval['jobs'] = jobs + rval["outputs"] = outputs + rval["output_collections"] = output_collections + rval["jobs"] = jobs return rval class WorkflowRequestInputParameter(Base, Dictifiable, RepresentById): - """ Workflow-related parameters not tied to steps or inputs. - """ - __tablename__ = 'workflow_request_input_parameters' + """Workflow-related parameters not tied to steps or inputs.""" + + __tablename__ = "workflow_request_input_parameters" id = Column(Integer, primary_key=True) workflow_invocation_id = Column( - Integer, ForeignKey('workflow_invocation.id', onupdate='CASCADE', ondelete='CASCADE')) + Integer, ForeignKey("workflow_invocation.id", onupdate="CASCADE", ondelete="CASCADE") + ) name = Column(Unicode(255)) value = Column(TEXT) type = Column(Unicode(255)) - workflow_invocation = relationship('WorkflowInvocation', back_populates='input_parameters') + workflow_invocation = relationship("WorkflowInvocation", back_populates="input_parameters") - dict_collection_visible_keys = ['id', 'name', 'value', 'type'] + dict_collection_visible_keys = ["id", "name", "value", "type"] class types(str, Enum): - REPLACEMENT_PARAMETERS = 'replacements' - STEP_PARAMETERS = 'step' - META_PARAMETERS = 'meta' - RESOURCE_PARAMETERS = 'resource' + REPLACEMENT_PARAMETERS = "replacements" + STEP_PARAMETERS = "step" + META_PARAMETERS = "meta" + RESOURCE_PARAMETERS = "resource" class WorkflowRequestStepState(Base, Dictifiable, RepresentById): - """ Workflow step value parameters. - """ - __tablename__ = 'workflow_request_step_states' + """Workflow step value parameters.""" + + __tablename__ = "workflow_request_step_states" id = Column(Integer, primary_key=True) - workflow_invocation_id = Column(Integer, - ForeignKey('workflow_invocation.id', onupdate='CASCADE', ondelete='CASCADE')) - workflow_step_id = Column(Integer, ForeignKey('workflow_step.id')) + workflow_invocation_id = Column( + Integer, ForeignKey("workflow_invocation.id", onupdate="CASCADE", ondelete="CASCADE") + ) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id")) value = Column(MutableJSONType) - workflow_step = relationship('WorkflowStep') - workflow_invocation = relationship('WorkflowInvocation', back_populates='step_states') + workflow_step = relationship("WorkflowStep") + workflow_invocation = relationship("WorkflowInvocation", back_populates="step_states") - dict_collection_visible_keys = ['id', 'name', 'value', 'workflow_step_id'] + dict_collection_visible_keys = ["id", "name", "value", "workflow_step_id"] class WorkflowRequestToInputDatasetAssociation(Base, Dictifiable, RepresentById): - """ Workflow step input dataset parameters. - """ - __tablename__ = 'workflow_request_to_input_dataset' + """Workflow step input dataset parameters.""" + + __tablename__ = "workflow_request_to_input_dataset" id = Column(Integer, primary_key=True) name = Column(String(255)) @@ -7337,167 +7755,165 @@ class WorkflowRequestToInputDatasetAssociation(Base, Dictifiable, RepresentById) workflow_step_id = Column(Integer, ForeignKey("workflow_step.id")) dataset_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) - workflow_step = relationship('WorkflowStep') - dataset = relationship('HistoryDatasetAssociation') - workflow_invocation = relationship('WorkflowInvocation', back_populates='input_datasets') + workflow_step = relationship("WorkflowStep") + dataset = relationship("HistoryDatasetAssociation") + workflow_invocation = relationship("WorkflowInvocation", back_populates="input_datasets") history_content_type = "dataset" - dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_id', 'name'] + dict_collection_visible_keys = ["id", "workflow_invocation_id", "workflow_step_id", "dataset_id", "name"] class WorkflowRequestToInputDatasetCollectionAssociation(Base, Dictifiable, RepresentById): - """ Workflow step input dataset collection parameters. - """ - __tablename__ = 'workflow_request_to_input_collection_dataset' + """Workflow step input dataset collection parameters.""" + + __tablename__ = "workflow_request_to_input_collection_dataset" id = Column(Integer, primary_key=True) name = Column(String(255)) - workflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id'), index=True) - workflow_step_id = Column(Integer, ForeignKey('workflow_step.id')) - dataset_collection_id = Column(Integer, - ForeignKey('history_dataset_collection_association.id'), index=True) - workflow_step = relationship('WorkflowStep') - dataset_collection = relationship('HistoryDatasetCollectionAssociation') - workflow_invocation = relationship('WorkflowInvocation', - back_populates='input_dataset_collections') + workflow_invocation_id = Column(Integer, ForeignKey("workflow_invocation.id"), index=True) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id")) + dataset_collection_id = Column(Integer, ForeignKey("history_dataset_collection_association.id"), index=True) + workflow_step = relationship("WorkflowStep") + dataset_collection = relationship("HistoryDatasetCollectionAssociation") + workflow_invocation = relationship("WorkflowInvocation", back_populates="input_dataset_collections") history_content_type = "dataset_collection" - dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_collection_id', 'name'] + dict_collection_visible_keys = ["id", "workflow_invocation_id", "workflow_step_id", "dataset_collection_id", "name"] class WorkflowRequestInputStepParameter(Base, Dictifiable, RepresentById): - """ Workflow step parameter inputs. - """ - __tablename__ = 'workflow_request_input_step_parameter' + """Workflow step parameter inputs.""" + + __tablename__ = "workflow_request_input_step_parameter" id = Column(Integer, primary_key=True) - workflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id'), index=True) - workflow_step_id = Column(Integer, ForeignKey('workflow_step.id')) + workflow_invocation_id = Column(Integer, ForeignKey("workflow_invocation.id"), index=True) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id")) parameter_value = Column(MutableJSONType) - workflow_step = relationship('WorkflowStep') - workflow_invocation = relationship('WorkflowInvocation', back_populates='input_step_parameters') + workflow_step = relationship("WorkflowStep") + workflow_invocation = relationship("WorkflowInvocation", back_populates="input_step_parameters") - dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'parameter_value'] + dict_collection_visible_keys = ["id", "workflow_invocation_id", "workflow_step_id", "parameter_value"] class WorkflowInvocationOutputDatasetAssociation(Base, Dictifiable, RepresentById): """Represents links to output datasets for the workflow.""" - __tablename__ = 'workflow_invocation_output_dataset_association' + + __tablename__ = "workflow_invocation_output_dataset_association" id = Column(Integer, primary_key=True) - workflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id'), index=True) - workflow_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True) - dataset_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) - workflow_output_id = Column(Integer, ForeignKey('workflow_output.id'), index=True) + workflow_invocation_id = Column(Integer, ForeignKey("workflow_invocation.id"), index=True) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id"), index=True) + dataset_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) + workflow_output_id = Column(Integer, ForeignKey("workflow_output.id"), index=True) - workflow_invocation = relationship('WorkflowInvocation', back_populates='output_datasets') - workflow_step = relationship('WorkflowStep') - dataset = relationship('HistoryDatasetAssociation') - workflow_output = relationship('WorkflowOutput') + workflow_invocation = relationship("WorkflowInvocation", back_populates="output_datasets") + workflow_step = relationship("WorkflowStep") + dataset = relationship("HistoryDatasetAssociation") + workflow_output = relationship("WorkflowOutput") history_content_type = "dataset" - dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_id', 'name'] + dict_collection_visible_keys = ["id", "workflow_invocation_id", "workflow_step_id", "dataset_id", "name"] class WorkflowInvocationOutputDatasetCollectionAssociation(Base, Dictifiable, RepresentById): """Represents links to output dataset collections for the workflow.""" - __tablename__ = 'workflow_invocation_output_dataset_collection_association' + + __tablename__ = "workflow_invocation_output_dataset_collection_association" id = Column(Integer, primary_key=True) - workflow_invocation_id = Column(Integer, - ForeignKey('workflow_invocation.id', name='fk_wiodca_wii'), index=True) - workflow_step_id = Column(Integer, - ForeignKey('workflow_step.id', name='fk_wiodca_wsi'), index=True) - dataset_collection_id = Column(Integer, - ForeignKey('history_dataset_collection_association.id', name='fk_wiodca_dci'), index=True) - workflow_output_id = Column(Integer, - ForeignKey('workflow_output.id', name='fk_wiodca_woi'), index=True) + workflow_invocation_id = Column(Integer, ForeignKey("workflow_invocation.id", name="fk_wiodca_wii"), index=True) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id", name="fk_wiodca_wsi"), index=True) + dataset_collection_id = Column( + Integer, ForeignKey("history_dataset_collection_association.id", name="fk_wiodca_dci"), index=True + ) + workflow_output_id = Column(Integer, ForeignKey("workflow_output.id", name="fk_wiodca_woi"), index=True) - workflow_invocation = relationship('WorkflowInvocation', - back_populates='output_dataset_collections') - workflow_step = relationship('WorkflowStep') - dataset_collection = relationship('HistoryDatasetCollectionAssociation') - workflow_output = relationship('WorkflowOutput') + workflow_invocation = relationship("WorkflowInvocation", back_populates="output_dataset_collections") + workflow_step = relationship("WorkflowStep") + dataset_collection = relationship("HistoryDatasetCollectionAssociation") + workflow_output = relationship("WorkflowOutput") history_content_type = "dataset_collection" - dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_collection_id', 'name'] + dict_collection_visible_keys = ["id", "workflow_invocation_id", "workflow_step_id", "dataset_collection_id", "name"] class WorkflowInvocationOutputValue(Base, Dictifiable, RepresentById): """Represents a link to a specified or computed workflow parameter.""" - __tablename__ = 'workflow_invocation_output_value' + + __tablename__ = "workflow_invocation_output_value" id = Column(Integer, primary_key=True) - workflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id'), index=True) - workflow_step_id = Column(Integer, ForeignKey('workflow_step.id')) - workflow_output_id = Column(Integer, ForeignKey('workflow_output.id'), index=True) + workflow_invocation_id = Column(Integer, ForeignKey("workflow_invocation.id"), index=True) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id")) + workflow_output_id = Column(Integer, ForeignKey("workflow_output.id"), index=True) value = Column(MutableJSONType) - workflow_invocation = relationship('WorkflowInvocation', back_populates="output_values") + workflow_invocation = relationship("WorkflowInvocation", back_populates="output_values") - workflow_invocation_step = relationship('WorkflowInvocationStep', - foreign_keys='[WorkflowInvocationStep.workflow_invocation_id, WorkflowInvocationStep.workflow_step_id]', - primaryjoin=(lambda: and_( - WorkflowInvocationStep.workflow_invocation_id == WorkflowInvocationOutputValue.workflow_invocation_id, - WorkflowInvocationStep.workflow_step_id == WorkflowInvocationOutputValue.workflow_step_id, - )), - back_populates='output_value', - viewonly=True + workflow_invocation_step = relationship( + "WorkflowInvocationStep", + foreign_keys="[WorkflowInvocationStep.workflow_invocation_id, WorkflowInvocationStep.workflow_step_id]", + primaryjoin=( + lambda: and_( + WorkflowInvocationStep.workflow_invocation_id == WorkflowInvocationOutputValue.workflow_invocation_id, + WorkflowInvocationStep.workflow_step_id == WorkflowInvocationOutputValue.workflow_step_id, + ) + ), + back_populates="output_value", + viewonly=True, ) - workflow_step = relationship('WorkflowStep') - workflow_output = relationship('WorkflowOutput') + workflow_step = relationship("WorkflowStep") + workflow_output = relationship("WorkflowOutput") - dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'value'] + dict_collection_visible_keys = ["id", "workflow_invocation_id", "workflow_step_id", "value"] class WorkflowInvocationStepOutputDatasetAssociation(Base, Dictifiable, RepresentById): """Represents links to output datasets for the workflow.""" - __tablename__ = 'workflow_invocation_step_output_dataset_association' + + __tablename__ = "workflow_invocation_step_output_dataset_association" id = Column(Integer, primary_key=True) - workflow_invocation_step_id = Column(Integer, - ForeignKey('workflow_invocation_step.id'), index=True) - dataset_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + workflow_invocation_step_id = Column(Integer, ForeignKey("workflow_invocation_step.id"), index=True) + dataset_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) output_name = Column(String(255), nullable=True) - workflow_invocation_step = relationship('WorkflowInvocationStep', - back_populates='output_datasets') - dataset = relationship('HistoryDatasetAssociation') + workflow_invocation_step = relationship("WorkflowInvocationStep", back_populates="output_datasets") + dataset = relationship("HistoryDatasetAssociation") - dict_collection_visible_keys = ['id', 'workflow_invocation_step_id', 'dataset_id', 'output_name'] + dict_collection_visible_keys = ["id", "workflow_invocation_step_id", "dataset_id", "output_name"] class WorkflowInvocationStepOutputDatasetCollectionAssociation(Base, Dictifiable, RepresentById): """Represents links to output dataset collections for the workflow.""" - __tablename__ = 'workflow_invocation_step_output_dataset_collection_association' + + __tablename__ = "workflow_invocation_step_output_dataset_collection_association" id = Column(Integer, primary_key=True) workflow_invocation_step_id = Column( - Integer, ForeignKey('workflow_invocation_step.id', name='fk_wisodca_wisi'), index=True) - workflow_step_id = Column( - Integer, ForeignKey('workflow_step.id', name='fk_wisodca_wsi'), index=True) + Integer, ForeignKey("workflow_invocation_step.id", name="fk_wisodca_wisi"), index=True + ) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id", name="fk_wisodca_wsi"), index=True) dataset_collection_id = Column( - Integer, - ForeignKey('history_dataset_collection_association.id', name='fk_wisodca_dci'), index=True) + Integer, ForeignKey("history_dataset_collection_association.id", name="fk_wisodca_dci"), index=True + ) output_name = Column(String(255), nullable=True) - workflow_invocation_step = relationship('WorkflowInvocationStep', - back_populates='output_dataset_collections') - dataset_collection = relationship('HistoryDatasetCollectionAssociation') + workflow_invocation_step = relationship("WorkflowInvocationStep", back_populates="output_dataset_collections") + dataset_collection = relationship("HistoryDatasetCollectionAssociation") - dict_collection_visible_keys = ['id', 'workflow_invocation_step_id', 'dataset_collection_id', 'output_name'] + dict_collection_visible_keys = ["id", "workflow_invocation_step_id", "dataset_collection_id", "output_name"] class MetadataFile(Base, StorableObject, Serializable): - __tablename__ = 'metadata_file' + __tablename__ = "metadata_file" id = Column(Integer, primary_key=True) name = Column(TEXT) - hda_id = Column(Integer, - ForeignKey('history_dataset_association.id'), index=True, nullable=True) - lda_id = Column(Integer, - ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) + hda_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True, nullable=True) + lda_id = Column(Integer, ForeignKey("library_dataset_dataset_association.id"), index=True, nullable=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, index=True, default=now, onupdate=now) object_store_id = Column(TrimmedString(255), index=True) @@ -7505,8 +7921,8 @@ class MetadataFile(Base, StorableObject, Serializable): deleted = Column(Boolean, index=True, default=False) purged = Column(Boolean, index=True, default=False) - history_dataset = relationship('HistoryDatasetAssociation') - library_dataset = relationship('LibraryDatasetDatasetAssociation') + history_dataset = relationship("HistoryDatasetAssociation") + library_dataset = relationship("LibraryDatasetDatasetAssociation") def __init__(self, dataset=None, name=None, uuid=None): self.uuid = get_uuid(uuid) @@ -7525,18 +7941,22 @@ class MetadataFile(Base, StorableObject, Serializable): self.object_store_id = da.dataset.object_store_id object_store = da.dataset.object_store store_by = object_store.get_store_by(da.dataset) - if store_by == 'id' and self.id is None: + if store_by == "id" and self.id is None: self.flush() identifier = getattr(self, store_by) alt_name = f"metadata_{identifier}.dat" - if not object_store.exists(self, extra_dir='_metadata_files', extra_dir_at_root=True, alt_name=alt_name): - object_store.create(self, extra_dir='_metadata_files', extra_dir_at_root=True, alt_name=alt_name) - path = object_store.get_filename(self, extra_dir='_metadata_files', extra_dir_at_root=True, alt_name=alt_name) + if not object_store.exists(self, extra_dir="_metadata_files", extra_dir_at_root=True, alt_name=alt_name): + object_store.create(self, extra_dir="_metadata_files", extra_dir_at_root=True, alt_name=alt_name) + path = object_store.get_filename( + self, extra_dir="_metadata_files", extra_dir_at_root=True, alt_name=alt_name + ) return path except AttributeError: - assert self.id is not None, "ID must be set before MetadataFile used without an HDA/LDDA (commit the object)" + assert ( + self.id is not None + ), "ID must be set before MetadataFile used without an HDA/LDDA (commit the object)" # In case we're not working with the history_dataset - path = os.path.join(Dataset.file_path, '_metadata_files', *directory_hash_id(self.id)) + path = os.path.join(Dataset.file_path, "_metadata_files", *directory_hash_id(self.id)) # Create directory if it does not exist try: os.makedirs(path) @@ -7550,50 +7970,70 @@ class MetadataFile(Base, StorableObject, Serializable): def _serialize(self, id_encoder, serialization_options): as_dict = dict_for(self) serialization_options.attach_identifier(id_encoder, self, as_dict) - as_dict["uuid"] = str(self.uuid or '') or None + as_dict["uuid"] = str(self.uuid or "") or None return as_dict class FormDefinition(Base, Dictifiable, RepresentById): - __tablename__ = 'form_definition' + __tablename__ = "form_definition" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) name = Column(TrimmedString(255), nullable=False) desc = Column(TEXT) - form_definition_current_id = Column(Integer, - ForeignKey('form_definition_current.id', use_alter=True), index=True, nullable=False) + form_definition_current_id = Column( + Integer, ForeignKey("form_definition_current.id", use_alter=True), index=True, nullable=False + ) fields = Column(MutableJSONType) type = Column(TrimmedString(255), index=True) layout = Column(MutableJSONType) form_definition_current = relationship( - 'FormDefinitionCurrent', - back_populates='forms', - primaryjoin=(lambda: FormDefinitionCurrent.id == FormDefinition.form_definition_current_id)) # type: ignore[has-type] + "FormDefinitionCurrent", + back_populates="forms", + primaryjoin=(lambda: FormDefinitionCurrent.id == FormDefinition.form_definition_current_id), # type: ignore[has-type] + ) # The following form_builder classes are supported by the FormDefinition class. - supported_field_types = [AddressField, CheckboxField, PasswordField, SelectField, TextArea, TextField, WorkflowField, WorkflowMappingField, HistoryField] + supported_field_types = [ + AddressField, + CheckboxField, + PasswordField, + SelectField, + TextArea, + TextField, + WorkflowField, + WorkflowMappingField, + HistoryField, + ] class types(str, Enum): - USER_INFO = 'User Information' + USER_INFO = "User Information" - dict_collection_visible_keys = ['id', 'name'] - dict_element_visible_keys = ['id', 'name', 'desc', 'form_definition_current_id', 'fields', 'layout'] + dict_collection_visible_keys = ["id", "name"] + dict_element_visible_keys = ["id", "name", "desc", "form_definition_current_id", "fields", "layout"] def to_dict(self, user=None, values=None, security=None): values = values or {} - form_def = {'id': security.encode_id(self.id) if security else self.id, 'name': self.name, 'inputs': []} + form_def = {"id": security.encode_id(self.id) if security else self.id, "name": self.name, "inputs": []} for field in self.fields: - FieldClass = ({'AddressField': AddressField, - 'CheckboxField': CheckboxField, - 'HistoryField': HistoryField, - 'PasswordField': PasswordField, - 'SelectField': SelectField, - 'TextArea': TextArea, - 'TextField': TextField, - 'WorkflowField': WorkflowField}).get(field['type'], TextField) - form_def['inputs'].append(FieldClass(user=user, value=values.get(field['name'], field['default']), security=security, **field).to_dict()) + FieldClass = ( + { + "AddressField": AddressField, + "CheckboxField": CheckboxField, + "HistoryField": HistoryField, + "PasswordField": PasswordField, + "SelectField": SelectField, + "TextArea": TextArea, + "TextField": TextField, + "WorkflowField": WorkflowField, + } + ).get(field["type"], TextField) + form_def["inputs"].append( + FieldClass( + user=user, value=values.get(field["name"], field["default"]), security=security, **field + ).to_dict() + ) return form_def def grid_fields(self, grid_index): @@ -7601,44 +8041,46 @@ class FormDefinition(Base, Dictifiable, RepresentById): # on the grid and whose values are the field. gridfields = {} for i, f in enumerate(self.fields): - if str(f['layout']) == str(grid_index): + if str(f["layout"]) == str(grid_index): gridfields[i] = f return gridfields class FormDefinitionCurrent(Base, RepresentById): - __tablename__ = 'form_definition_current' + __tablename__ = "form_definition_current" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - latest_form_id = Column(Integer, ForeignKey('form_definition.id'), index=True) + latest_form_id = Column(Integer, ForeignKey("form_definition.id"), index=True) deleted = Column(Boolean, index=True, default=False) forms = relationship( - 'FormDefinition', - back_populates='form_definition_current', - cascade='all, delete-orphan', - primaryjoin=(lambda: FormDefinitionCurrent.id == FormDefinition.form_definition_current_id)) + "FormDefinition", + back_populates="form_definition_current", + cascade="all, delete-orphan", + primaryjoin=(lambda: FormDefinitionCurrent.id == FormDefinition.form_definition_current_id), + ) latest_form = relationship( - 'FormDefinition', + "FormDefinition", post_update=True, - primaryjoin=(lambda: FormDefinitionCurrent.latest_form_id == FormDefinition.id)) + primaryjoin=(lambda: FormDefinitionCurrent.latest_form_id == FormDefinition.id), + ) def __init__(self, form_definition=None): self.latest_form = form_definition class FormValues(Base, RepresentById): - __tablename__ = 'form_values' + __tablename__ = "form_values" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - form_definition_id = Column(Integer, ForeignKey('form_definition.id'), index=True) + form_definition_id = Column(Integer, ForeignKey("form_definition.id"), index=True) content = Column(MutableJSONType) form_definition = relationship( - 'FormDefinition', - primaryjoin=(lambda: FormValues.form_definition_id == FormDefinition.id)) + "FormDefinition", primaryjoin=(lambda: FormValues.form_definition_id == FormDefinition.id) + ) def __init__(self, form_def=None, content=None): self.form_definition = form_def @@ -7646,12 +8088,12 @@ class FormValues(Base, RepresentById): class UserAddress(Base, RepresentById): - __tablename__ = 'user_address' + __tablename__ = "user_address" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) desc = Column(TrimmedString(255)) name = Column(TrimmedString(255), nullable=False) institution = Column(TrimmedString(255)) @@ -7665,23 +8107,25 @@ class UserAddress(Base, RepresentById): purged = Column(Boolean, index=True, default=False) # `desc` needs to be fully qualified because it is shadowed by `desc` Column defined above # TODO: db migration to rename column, then use `desc` - user = relationship('User', back_populates='addresses', order_by=sqlalchemy.desc('update_time')) + user = relationship("User", back_populates="addresses", order_by=sqlalchemy.desc("update_time")) def to_dict(self, trans): - return {'id': trans.security.encode_id(self.id), - 'name': sanitize_html(self.name), - 'desc': sanitize_html(self.desc), - 'institution': sanitize_html(self.institution), - 'address': sanitize_html(self.address), - 'city': sanitize_html(self.city), - 'state': sanitize_html(self.state), - 'postal_code': sanitize_html(self.postal_code), - 'country': sanitize_html(self.country), - 'phone': sanitize_html(self.phone)} + return { + "id": trans.security.encode_id(self.id), + "name": sanitize_html(self.name), + "desc": sanitize_html(self.desc), + "institution": sanitize_html(self.institution), + "address": sanitize_html(self.address), + "city": sanitize_html(self.city), + "state": sanitize_html(self.state), + "postal_code": sanitize_html(self.postal_code), + "country": sanitize_html(self.country), + "phone": sanitize_html(self.phone), + } class PSAAssociation(Base, AssociationMixin, RepresentById): - __tablename__ = 'psa_association' + __tablename__ = "psa_association" id = Column(Integer, primary_key=True) server_url = Column(VARCHAR(255)) @@ -7717,14 +8161,12 @@ class PSAAssociation(Base, AssociationMixin, RepresentById): @classmethod def remove(cls, ids_to_delete): - cls.sa_session.query(cls).filter(cls.id.in_(ids_to_delete)).delete(synchronize_session='fetch') + cls.sa_session.query(cls).filter(cls.id.in_(ids_to_delete)).delete(synchronize_session="fetch") class PSACode(Base, CodeMixin, RepresentById): - __tablename__ = 'psa_code' - __table_args__ = ( - UniqueConstraint('code', 'email'), - ) + __tablename__ = "psa_code" + __table_args__ = (UniqueConstraint("code", "email"),) id = Column(Integer, primary_key=True) email = Column(VARCHAR(200)) @@ -7747,7 +8189,7 @@ class PSACode(Base, CodeMixin, RepresentById): class PSANonce(Base, NonceMixin, RepresentById): - __tablename__ = 'psa_nonce' + __tablename__ = "psa_nonce" id = Column(Integer, primary_key=True) server_url = Column(VARCHAR(255)) @@ -7778,7 +8220,7 @@ class PSANonce(Base, NonceMixin, RepresentById): class PSAPartial(Base, PartialMixin, RepresentById): - __tablename__ = 'psa_partial' + __tablename__ = "psa_partial" id = Column(Integer, primary_key=True) token = Column(VARCHAR(32)) @@ -7811,8 +8253,8 @@ class PSAPartial(Base, PartialMixin, RepresentById): class UserAuthnzToken(Base, UserMixin, RepresentById): - __tablename__ = 'oidc_user_authnz_tokens' - __table_args__ = (UniqueConstraint('provider', 'uid'),) + __tablename__ = "oidc_user_authnz_tokens" + __table_args__ = (UniqueConstraint("provider", "uid"),) id = Column(Integer, primary_key=True) user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) @@ -7821,7 +8263,7 @@ class UserAuthnzToken(Base, UserMixin, RepresentById): extra_data = Column(MutableJSONType, nullable=True) lifetime = Column(Integer) assoc_type = Column(VARCHAR(64)) - user = relationship('User', back_populates='social_auth') + user = relationship("User", back_populates="social_auth") # This static property is set at: galaxy.authnz.psa_authnz.PSAAuthnz sa_session = None @@ -7839,7 +8281,7 @@ class UserAuthnzToken(Base, UserMixin, RepresentById): # Access and ID tokens have same expiration time; # hence, if one is expired, the other is expired too. self.refresh_token(strategy) - return self.extra_data.get('id_token', None) if self.extra_data is not None else None + return self.extra_data.get("id_token", None) if self.extra_data is not None else None def set_extra_data(self, extra_data=None): if super().set_extra_data(extra_data): @@ -7875,7 +8317,7 @@ class UserAuthnzToken(Base, UserMixin, RepresentById): @classmethod def get_username(cls, user): - return getattr(user, 'username', None) + return getattr(user, "username", None) @classmethod def create_user(cls, *args, **kwargs): @@ -7927,14 +8369,14 @@ class UserAuthnzToken(Base, UserMixin, RepresentById): class CustosAuthnzToken(Base, RepresentById): - __tablename__ = 'custos_authnz_token' + __tablename__ = "custos_authnz_token" __table_args__ = ( - UniqueConstraint('user_id', 'external_user_id', 'provider'), - UniqueConstraint('external_user_id', 'provider'), + UniqueConstraint("user_id", "external_user_id", "provider"), + UniqueConstraint("external_user_id", "provider"), ) id = Column(Integer, primary_key=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id')) + user_id = Column(Integer, ForeignKey("galaxy_user.id")) external_user_id = Column(String(64)) provider = Column(String(255)) access_token = Column(Text) @@ -7942,24 +8384,24 @@ class CustosAuthnzToken(Base, RepresentById): refresh_token = Column(Text) expiration_time = Column(DateTime) refresh_expiration_time = Column(DateTime) - user = relationship('User', back_populates='custos_auth') + user = relationship("User", back_populates="custos_auth") class CloudAuthz(Base, _HasTable): - __tablename__ = 'cloudauthz' + __tablename__ = "cloudauthz" id = Column(Integer, primary_key=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) provider = Column(String(255)) config = Column(MutableJSONType) - authn_id = Column(Integer, ForeignKey('oidc_user_authnz_tokens.id'), index=True) + authn_id = Column(Integer, ForeignKey("oidc_user_authnz_tokens.id"), index=True) tokens = Column(MutableJSONType) last_update = Column(DateTime) last_activity = Column(DateTime) description = Column(TEXT) create_time = Column(DateTime, default=now) - user = relationship('User', back_populates='cloudauthz') - authn = relationship('UserAuthnzToken') + user = relationship("User", back_populates="cloudauthz") + authn = relationship("UserAuthnzToken") def __init__(self, user_id, provider, config, authn_id, description=None): self.user_id = user_id @@ -7971,73 +8413,80 @@ class CloudAuthz(Base, _HasTable): self.description = description def equals(self, user_id, provider, authn_id, config): - return (self.user_id == user_id - and self.provider == provider - and self.authn_id - and self.authn_id == authn_id - and len({k: self.config[k] for k in self.config if k in config - and self.config[k] == config[k]}) == len(self.config)) + return ( + self.user_id == user_id + and self.provider == provider + and self.authn_id + and self.authn_id == authn_id + and len({k: self.config[k] for k in self.config if k in config and self.config[k] == config[k]}) + == len(self.config) + ) class Page(Base, Dictifiable, RepresentById): - __tablename__ = 'page' - __table_args__ = ( - Index('ix_page_slug', 'slug', mysql_length=200), - ) + __tablename__ = "page" + __table_args__ = (Index("ix_page_slug", "slug", mysql_length=200),) id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True, nullable=False) - latest_revision_id = Column(Integer, - ForeignKey('page_revision.id', use_alter=True, name='page_latest_revision_id_fk'), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True, nullable=False) + latest_revision_id = Column( + Integer, ForeignKey("page_revision.id", use_alter=True, name="page_latest_revision_id_fk"), index=True + ) title = Column(TEXT) deleted = Column(Boolean, index=True, default=False) importable = Column(Boolean, index=True, default=False) slug = Column(TEXT) published = Column(Boolean, index=True, default=False) - user = relationship('User') + user = relationship("User") revisions = relationship( - 'PageRevision', + "PageRevision", cascade="all, delete-orphan", primaryjoin=(lambda: Page.id == PageRevision.page_id), # type: ignore[has-type] - back_populates='page') + back_populates="page", + ) latest_revision = relationship( - 'PageRevision', + "PageRevision", post_update=True, primaryjoin=(lambda: Page.latest_revision_id == PageRevision.id), # type: ignore[has-type] - lazy=False) - tags = relationship( - 'PageTagAssociation', - order_by=lambda: PageTagAssociation.id, - back_populates='page') + lazy=False, + ) + tags = relationship("PageTagAssociation", order_by=lambda: PageTagAssociation.id, back_populates="page") annotations = relationship( - 'PageAnnotationAssociation', - order_by=lambda: PageAnnotationAssociation.id, - back_populates='page') + "PageAnnotationAssociation", order_by=lambda: PageAnnotationAssociation.id, back_populates="page" + ) ratings = relationship( - 'PageRatingAssociation', + "PageRatingAssociation", order_by=lambda: PageRatingAssociation.id, # type: ignore[has-type] - back_populates='page') - users_shared_with = relationship( - 'PageUserShareAssociation', - back_populates='page') + back_populates="page", + ) + users_shared_with = relationship("PageUserShareAssociation", back_populates="page") average_rating: column_property # defined at the end of this module # Set up proxy so that # Page.users_shared_with # returns a list of users that page is shared with. - users_shared_with_dot_users = association_proxy('users_shared_with', 'user') + users_shared_with_dot_users = association_proxy("users_shared_with", "user") - dict_element_visible_keys = ['id', 'title', 'latest_revision_id', 'slug', 'published', 'importable', 'deleted', 'username'] + dict_element_visible_keys = [ + "id", + "title", + "latest_revision_id", + "slug", + "published", + "importable", + "deleted", + "username", + ] - def to_dict(self, view='element'): + def to_dict(self, view="element"): rval = super().to_dict(view=view) rev = [] for a in self.revisions: rev.append(a.id) - rval['revision_ids'] = rev + rval["revision_ids"] = rev return rval # username needed for slug generation @@ -8047,54 +8496,55 @@ class Page(Base, Dictifiable, RepresentById): class PageRevision(Base, Dictifiable, RepresentById): - __tablename__ = 'page_revision' + __tablename__ = "page_revision" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - page_id = Column(Integer, ForeignKey('page.id'), index=True, nullable=False) + page_id = Column(Integer, ForeignKey("page.id"), index=True, nullable=False) title = Column(TEXT) content = Column(TEXT) content_format = Column(TrimmedString(32)) - page = relationship('Page', - primaryjoin=(lambda: Page.id == PageRevision.page_id)) - DEFAULT_CONTENT_FORMAT = 'html' - dict_element_visible_keys = ['id', 'page_id', 'title', 'content', 'content_format'] + page = relationship("Page", primaryjoin=(lambda: Page.id == PageRevision.page_id)) + DEFAULT_CONTENT_FORMAT = "html" + dict_element_visible_keys = ["id", "page_id", "title", "content", "content_format"] def __init__(self): self.content_format = PageRevision.DEFAULT_CONTENT_FORMAT - def to_dict(self, view='element'): + def to_dict(self, view="element"): rval = super().to_dict(view=view) - rval['create_time'] = str(self.create_time) - rval['update_time'] = str(self.update_time) + rval["create_time"] = str(self.create_time) + rval["update_time"] = str(self.update_time) return rval class PageUserShareAssociation(Base, UserShareAssociation): - __tablename__ = 'page_user_share_association' + __tablename__ = "page_user_share_association" id = Column(Integer, primary_key=True) page_id = Column(Integer, ForeignKey("page.id"), index=True) user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) - user = relationship('User') - page = relationship('Page', back_populates='users_shared_with') + user = relationship("User") + page = relationship("Page", back_populates="users_shared_with") class Visualization(Base, RepresentById): - __tablename__ = 'visualization' + __tablename__ = "visualization" __table_args__ = ( - Index('ix_visualization_dbkey', 'dbkey', mysql_length=200), - Index('ix_visualization_slug', 'slug', mysql_length=200), + Index("ix_visualization_dbkey", "dbkey", mysql_length=200), + Index("ix_visualization_slug", "slug", mysql_length=200), ) id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True, nullable=False) - latest_revision_id = Column(Integer, - ForeignKey('visualization_revision.id', use_alter=True, name='visualization_latest_revision_id_fk'), - index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True, nullable=False) + latest_revision_id = Column( + Integer, + ForeignKey("visualization_revision.id", use_alter=True, name="visualization_latest_revision_id_fk"), + index=True, + ) title = Column(TEXT) type = Column(TEXT) dbkey = Column(TEXT) @@ -8103,32 +8553,40 @@ class Visualization(Base, RepresentById): slug = Column(TEXT) published = Column(Boolean, default=False, index=True) - user = relationship('User') - revisions = relationship('VisualizationRevision', - back_populates='visualization', + user = relationship("User") + revisions = relationship( + "VisualizationRevision", + back_populates="visualization", cascade="all, delete-orphan", - primaryjoin=(lambda: Visualization.id == VisualizationRevision.visualization_id)) - latest_revision = relationship('VisualizationRevision', + primaryjoin=(lambda: Visualization.id == VisualizationRevision.visualization_id), + ) + latest_revision = relationship( + "VisualizationRevision", post_update=True, primaryjoin=(lambda: Visualization.latest_revision_id == VisualizationRevision.id), - lazy=False) - tags = relationship('VisualizationTagAssociation', - order_by=lambda: VisualizationTagAssociation.id, - back_populates="visualization") - annotations = relationship('VisualizationAnnotationAssociation', + lazy=False, + ) + tags = relationship( + "VisualizationTagAssociation", order_by=lambda: VisualizationTagAssociation.id, back_populates="visualization" + ) + annotations = relationship( + "VisualizationAnnotationAssociation", order_by=lambda: VisualizationAnnotationAssociation.id, - back_populates="visualization") - ratings = relationship('VisualizationRatingAssociation', + back_populates="visualization", + ) + ratings = relationship( + "VisualizationRatingAssociation", order_by=lambda: VisualizationRatingAssociation.id, # type: ignore[has-type] - back_populates="visualization") - users_shared_with = relationship('VisualizationUserShareAssociation', back_populates='visualization') + back_populates="visualization", + ) + users_shared_with = relationship("VisualizationUserShareAssociation", back_populates="visualization") average_rating: column_property # defined at the end of this module # Set up proxy so that # Visualization.users_shared_with # returns a list of users that visualization is shared with. - users_shared_with_dot_users = association_proxy('users_shared_with', 'user') + users_shared_with_dot_users = association_proxy("users_shared_with", "user") def __init__(self, **kwd): super().__init__(**kwd) @@ -8158,21 +8616,21 @@ class Visualization(Base, RepresentById): class VisualizationRevision(Base, RepresentById): - __tablename__ = 'visualization_revision' - __table_args__ = ( - Index('ix_visualization_revision_dbkey', 'dbkey', mysql_length=200), - ) + __tablename__ = "visualization_revision" + __table_args__ = (Index("ix_visualization_revision_dbkey", "dbkey", mysql_length=200),) id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) - visualization_id = Column(Integer, ForeignKey('visualization.id'), index=True, nullable=False) + visualization_id = Column(Integer, ForeignKey("visualization.id"), index=True, nullable=False) title = Column(TEXT) dbkey = Column(TEXT) config = Column(MutableJSONType) - visualization = relationship('Visualization', - back_populates='revisions', - primaryjoin=(lambda: Visualization.id == VisualizationRevision.visualization_id)) + visualization = relationship( + "Visualization", + back_populates="revisions", + primaryjoin=(lambda: Visualization.id == VisualizationRevision.visualization_id), + ) def copy(self, visualization=None): """ @@ -8182,42 +8640,37 @@ class VisualizationRevision(Base, RepresentById): visualization = self.visualization return VisualizationRevision( - visualization=visualization, - title=self.title, - dbkey=self.dbkey, - config=self.config + visualization=visualization, title=self.title, dbkey=self.dbkey, config=self.config ) class VisualizationUserShareAssociation(Base, UserShareAssociation): - __tablename__ = 'visualization_user_share_association' + __tablename__ = "visualization_user_share_association" id = Column(Integer, primary_key=True) - visualization_id = Column(Integer, ForeignKey('visualization.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) - user = relationship('User') - visualization = relationship('Visualization', back_populates='users_shared_with') + visualization_id = Column(Integer, ForeignKey("visualization.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) + user = relationship("User") + visualization = relationship("Visualization", back_populates="users_shared_with") class Tag(Base, RepresentById): - __tablename__ = 'tag' - __table_args__ = ( - UniqueConstraint('name'), - ) + __tablename__ = "tag" + __table_args__ = (UniqueConstraint("name"),) id = Column(Integer, primary_key=True) type = Column(Integer) - parent_id = Column(Integer, ForeignKey('tag.id')) + parent_id = Column(Integer, ForeignKey("tag.id")) name = Column(TrimmedString(255)) - children = relationship('Tag', back_populates='parent') - parent = relationship('Tag', back_populates='children', remote_side=[id]) + children = relationship("Tag", back_populates="parent") + parent = relationship("Tag", back_populates="children", remote_side=[id]) def __str__(self): return "Tag(id=%s, type=%i, parent_id=%s, name=%s)" % (self.id, self.type or -1, self.parent_id, self.name) class ItemTagAssociation(Dictifiable): - dict_collection_visible_keys = ['id', 'user_tname', 'user_value'] + dict_collection_visible_keys = ["id", "user_tname", "user_value"] dict_element_visible_keys = dict_collection_visible_keys associated_item_names: List[str] = [] user_tname: Column @@ -8240,268 +8693,248 @@ class ItemTagAssociation(Dictifiable): class HistoryTagAssociation(Base, ItemTagAssociation, RepresentById): - __tablename__ = 'history_tag_association' + __tablename__ = "history_tag_association" id = Column(Integer, primary_key=True) - history_id = Column(Integer, ForeignKey('history.id'), index=True) - tag_id = Column(Integer, ForeignKey('tag.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + history_id = Column(Integer, ForeignKey("history.id"), index=True) + tag_id = Column(Integer, ForeignKey("tag.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) user_tname = Column(TrimmedString(255), index=True) value = Column(TrimmedString(255), index=True) - history = relationship('History', back_populates='tags') - tag = relationship('Tag') - user = relationship('User') + history = relationship("History", back_populates="tags") + tag = relationship("Tag") + user = relationship("User") class HistoryDatasetAssociationTagAssociation(Base, ItemTagAssociation, RepresentById): - __tablename__ = 'history_dataset_association_tag_association' + __tablename__ = "history_dataset_association_tag_association" id = Column(Integer, primary_key=True) - history_dataset_association_id = Column(Integer, - ForeignKey('history_dataset_association.id'), index=True) - tag_id = Column(Integer, ForeignKey('tag.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + history_dataset_association_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) + tag_id = Column(Integer, ForeignKey("tag.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) user_tname = Column(TrimmedString(255), index=True) value = Column(TrimmedString(255), index=True) - history_dataset_association = relationship('HistoryDatasetAssociation', back_populates='tags') - tag = relationship('Tag') - user = relationship('User') + history_dataset_association = relationship("HistoryDatasetAssociation", back_populates="tags") + tag = relationship("Tag") + user = relationship("User") class LibraryDatasetDatasetAssociationTagAssociation(Base, ItemTagAssociation, RepresentById): - __tablename__ = 'library_dataset_dataset_association_tag_association' + __tablename__ = "library_dataset_dataset_association_tag_association" id = Column(Integer, primary_key=True) library_dataset_dataset_association_id = Column( - Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) - tag_id = Column(Integer, ForeignKey('tag.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + Integer, ForeignKey("library_dataset_dataset_association.id"), index=True + ) + tag_id = Column(Integer, ForeignKey("tag.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) user_tname = Column(TrimmedString(255), index=True) value = Column(TrimmedString(255), index=True) - library_dataset_dataset_association = relationship( - 'LibraryDatasetDatasetAssociation', back_populates='tags') - tag = relationship('Tag') - user = relationship('User') + library_dataset_dataset_association = relationship("LibraryDatasetDatasetAssociation", back_populates="tags") + tag = relationship("Tag") + user = relationship("User") class PageTagAssociation(Base, ItemTagAssociation, RepresentById): - __tablename__ = 'page_tag_association' + __tablename__ = "page_tag_association" id = Column(Integer, primary_key=True) - page_id = Column(Integer, ForeignKey('page.id'), index=True) - tag_id = Column(Integer, ForeignKey('tag.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + page_id = Column(Integer, ForeignKey("page.id"), index=True) + tag_id = Column(Integer, ForeignKey("tag.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) user_tname = Column(TrimmedString(255), index=True) value = Column(TrimmedString(255), index=True) - page = relationship('Page', back_populates='tags') - tag = relationship('Tag') - user = relationship('User') + page = relationship("Page", back_populates="tags") + tag = relationship("Tag") + user = relationship("User") class WorkflowStepTagAssociation(Base, ItemTagAssociation, RepresentById): - __tablename__ = 'workflow_step_tag_association' + __tablename__ = "workflow_step_tag_association" id = Column(Integer, primary_key=True) - workflow_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True) - tag_id = Column(Integer, ForeignKey('tag.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id"), index=True) + tag_id = Column(Integer, ForeignKey("tag.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) user_tname = Column(TrimmedString(255), index=True) value = Column(TrimmedString(255), index=True) - workflow_step = relationship('WorkflowStep', back_populates='tags') - tag = relationship('Tag') - user = relationship('User') + workflow_step = relationship("WorkflowStep", back_populates="tags") + tag = relationship("Tag") + user = relationship("User") class StoredWorkflowTagAssociation(Base, ItemTagAssociation, RepresentById): - __tablename__ = 'stored_workflow_tag_association' + __tablename__ = "stored_workflow_tag_association" id = Column(Integer, primary_key=True) - stored_workflow_id = Column(Integer, ForeignKey('stored_workflow.id'), index=True) - tag_id = Column(Integer, ForeignKey('tag.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + stored_workflow_id = Column(Integer, ForeignKey("stored_workflow.id"), index=True) + tag_id = Column(Integer, ForeignKey("tag.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) user_tname = Column(TrimmedString(255), index=True) value = Column(TrimmedString(255), index=True) - stored_workflow = relationship('StoredWorkflow', back_populates='tags') - tag = relationship('Tag') - user = relationship('User') + stored_workflow = relationship("StoredWorkflow", back_populates="tags") + tag = relationship("Tag") + user = relationship("User") class VisualizationTagAssociation(Base, ItemTagAssociation, RepresentById): - __tablename__ = 'visualization_tag_association' + __tablename__ = "visualization_tag_association" id = Column(Integer, primary_key=True) - visualization_id = Column(Integer, ForeignKey('visualization.id'), index=True) - tag_id = Column(Integer, ForeignKey('tag.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + visualization_id = Column(Integer, ForeignKey("visualization.id"), index=True) + tag_id = Column(Integer, ForeignKey("tag.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) user_tname = Column(TrimmedString(255), index=True) value = Column(TrimmedString(255), index=True) - visualization = relationship('Visualization', back_populates='tags') - tag = relationship('Tag') - user = relationship('User') + visualization = relationship("Visualization", back_populates="tags") + tag = relationship("Tag") + user = relationship("User") class HistoryDatasetCollectionTagAssociation(Base, ItemTagAssociation, RepresentById): - __tablename__ = 'history_dataset_collection_tag_association' + __tablename__ = "history_dataset_collection_tag_association" id = Column(Integer, primary_key=True) - history_dataset_collection_id = Column( - Integer, ForeignKey('history_dataset_collection_association.id'), index=True) - tag_id = Column(Integer, ForeignKey('tag.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + history_dataset_collection_id = Column(Integer, ForeignKey("history_dataset_collection_association.id"), index=True) + tag_id = Column(Integer, ForeignKey("tag.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) user_tname = Column(TrimmedString(255), index=True) value = Column(TrimmedString(255), index=True) - dataset_collection = relationship('HistoryDatasetCollectionAssociation', back_populates='tags') - tag = relationship('Tag') - user = relationship('User') + dataset_collection = relationship("HistoryDatasetCollectionAssociation", back_populates="tags") + tag = relationship("Tag") + user = relationship("User") class LibraryDatasetCollectionTagAssociation(Base, ItemTagAssociation, RepresentById): - __tablename__ = 'library_dataset_collection_tag_association' + __tablename__ = "library_dataset_collection_tag_association" id = Column(Integer, primary_key=True) - library_dataset_collection_id = Column( - Integer, ForeignKey('library_dataset_collection_association.id'), index=True) - tag_id = Column(Integer, ForeignKey('tag.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + library_dataset_collection_id = Column(Integer, ForeignKey("library_dataset_collection_association.id"), index=True) + tag_id = Column(Integer, ForeignKey("tag.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) user_tname = Column(TrimmedString(255), index=True) value = Column(TrimmedString(255), index=True) - dataset_collection = relationship('LibraryDatasetCollectionAssociation', back_populates='tags') - tag = relationship('Tag') - user = relationship('User') + dataset_collection = relationship("LibraryDatasetCollectionAssociation", back_populates="tags") + tag = relationship("Tag") + user = relationship("User") class ToolTagAssociation(Base, ItemTagAssociation, RepresentById): - __tablename__ = 'tool_tag_association' + __tablename__ = "tool_tag_association" id = Column(Integer, primary_key=True) tool_id = Column(TrimmedString(255), index=True) - tag_id = Column(Integer, ForeignKey('tag.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + tag_id = Column(Integer, ForeignKey("tag.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) user_tname = Column(TrimmedString(255), index=True) value = Column(TrimmedString(255), index=True) - tag = relationship('Tag') - user = relationship('User') + tag = relationship("Tag") + user = relationship("User") # Item annotation classes. class HistoryAnnotationAssociation(Base, RepresentById): - __tablename__ = 'history_annotation_association' - __table_args__ = ( - Index('ix_history_anno_assoc_annotation', 'annotation', mysql_length=200), - ) + __tablename__ = "history_annotation_association" + __table_args__ = (Index("ix_history_anno_assoc_annotation", "annotation", mysql_length=200),) id = Column(Integer, primary_key=True) - history_id = Column(Integer, ForeignKey('history.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + history_id = Column(Integer, ForeignKey("history.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) annotation = Column(TEXT) - history = relationship('History', back_populates='annotations') - user = relationship('User') + history = relationship("History", back_populates="annotations") + user = relationship("User") class HistoryDatasetAssociationAnnotationAssociation(Base, RepresentById): - __tablename__ = 'history_dataset_association_annotation_association' - __table_args__ = ( - Index('ix_history_dataset_anno_assoc_annotation', 'annotation', mysql_length=200), - ) + __tablename__ = "history_dataset_association_annotation_association" + __table_args__ = (Index("ix_history_dataset_anno_assoc_annotation", "annotation", mysql_length=200),) id = Column(Integer, primary_key=True) - history_dataset_association_id = Column(Integer, - ForeignKey('history_dataset_association.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + history_dataset_association_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) annotation = Column(TEXT) - hda = relationship('HistoryDatasetAssociation', back_populates='annotations') - user = relationship('User') + hda = relationship("HistoryDatasetAssociation", back_populates="annotations") + user = relationship("User") class StoredWorkflowAnnotationAssociation(Base, RepresentById): - __tablename__ = 'stored_workflow_annotation_association' - __table_args__ = ( - Index('ix_stored_workflow_ann_assoc_annotation', 'annotation', mysql_length=200), - ) + __tablename__ = "stored_workflow_annotation_association" + __table_args__ = (Index("ix_stored_workflow_ann_assoc_annotation", "annotation", mysql_length=200),) id = Column(Integer, primary_key=True) - stored_workflow_id = Column(Integer, ForeignKey('stored_workflow.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + stored_workflow_id = Column(Integer, ForeignKey("stored_workflow.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) annotation = Column(TEXT) - stored_workflow = relationship('StoredWorkflow', back_populates='annotations') - user = relationship('User') + stored_workflow = relationship("StoredWorkflow", back_populates="annotations") + user = relationship("User") class WorkflowStepAnnotationAssociation(Base, RepresentById): - __tablename__ = 'workflow_step_annotation_association' - __table_args__ = ( - Index('ix_workflow_step_ann_assoc_annotation', 'annotation', mysql_length=200), - ) + __tablename__ = "workflow_step_annotation_association" + __table_args__ = (Index("ix_workflow_step_ann_assoc_annotation", "annotation", mysql_length=200),) id = Column(Integer, primary_key=True) - workflow_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) annotation = Column(TEXT) - workflow_step = relationship('WorkflowStep', back_populates='annotations') - user = relationship('User') + workflow_step = relationship("WorkflowStep", back_populates="annotations") + user = relationship("User") class PageAnnotationAssociation(Base, RepresentById): - __tablename__ = 'page_annotation_association' - __table_args__ = ( - Index('ix_page_annotation_association_annotation', 'annotation', mysql_length=200), - ) + __tablename__ = "page_annotation_association" + __table_args__ = (Index("ix_page_annotation_association_annotation", "annotation", mysql_length=200),) id = Column(Integer, primary_key=True) - page_id = Column(Integer, ForeignKey('page.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + page_id = Column(Integer, ForeignKey("page.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) annotation = Column(TEXT) - page = relationship('Page', back_populates='annotations') - user = relationship('User') + page = relationship("Page", back_populates="annotations") + user = relationship("User") class VisualizationAnnotationAssociation(Base, RepresentById): - __tablename__ = 'visualization_annotation_association' - __table_args__ = ( - Index('ix_visualization_annotation_association_annotation', 'annotation', mysql_length=200), - ) + __tablename__ = "visualization_annotation_association" + __table_args__ = (Index("ix_visualization_annotation_association_annotation", "annotation", mysql_length=200),) id = Column(Integer, primary_key=True) - visualization_id = Column(Integer, ForeignKey('visualization.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + visualization_id = Column(Integer, ForeignKey("visualization.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) annotation = Column(TEXT) - visualization = relationship('Visualization', back_populates='annotations') - user = relationship('User') + visualization = relationship("Visualization", back_populates="annotations") + user = relationship("User") class HistoryDatasetCollectionAssociationAnnotationAssociation(Base, RepresentById): - __tablename__ = 'history_dataset_collection_annotation_association' + __tablename__ = "history_dataset_collection_annotation_association" id = Column(Integer, primary_key=True) - history_dataset_collection_id = Column( - Integer, ForeignKey('history_dataset_collection_association.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + history_dataset_collection_id = Column(Integer, ForeignKey("history_dataset_collection_association.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) annotation = Column(TEXT) - history_dataset_collection = relationship('HistoryDatasetCollectionAssociation', - back_populates='annotations') - user = relationship('User') + history_dataset_collection = relationship("HistoryDatasetCollectionAssociation", back_populates="annotations") + user = relationship("User") class LibraryDatasetCollectionAnnotationAssociation(Base, RepresentById): - __tablename__ = 'library_dataset_collection_annotation_association' + __tablename__ = "library_dataset_collection_annotation_association" id = Column(Integer, primary_key=True) - library_dataset_collection_id = Column( - Integer, ForeignKey('library_dataset_collection_association.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + library_dataset_collection_id = Column(Integer, ForeignKey("library_dataset_collection_association.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) annotation = Column(TEXT) - dataset_collection = relationship('LibraryDatasetCollectionAssociation', - back_populates='annotations') - user = relationship('User') + dataset_collection = relationship("LibraryDatasetCollectionAssociation", back_populates="annotations") + user = relationship("User") class Vault(Base): - __tablename__ = 'vault' + __tablename__ = "vault" key = Column(Text, primary_key=True) parent_key = Column(Text, ForeignKey(key), index=True, nullable=True) - children = relationship('Vault', back_populates='parent') - parent = relationship('Vault', back_populates='children', remote_side=[key]) + children = relationship("Vault", back_populates="parent") + parent = relationship("Vault", back_populates="children", remote_side=[key]) value = Column(Text, nullable=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) @@ -8517,106 +8950,103 @@ class ItemRatingAssociation(Base): self._set_item(item) def _set_item(self, item): - """ Set association's item. """ + """Set association's item.""" raise NotImplementedError() class HistoryRatingAssociation(ItemRatingAssociation, RepresentById): - __tablename__ = 'history_rating_association' + __tablename__ = "history_rating_association" id = Column(Integer, primary_key=True) - history_id = Column(Integer, ForeignKey('history.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + history_id = Column(Integer, ForeignKey("history.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) rating = Column(Integer, index=True) - history = relationship('History', back_populates='ratings') - user = relationship('User') + history = relationship("History", back_populates="ratings") + user = relationship("User") def _set_item(self, history): self.history = history class HistoryDatasetAssociationRatingAssociation(ItemRatingAssociation, RepresentById): - __tablename__ = 'history_dataset_association_rating_association' + __tablename__ = "history_dataset_association_rating_association" id = Column(Integer, primary_key=True) - history_dataset_association_id = Column(Integer, - ForeignKey('history_dataset_association.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + history_dataset_association_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) rating = Column(Integer, index=True) - history_dataset_association = relationship('HistoryDatasetAssociation', back_populates='ratings') - user = relationship('User') + history_dataset_association = relationship("HistoryDatasetAssociation", back_populates="ratings") + user = relationship("User") def _set_item(self, history_dataset_association): self.history_dataset_association = history_dataset_association class StoredWorkflowRatingAssociation(ItemRatingAssociation, RepresentById): - __tablename__ = 'stored_workflow_rating_association' + __tablename__ = "stored_workflow_rating_association" id = Column(Integer, primary_key=True) - stored_workflow_id = Column(Integer, ForeignKey('stored_workflow.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + stored_workflow_id = Column(Integer, ForeignKey("stored_workflow.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) rating = Column(Integer, index=True) - stored_workflow = relationship('StoredWorkflow', back_populates='ratings') - user = relationship('User') + stored_workflow = relationship("StoredWorkflow", back_populates="ratings") + user = relationship("User") def _set_item(self, stored_workflow): self.stored_workflow = stored_workflow class PageRatingAssociation(ItemRatingAssociation, RepresentById): - __tablename__ = 'page_rating_association' + __tablename__ = "page_rating_association" id = Column(Integer, primary_key=True) - page_id = Column(Integer, ForeignKey('page.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + page_id = Column(Integer, ForeignKey("page.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) rating = Column(Integer, index=True) - page = relationship('Page', back_populates='ratings') - user = relationship('User') + page = relationship("Page", back_populates="ratings") + user = relationship("User") def _set_item(self, page): self.page = page class VisualizationRatingAssociation(ItemRatingAssociation, RepresentById): - __tablename__ = 'visualization_rating_association' + __tablename__ = "visualization_rating_association" id = Column(Integer, primary_key=True) - visualization_id = Column(Integer, ForeignKey('visualization.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + visualization_id = Column(Integer, ForeignKey("visualization.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) rating = Column(Integer, index=True) - visualization = relationship('Visualization', back_populates='ratings') - user = relationship('User') + visualization = relationship("Visualization", back_populates="ratings") + user = relationship("User") def _set_item(self, visualization): self.visualization = visualization class HistoryDatasetCollectionRatingAssociation(ItemRatingAssociation, RepresentById): - __tablename__ = 'history_dataset_collection_rating_association' + __tablename__ = "history_dataset_collection_rating_association" id = Column(Integer, primary_key=True) - history_dataset_collection_id = Column( - Integer, ForeignKey('history_dataset_collection_association.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + history_dataset_collection_id = Column(Integer, ForeignKey("history_dataset_collection_association.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) rating = Column(Integer, index=True) - dataset_collection = relationship('HistoryDatasetCollectionAssociation', back_populates='ratings') - user = relationship('User') + dataset_collection = relationship("HistoryDatasetCollectionAssociation", back_populates="ratings") + user = relationship("User") def _set_item(self, dataset_collection): self.dataset_collection = dataset_collection class LibraryDatasetCollectionRatingAssociation(ItemRatingAssociation, RepresentById): - __tablename__ = 'library_dataset_collection_rating_association' + __tablename__ = "library_dataset_collection_rating_association" id = Column(Integer, primary_key=True) - library_dataset_collection_id = Column( - Integer, ForeignKey('library_dataset_collection_association.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + library_dataset_collection_id = Column(Integer, ForeignKey("library_dataset_collection_association.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) rating = Column(Integer, index=True) - dataset_collection = relationship('LibraryDatasetCollectionAssociation', back_populates='ratings') - user = relationship('User') + dataset_collection = relationship("LibraryDatasetCollectionAssociation", back_populates="ratings") + user = relationship("User") def _set_item(self, dataset_collection): self.dataset_collection = dataset_collection @@ -8624,36 +9054,34 @@ class LibraryDatasetCollectionRatingAssociation(ItemRatingAssociation, Represent # Data manager classes. class DataManagerHistoryAssociation(Base, RepresentById): - __tablename__ = 'data_manager_history_association' + __tablename__ = "data_manager_history_association" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, index=True, default=now, onupdate=now) - history_id = Column(Integer, ForeignKey('history.id'), index=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) - history = relationship('History') - user = relationship('User', back_populates='data_manager_histories') + history_id = Column(Integer, ForeignKey("history.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) + history = relationship("History") + user = relationship("User", back_populates="data_manager_histories") class DataManagerJobAssociation(Base, RepresentById): - __tablename__ = 'data_manager_job_association' - __table_args__ = ( - Index('ix_data_manager_job_association_data_manager_id', 'data_manager_id', mysql_length=200), - ) + __tablename__ = "data_manager_job_association" + __table_args__ = (Index("ix_data_manager_job_association_data_manager_id", "data_manager_id", mysql_length=200),) id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, index=True, default=now, onupdate=now) job_id = Column(Integer, ForeignKey("job.id"), index=True) data_manager_id = Column(TEXT) - job = relationship('Job', back_populates='data_manager_association', uselist=False) + job = relationship("Job", back_populates="data_manager_association", uselist=False) class UserPreference(Base, RepresentById): - __tablename__ = 'user_preference' + __tablename__ = "user_preference" id = Column(Integer, primary_key=True) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) name = Column(Unicode(255), index=True) value = Column(Text) @@ -8665,7 +9093,7 @@ class UserPreference(Base, RepresentById): class UserAction(Base, RepresentById): - __tablename__ = 'user_action' + __tablename__ = "user_action" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) @@ -8674,17 +9102,17 @@ class UserAction(Base, RepresentById): action = Column(Unicode(255)) context = Column(Unicode(512)) params = Column(Unicode(1024)) - user = relationship('User') + user = relationship("User") class APIKeys(Base, RepresentById): - __tablename__ = 'api_keys' + __tablename__ = "api_keys" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) - user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) key = Column(TrimmedString(32), index=True, unique=True) - user = relationship('User', back_populates='api_keys') + user = relationship("User", back_populates="api_keys") def copy_list(lst, *args, **kwds): @@ -8695,7 +9123,7 @@ def copy_list(lst, *args, **kwds): def _prepare_metadata_for_serialization(id_encoder, serialization_options, metadata): - """ Prepare metatdata for exporting. """ + """Prepare metatdata for exporting.""" processed_metadata = {} for name, value in metadata.items(): # Metadata files are not needed for export because they can be @@ -8713,8 +9141,9 @@ def _prepare_metadata_for_serialization(id_encoder, serialization_options, metad # The following CleanupEvent* models could be defined as tables only; # however making them models keeps things simple and consistent. + class CleanupEvent(Base): - __tablename__ = 'cleanup_event' + __tablename__ = "cleanup_event" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) @@ -8722,84 +9151,84 @@ class CleanupEvent(Base): class CleanupEventDatasetAssociation(Base): - __tablename__ = 'cleanup_event_dataset_association' + __tablename__ = "cleanup_event_dataset_association" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) - cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) - dataset_id = Column(Integer, ForeignKey('dataset.id'), index=True) + cleanup_event_id = Column(Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True) + dataset_id = Column(Integer, ForeignKey("dataset.id"), index=True) class CleanupEventMetadataFileAssociation(Base): - __tablename__ = 'cleanup_event_metadata_file_association' + __tablename__ = "cleanup_event_metadata_file_association" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) - cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) - metadata_file_id = Column(Integer, ForeignKey('metadata_file.id'), index=True) + cleanup_event_id = Column(Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True) + metadata_file_id = Column(Integer, ForeignKey("metadata_file.id"), index=True) class CleanupEventHistoryAssociation(Base): - __tablename__ = 'cleanup_event_history_association' + __tablename__ = "cleanup_event_history_association" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) - cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) - history_id = Column(Integer, ForeignKey('history.id'), index=True) + cleanup_event_id = Column(Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True) + history_id = Column(Integer, ForeignKey("history.id"), index=True) class CleanupEventHistoryDatasetAssociationAssociation(Base): - __tablename__ = 'cleanup_event_hda_association' + __tablename__ = "cleanup_event_hda_association" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) - cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) - hda_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + cleanup_event_id = Column(Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True) + hda_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) class CleanupEventLibraryAssociation(Base): - __tablename__ = 'cleanup_event_library_association' + __tablename__ = "cleanup_event_library_association" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) - cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) - library_id = Column(Integer, ForeignKey('library.id'), index=True) + cleanup_event_id = Column(Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True) + library_id = Column(Integer, ForeignKey("library.id"), index=True) class CleanupEventLibraryFolderAssociation(Base): - __tablename__ = 'cleanup_event_library_folder_association' + __tablename__ = "cleanup_event_library_folder_association" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) - cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) - library_folder_id = Column(Integer, ForeignKey('library_folder.id'), index=True) + cleanup_event_id = Column(Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True) + library_folder_id = Column(Integer, ForeignKey("library_folder.id"), index=True) class CleanupEventLibraryDatasetAssociation(Base): - __tablename__ = 'cleanup_event_library_dataset_association' + __tablename__ = "cleanup_event_library_dataset_association" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) - cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) - library_dataset_id = Column(Integer, ForeignKey('library_dataset.id'), index=True) + cleanup_event_id = Column(Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True) + library_dataset_id = Column(Integer, ForeignKey("library_dataset.id"), index=True) class CleanupEventLibraryDatasetDatasetAssociationAssociation(Base): - __tablename__ = 'cleanup_event_ldda_association' + __tablename__ = "cleanup_event_ldda_association" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) - cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) - ldda_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) + cleanup_event_id = Column(Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True) + ldda_id = Column(Integer, ForeignKey("library_dataset_dataset_association.id"), index=True) class CleanupEventImplicitlyConvertedDatasetAssociationAssociation(Base): - __tablename__ = 'cleanup_event_icda_association' + __tablename__ = "cleanup_event_icda_association" id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) - cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) - icda_id = Column(Integer, ForeignKey('implicitly_converted_dataset_association.id'), index=True) + cleanup_event_id = Column(Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True) + icda_id = Column(Integer, ForeignKey("implicitly_converted_dataset_association.id"), index=True) # The following models (Dataset, HDA, LDDA) are mapped imperatively (for details see discussion in PR #12064) @@ -8809,294 +9238,391 @@ class CleanupEventImplicitlyConvertedDatasetAssociationAssociation(Base): # mapping into the test; however, having all models mapped in the same module is cleaner. Dataset.table = Table( - 'dataset', mapper_registry.metadata, - Column('id', Integer, primary_key=True), - Column('job_id', Integer, ForeignKey('job.id'), index=True, nullable=True), - Column('create_time', DateTime, default=now), - Column('update_time', DateTime, index=True, default=now, onupdate=now), - Column('state', TrimmedString(64), index=True), - Column('deleted', Boolean, index=True, default=False), - Column('purged', Boolean, index=True, default=False), - Column('purgable', Boolean, default=True), - Column('object_store_id', TrimmedString(255), index=True), - Column('external_filename', TEXT), - Column('_extra_files_path', TEXT), - Column('created_from_basename', TEXT), - Column('file_size', Numeric(15, 0)), - Column('total_size', Numeric(15, 0)), - Column('uuid', UUIDType())) + "dataset", + mapper_registry.metadata, + Column("id", Integer, primary_key=True), + Column("job_id", Integer, ForeignKey("job.id"), index=True, nullable=True), + Column("create_time", DateTime, default=now), + Column("update_time", DateTime, index=True, default=now, onupdate=now), + Column("state", TrimmedString(64), index=True), + Column("deleted", Boolean, index=True, default=False), + Column("purged", Boolean, index=True, default=False), + Column("purgable", Boolean, default=True), + Column("object_store_id", TrimmedString(255), index=True), + Column("external_filename", TEXT), + Column("_extra_files_path", TEXT), + Column("created_from_basename", TEXT), + Column("file_size", Numeric(15, 0)), + Column("total_size", Numeric(15, 0)), + Column("uuid", UUIDType()), +) HistoryDatasetAssociation.table = Table( - 'history_dataset_association', mapper_registry.metadata, - Column('id', Integer, primary_key=True), - Column('history_id', Integer, ForeignKey('history.id'), index=True), - Column('dataset_id', Integer, ForeignKey('dataset.id'), index=True), - Column('create_time', DateTime, default=now), - Column('update_time', DateTime, default=now, onupdate=now, index=True), - Column('state', TrimmedString(64), index=True, key='_state'), - Column('copied_from_history_dataset_association_id', Integer, - ForeignKey('history_dataset_association.id'), nullable=True), - Column('copied_from_library_dataset_dataset_association_id', Integer, - ForeignKey('library_dataset_dataset_association.id'), nullable=True), - Column('name', TrimmedString(255)), - Column('info', TrimmedString(255)), - Column('blurb', TrimmedString(255)), - Column('peek', TEXT, key='_peek'), - Column('tool_version', TEXT), - Column('extension', TrimmedString(64)), + "history_dataset_association", + mapper_registry.metadata, + Column("id", Integer, primary_key=True), + Column("history_id", Integer, ForeignKey("history.id"), index=True), + Column("dataset_id", Integer, ForeignKey("dataset.id"), index=True), + Column("create_time", DateTime, default=now), + Column("update_time", DateTime, default=now, onupdate=now, index=True), + Column("state", TrimmedString(64), index=True, key="_state"), + Column( + "copied_from_history_dataset_association_id", + Integer, + ForeignKey("history_dataset_association.id"), + nullable=True, + ), + Column( + "copied_from_library_dataset_dataset_association_id", + Integer, + ForeignKey("library_dataset_dataset_association.id"), + nullable=True, + ), + Column("name", TrimmedString(255)), + Column("info", TrimmedString(255)), + Column("blurb", TrimmedString(255)), + Column("peek", TEXT, key="_peek"), + Column("tool_version", TEXT), + Column("extension", TrimmedString(64)), Column("metadata", MetadataType, key="_metadata"), - Column('parent_id', Integer, ForeignKey('history_dataset_association.id'), nullable=True), - Column('designation', TrimmedString(255)), - Column('deleted', Boolean, index=True, default=False), - Column('visible', Boolean), - Column('extended_metadata_id', Integer, ForeignKey('extended_metadata.id'), index=True), - Column('version', Integer, default=1, nullable=True, index=True), - Column('hid', Integer), - Column('purged', Boolean, index=True, default=False), - Column('validated_state', TrimmedString(64), default='unvalidated', nullable=False), - Column('validated_state_message', TEXT), - Column('hidden_beneath_collection_instance_id', - ForeignKey('history_dataset_collection_association.id'), nullable=True)) + Column("parent_id", Integer, ForeignKey("history_dataset_association.id"), nullable=True), + Column("designation", TrimmedString(255)), + Column("deleted", Boolean, index=True, default=False), + Column("visible", Boolean), + Column("extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), index=True), + Column("version", Integer, default=1, nullable=True, index=True), + Column("hid", Integer), + Column("purged", Boolean, index=True, default=False), + Column("validated_state", TrimmedString(64), default="unvalidated", nullable=False), + Column("validated_state_message", TEXT), + Column( + "hidden_beneath_collection_instance_id", ForeignKey("history_dataset_collection_association.id"), nullable=True + ), +) LibraryDatasetDatasetAssociation.table = Table( - 'library_dataset_dataset_association', mapper_registry.metadata, - Column('id', Integer, primary_key=True), - Column('library_dataset_id', Integer, ForeignKey('library_dataset.id'), index=True), - Column('dataset_id', Integer, ForeignKey('dataset.id'), index=True), - Column('create_time', DateTime, default=now), - Column('update_time', DateTime, default=now, onupdate=now, index=True), - Column('state', TrimmedString(64), index=True, key='_state'), - Column('copied_from_history_dataset_association_id', Integer, - ForeignKey('history_dataset_association.id', - use_alter=True, name='history_dataset_association_dataset_id_fkey'), - nullable=True), - Column('copied_from_library_dataset_dataset_association_id', Integer, - ForeignKey('library_dataset_dataset_association.id', - use_alter=True, name='library_dataset_dataset_association_id_fkey'), - nullable=True), - Column('name', TrimmedString(255), index=True), - Column('info', TrimmedString(255)), - Column('blurb', TrimmedString(255)), - Column('peek', TEXT, key='_peek'), - Column('tool_version', TEXT), - Column('extension', TrimmedString(64)), + "library_dataset_dataset_association", + mapper_registry.metadata, + Column("id", Integer, primary_key=True), + Column("library_dataset_id", Integer, ForeignKey("library_dataset.id"), index=True), + Column("dataset_id", Integer, ForeignKey("dataset.id"), index=True), + Column("create_time", DateTime, default=now), + Column("update_time", DateTime, default=now, onupdate=now, index=True), + Column("state", TrimmedString(64), index=True, key="_state"), + Column( + "copied_from_history_dataset_association_id", + Integer, + ForeignKey( + "history_dataset_association.id", use_alter=True, name="history_dataset_association_dataset_id_fkey" + ), + nullable=True, + ), + Column( + "copied_from_library_dataset_dataset_association_id", + Integer, + ForeignKey( + "library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fkey" + ), + nullable=True, + ), + Column("name", TrimmedString(255), index=True), + Column("info", TrimmedString(255)), + Column("blurb", TrimmedString(255)), + Column("peek", TEXT, key="_peek"), + Column("tool_version", TEXT), + Column("extension", TrimmedString(64)), Column("metadata", MetadataType, key="_metadata"), - Column('parent_id', Integer, ForeignKey('library_dataset_dataset_association.id'), nullable=True), - Column('designation', TrimmedString(255)), - Column('deleted', Boolean, index=True, default=False), - Column('validated_state', TrimmedString(64), default='unvalidated', nullable=False), - Column('validated_state_message', TEXT), - Column('visible', Boolean), - Column('extended_metadata_id', Integer, ForeignKey('extended_metadata.id'), index=True), - Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True), - Column('message', TrimmedString(255))) + Column("parent_id", Integer, ForeignKey("library_dataset_dataset_association.id"), nullable=True), + Column("designation", TrimmedString(255)), + Column("deleted", Boolean, index=True, default=False), + Column("validated_state", TrimmedString(64), default="unvalidated", nullable=False), + Column("validated_state_message", TEXT), + Column("visible", Boolean), + Column("extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), index=True), + Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), + Column("message", TrimmedString(255)), +) mapper_registry.map_imperatively( Dataset, Dataset.table, properties=dict( - actions=relationship(DatasetPermissions, back_populates='dataset'), + actions=relationship(DatasetPermissions, back_populates="dataset"), job=relationship(Job, primaryjoin=(Dataset.table.c.job_id == Job.id)), - active_history_associations=relationship(HistoryDatasetAssociation, + active_history_associations=relationship( + HistoryDatasetAssociation, primaryjoin=( (Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id) & (HistoryDatasetAssociation.table.c.deleted == false()) - & (HistoryDatasetAssociation.table.c.purged == false())), - viewonly=True), - purged_history_associations=relationship(HistoryDatasetAssociation, + & (HistoryDatasetAssociation.table.c.purged == false()) + ), + viewonly=True, + ), + purged_history_associations=relationship( + HistoryDatasetAssociation, primaryjoin=( (Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id) - & (HistoryDatasetAssociation.table.c.purged == true())), - viewonly=True), - active_library_associations=relationship(LibraryDatasetDatasetAssociation, + & (HistoryDatasetAssociation.table.c.purged == true()) + ), + viewonly=True, + ), + active_library_associations=relationship( + LibraryDatasetDatasetAssociation, primaryjoin=( (Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id) - & (LibraryDatasetDatasetAssociation.table.c.deleted == false())), - viewonly=True), - hashes=relationship(DatasetHash, back_populates='dataset'), - sources=relationship(DatasetSource, back_populates='dataset'), - history_associations=relationship(HistoryDatasetAssociation, back_populates='dataset'), - library_associations=relationship(LibraryDatasetDatasetAssociation, + & (LibraryDatasetDatasetAssociation.table.c.deleted == false()) + ), + viewonly=True, + ), + hashes=relationship(DatasetHash, back_populates="dataset"), + sources=relationship(DatasetSource, back_populates="dataset"), + history_associations=relationship(HistoryDatasetAssociation, back_populates="dataset"), + library_associations=relationship( + LibraryDatasetDatasetAssociation, primaryjoin=(LibraryDatasetDatasetAssociation.table.c.dataset_id == Dataset.table.c.id), - back_populates='dataset'), - ) + back_populates="dataset", + ), + ), ) mapper_registry.map_imperatively( HistoryDatasetAssociation, HistoryDatasetAssociation.table, properties=dict( - dataset=relationship(Dataset, + dataset=relationship( + Dataset, primaryjoin=(Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id), lazy="joined", - back_populates='history_associations'), - copied_from_history_dataset_association=relationship(HistoryDatasetAssociation, - primaryjoin=(HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id - == HistoryDatasetAssociation.table.c.id), + back_populates="history_associations", + ), + copied_from_history_dataset_association=relationship( + HistoryDatasetAssociation, + primaryjoin=( + HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id + == HistoryDatasetAssociation.table.c.id + ), remote_side=[HistoryDatasetAssociation.table.c.id], uselist=False, - back_populates='copied_to_history_dataset_associations'), - copied_from_library_dataset_dataset_association=relationship(LibraryDatasetDatasetAssociation, - primaryjoin=(LibraryDatasetDatasetAssociation.table.c.id - == HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id), - back_populates='copied_to_history_dataset_associations'), - copied_to_history_dataset_associations=relationship(HistoryDatasetAssociation, - primaryjoin=(HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id - == HistoryDatasetAssociation.table.c.id), - back_populates='copied_from_history_dataset_association'), - copied_to_library_dataset_dataset_associations=relationship(LibraryDatasetDatasetAssociation, - primaryjoin=(HistoryDatasetAssociation.table.c.id - == LibraryDatasetDatasetAssociation.table.c.copied_from_history_dataset_association_id), - back_populates='copied_from_history_dataset_association'), - tags=relationship(HistoryDatasetAssociationTagAssociation, + back_populates="copied_to_history_dataset_associations", + ), + copied_from_library_dataset_dataset_association=relationship( + LibraryDatasetDatasetAssociation, + primaryjoin=( + LibraryDatasetDatasetAssociation.table.c.id + == HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id + ), + back_populates="copied_to_history_dataset_associations", + ), + copied_to_history_dataset_associations=relationship( + HistoryDatasetAssociation, + primaryjoin=( + HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id + == HistoryDatasetAssociation.table.c.id + ), + back_populates="copied_from_history_dataset_association", + ), + copied_to_library_dataset_dataset_associations=relationship( + LibraryDatasetDatasetAssociation, + primaryjoin=( + HistoryDatasetAssociation.table.c.id + == LibraryDatasetDatasetAssociation.table.c.copied_from_history_dataset_association_id + ), + back_populates="copied_from_history_dataset_association", + ), + tags=relationship( + HistoryDatasetAssociationTagAssociation, order_by=HistoryDatasetAssociationTagAssociation.id, - back_populates='history_dataset_association'), - annotations=relationship(HistoryDatasetAssociationAnnotationAssociation, + back_populates="history_dataset_association", + ), + annotations=relationship( + HistoryDatasetAssociationAnnotationAssociation, order_by=HistoryDatasetAssociationAnnotationAssociation.id, - back_populates="hda"), - ratings=relationship(HistoryDatasetAssociationRatingAssociation, + back_populates="hda", + ), + ratings=relationship( + HistoryDatasetAssociationRatingAssociation, order_by=HistoryDatasetAssociationRatingAssociation.id, - back_populates="history_dataset_association"), - extended_metadata=relationship(ExtendedMetadata, - primaryjoin=(HistoryDatasetAssociation.table.c.extended_metadata_id - == ExtendedMetadata.id)), - hidden_beneath_collection_instance=relationship(HistoryDatasetCollectionAssociation, - primaryjoin=(HistoryDatasetAssociation.table.c.hidden_beneath_collection_instance_id - == HistoryDatasetCollectionAssociation.id), - uselist=False), + back_populates="history_dataset_association", + ), + extended_metadata=relationship( + ExtendedMetadata, + primaryjoin=(HistoryDatasetAssociation.table.c.extended_metadata_id == ExtendedMetadata.id), + ), + hidden_beneath_collection_instance=relationship( + HistoryDatasetCollectionAssociation, + primaryjoin=( + HistoryDatasetAssociation.table.c.hidden_beneath_collection_instance_id + == HistoryDatasetCollectionAssociation.id + ), + uselist=False, + ), _metadata=deferred(HistoryDatasetAssociation.table.c._metadata), - dependent_jobs=relationship(JobToInputDatasetAssociation, back_populates='dataset'), - creating_job_associations=relationship( - JobToOutputDatasetAssociation, back_populates='dataset'), - history=relationship(History, back_populates='datasets'), - implicitly_converted_datasets=relationship(ImplicitlyConvertedDatasetAssociation, - primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_parent_id - == HistoryDatasetAssociation.id), - back_populates='parent_hda'), - implicitly_converted_parent_datasets=relationship(ImplicitlyConvertedDatasetAssociation, - primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_id - == HistoryDatasetAssociation.id), - back_populates='dataset') - ) + dependent_jobs=relationship(JobToInputDatasetAssociation, back_populates="dataset"), + creating_job_associations=relationship(JobToOutputDatasetAssociation, back_populates="dataset"), + history=relationship(History, back_populates="datasets"), + implicitly_converted_datasets=relationship( + ImplicitlyConvertedDatasetAssociation, + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_parent_id == HistoryDatasetAssociation.id), + back_populates="parent_hda", + ), + implicitly_converted_parent_datasets=relationship( + ImplicitlyConvertedDatasetAssociation, + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_id == HistoryDatasetAssociation.id), + back_populates="dataset", + ), + ), ) mapper_registry.map_imperatively( LibraryDatasetDatasetAssociation, LibraryDatasetDatasetAssociation.table, properties=dict( - dataset=relationship(Dataset, + dataset=relationship( + Dataset, primaryjoin=(LibraryDatasetDatasetAssociation.table.c.dataset_id == Dataset.table.c.id), - back_populates='library_associations'), - library_dataset=relationship(LibraryDataset, - foreign_keys=LibraryDatasetDatasetAssociation.table.c.library_dataset_id), + back_populates="library_associations", + ), + library_dataset=relationship( + LibraryDataset, foreign_keys=LibraryDatasetDatasetAssociation.table.c.library_dataset_id + ), user=relationship(User), - copied_from_library_dataset_dataset_association=relationship(LibraryDatasetDatasetAssociation, + copied_from_library_dataset_dataset_association=relationship( + LibraryDatasetDatasetAssociation, primaryjoin=( LibraryDatasetDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id - == LibraryDatasetDatasetAssociation.table.c.id), + == LibraryDatasetDatasetAssociation.table.c.id + ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id], uselist=False, - back_populates='copied_to_library_dataset_dataset_associations'), - copied_to_library_dataset_dataset_associations=relationship(LibraryDatasetDatasetAssociation, + back_populates="copied_to_library_dataset_dataset_associations", + ), + copied_to_library_dataset_dataset_associations=relationship( + LibraryDatasetDatasetAssociation, primaryjoin=( LibraryDatasetDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id - == LibraryDatasetDatasetAssociation.table.c.id), - back_populates='copied_from_library_dataset_dataset_association'), - copied_to_history_dataset_associations=relationship(HistoryDatasetAssociation, - primaryjoin=(LibraryDatasetDatasetAssociation.table.c.id - == HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id), - back_populates='copied_from_library_dataset_dataset_association'), - implicitly_converted_datasets=relationship(ImplicitlyConvertedDatasetAssociation, - primaryjoin=(ImplicitlyConvertedDatasetAssociation.ldda_parent_id - == LibraryDatasetDatasetAssociation.table.c.id), - back_populates='parent_ldda'), - tags=relationship(LibraryDatasetDatasetAssociationTagAssociation, - order_by=LibraryDatasetDatasetAssociationTagAssociation.id, - back_populates='library_dataset_dataset_association'), - extended_metadata=relationship(ExtendedMetadata, - primaryjoin=(LibraryDatasetDatasetAssociation.table.c.extended_metadata_id - == ExtendedMetadata.id) + == LibraryDatasetDatasetAssociation.table.c.id + ), + back_populates="copied_from_library_dataset_dataset_association", + ), + copied_to_history_dataset_associations=relationship( + HistoryDatasetAssociation, + primaryjoin=( + LibraryDatasetDatasetAssociation.table.c.id + == HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id + ), + back_populates="copied_from_library_dataset_dataset_association", + ), + implicitly_converted_datasets=relationship( + ImplicitlyConvertedDatasetAssociation, + primaryjoin=( + ImplicitlyConvertedDatasetAssociation.ldda_parent_id == LibraryDatasetDatasetAssociation.table.c.id + ), + back_populates="parent_ldda", + ), + tags=relationship( + LibraryDatasetDatasetAssociationTagAssociation, + order_by=LibraryDatasetDatasetAssociationTagAssociation.id, + back_populates="library_dataset_dataset_association", + ), + extended_metadata=relationship( + ExtendedMetadata, + primaryjoin=(LibraryDatasetDatasetAssociation.table.c.extended_metadata_id == ExtendedMetadata.id), ), _metadata=deferred(LibraryDatasetDatasetAssociation.table.c._metadata), actions=relationship( - LibraryDatasetDatasetAssociationPermissions, - back_populates='library_dataset_dataset_association'), - dependent_jobs=relationship( - JobToInputLibraryDatasetAssociation, back_populates='dataset'), - creating_job_associations=relationship( - JobToOutputLibraryDatasetAssociation, back_populates='dataset'), - implicitly_converted_parent_datasets=relationship(ImplicitlyConvertedDatasetAssociation, - primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.ldda_id - == LibraryDatasetDatasetAssociation.id), - back_populates='dataset_ldda'), - copied_from_history_dataset_association=relationship(HistoryDatasetAssociation, - primaryjoin=(HistoryDatasetAssociation.table.c.id - == LibraryDatasetDatasetAssociation.table.c.copied_from_history_dataset_association_id), - back_populates='copied_to_library_dataset_dataset_associations'), - ) + LibraryDatasetDatasetAssociationPermissions, back_populates="library_dataset_dataset_association" + ), + dependent_jobs=relationship(JobToInputLibraryDatasetAssociation, back_populates="dataset"), + creating_job_associations=relationship(JobToOutputLibraryDatasetAssociation, back_populates="dataset"), + implicitly_converted_parent_datasets=relationship( + ImplicitlyConvertedDatasetAssociation, + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.ldda_id == LibraryDatasetDatasetAssociation.id), + back_populates="dataset_ldda", + ), + copied_from_history_dataset_association=relationship( + HistoryDatasetAssociation, + primaryjoin=( + HistoryDatasetAssociation.table.c.id + == LibraryDatasetDatasetAssociation.table.c.copied_from_history_dataset_association_id + ), + back_populates="copied_to_library_dataset_dataset_associations", + ), + ), ) # ---------------------------------------------------------------------------------------- # The following statements must not precede the mapped models defined above. Job.any_output_dataset_collection_instances_deleted = column_property( - exists(HistoryDatasetCollectionAssociation.id).where(and_( - Job.id == JobToOutputDatasetCollectionAssociation.job_id, - HistoryDatasetCollectionAssociation.id == JobToOutputDatasetCollectionAssociation.dataset_collection_id, - HistoryDatasetCollectionAssociation.deleted == true()) + exists(HistoryDatasetCollectionAssociation.id).where( + and_( + Job.id == JobToOutputDatasetCollectionAssociation.job_id, + HistoryDatasetCollectionAssociation.id == JobToOutputDatasetCollectionAssociation.dataset_collection_id, + HistoryDatasetCollectionAssociation.deleted == true(), + ) ) ) Job.any_output_dataset_deleted = column_property( - exists(HistoryDatasetAssociation).where(and_( - Job.id == JobToOutputDatasetAssociation.job_id, - HistoryDatasetAssociation.table.c.id == JobToOutputDatasetAssociation.dataset_id, - HistoryDatasetAssociation.table.c.deleted == true()) + exists(HistoryDatasetAssociation).where( + and_( + Job.id == JobToOutputDatasetAssociation.job_id, + HistoryDatasetAssociation.table.c.id == JobToOutputDatasetAssociation.dataset_id, + HistoryDatasetAssociation.table.c.deleted == true(), + ) ) ) History.average_rating = column_property( - select(func.avg(HistoryRatingAssociation.rating)).where( - HistoryRatingAssociation.history_id == History.id).scalar_subquery(), - deferred=True + select(func.avg(HistoryRatingAssociation.rating)) + .where(HistoryRatingAssociation.history_id == History.id) + .scalar_subquery(), + deferred=True, ) History.users_shared_with_count = column_property( - select(func.count(HistoryUserShareAssociation.id)).where( - History.id == HistoryUserShareAssociation.history_id).scalar_subquery(), - deferred=True + select(func.count(HistoryUserShareAssociation.id)) + .where(History.id == HistoryUserShareAssociation.history_id) + .scalar_subquery(), + deferred=True, ) Page.average_rating = column_property( - select(func.avg(PageRatingAssociation.rating)).where( - PageRatingAssociation.page_id == Page.id).scalar_subquery(), - deferred=True + select(func.avg(PageRatingAssociation.rating)).where(PageRatingAssociation.page_id == Page.id).scalar_subquery(), + deferred=True, ) StoredWorkflow.average_rating = column_property( - select(func.avg(StoredWorkflowRatingAssociation.rating)).where( - StoredWorkflowRatingAssociation.stored_workflow_id == StoredWorkflow.id).scalar_subquery(), - deferred=True + select(func.avg(StoredWorkflowRatingAssociation.rating)) + .where(StoredWorkflowRatingAssociation.stored_workflow_id == StoredWorkflow.id) + .scalar_subquery(), + deferred=True, ) Visualization.average_rating = column_property( - select(func.avg(VisualizationRatingAssociation.rating)).where( - VisualizationRatingAssociation.visualization_id == Visualization.id).scalar_subquery(), - deferred=True + select(func.avg(VisualizationRatingAssociation.rating)) + .where(VisualizationRatingAssociation.visualization_id == Visualization.id) + .scalar_subquery(), + deferred=True, ) Workflow.step_count = column_property( - select(func.count(WorkflowStep.id)).where(Workflow.id == WorkflowStep.workflow_id).scalar_subquery(), - deferred=True + select(func.count(WorkflowStep.id)).where(Workflow.id == WorkflowStep.workflow_id).scalar_subquery(), deferred=True ) WorkflowInvocationStep.subworkflow_invocation_id = column_property( - select(WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id).where(and_( - WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id == WorkflowInvocationStep.workflow_invocation_id, - WorkflowInvocationToSubworkflowInvocationAssociation.workflow_step_id == WorkflowInvocationStep.workflow_step_id, - )).scalar_subquery(), + select(WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id) + .where( + and_( + WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id + == WorkflowInvocationStep.workflow_invocation_id, + WorkflowInvocationToSubworkflowInvocationAssociation.workflow_step_id + == WorkflowInvocationStep.workflow_step_id, + ) + ) + .scalar_subquery(), ) # Set up proxy so that this syntax is possible: # .preferences[pref_name] = pref_value -User.preferences = association_proxy('_preferences', 'value', creator=UserPreference) +User.preferences = association_proxy("_preferences", "value", creator=UserPreference) diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index 2d762cfbf9a..6b854e2c23e 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -16,12 +16,12 @@ from sqlalchemy.types import ( CHAR, LargeBinary, String, - TypeDecorator + TypeDecorator, ) from galaxy.util import ( smart_str, - unicodify + unicodify, ) from galaxy.util.aliaspickler import AliasPickleModule @@ -52,9 +52,9 @@ def _sniffnfix_pg9_hex(value): Sniff for and fix postgres 9 hex decoding issue """ try: - if value[0] == 'x': + if value[0] == "x": return binascii.unhexlify(value[1:]) - elif smart_str(value).startswith(b'\\x'): + elif smart_str(value).startswith(b"\\x"): return binascii.unhexlify(value[2:]) else: return value @@ -72,10 +72,11 @@ class GalaxyLargeBinary(LargeBinary): def process(value): if value is not None: if isinstance(value, str): - value = bytes(value, encoding='utf-8') + value = bytes(value, encoding="utf-8") else: value = bytes(value) return value + return process @@ -113,7 +114,7 @@ class JSONType(TypeDecorator): return copy.deepcopy(value) def compare_values(self, x, y): - return (x == y) + return x == y class MutableJSONType(JSONType): @@ -173,20 +174,20 @@ class MutationObj(Mutable): def pickle(state, state_dict): val = state.dict.get(key, None) if isinstance(val, cls): - if 'ext.mutable.values' not in state_dict: - state_dict['ext.mutable.values'] = [] - state_dict['ext.mutable.values'].append(val) + if "ext.mutable.values" not in state_dict: + state_dict["ext.mutable.values"] = [] + state_dict["ext.mutable.values"].append(val) def unpickle(state, state_dict): - if 'ext.mutable.values' in state_dict: - for val in state_dict['ext.mutable.values']: + if "ext.mutable.values" in state_dict: + for val in state_dict["ext.mutable.values"]: val._parents[state] = key - sqlalchemy.event.listen(parent_cls, 'load', load, raw=True, propagate=True) - sqlalchemy.event.listen(parent_cls, 'refresh', load, raw=True, propagate=True) - sqlalchemy.event.listen(attribute, 'set', set, raw=True, retval=True, propagate=True) - sqlalchemy.event.listen(parent_cls, 'pickle', pickle, raw=True, propagate=True) - sqlalchemy.event.listen(parent_cls, 'unpickle', unpickle, raw=True, propagate=True) + sqlalchemy.event.listen(parent_cls, "load", load, raw=True, propagate=True) + sqlalchemy.event.listen(parent_cls, "refresh", load, raw=True, propagate=True) + sqlalchemy.event.listen(attribute, "set", set, raw=True, retval=True, propagate=True) + sqlalchemy.event.listen(parent_cls, "pickle", pickle, raw=True, propagate=True) + sqlalchemy.event.listen(parent_cls, "unpickle", unpickle, raw=True, propagate=True) class MutationDict(MutationObj, dict): @@ -279,13 +280,11 @@ class MutationList(MutationObj, list): MutationObj.associate_with(MutableJSONType) -metadata_pickler = AliasPickleModule({ - ("cookbook.patterns", "Bunch"): ("galaxy.util.bunch", "Bunch") -}) +metadata_pickler = AliasPickleModule({("cookbook.patterns", "Bunch"): ("galaxy.util.bunch", "Bunch")}) def total_size(o, handlers=None, verbose=False): - """ Returns the approximate memory footprint an object and all of its contents. + """Returns the approximate memory footprint an object and all of its contents. Automatically finds the contents of the following builtin containers and their subclasses: tuple, list, deque, dict, set and frozenset. @@ -301,18 +300,13 @@ def total_size(o, handlers=None, verbose=False): def dict_handler(d): return chain.from_iterable(d.items()) - all_handlers = {tuple: iter, - list: iter, - deque: iter, - dict: dict_handler, - set: iter, - frozenset: iter} - all_handlers.update(handlers) # user handlers take precedence - seen = set() # track which object id's have already been seen - default_size = getsizeof(0) # estimate sizeof object without __sizeof__ + all_handlers = {tuple: iter, list: iter, deque: iter, dict: dict_handler, set: iter, frozenset: iter} + all_handlers.update(handlers) # user handlers take precedence + seen = set() # track which object id's have already been seen + default_size = getsizeof(0) # estimate sizeof object without __sizeof__ def sizeof(o): - if id(o) in seen: # do not double count the same object + if id(o) in seen: # do not double count the same object return 0 seen.add(id(o)) s = getsizeof(o, default_size) @@ -339,7 +333,7 @@ class MetadataType(JSONType): sz = total_size(v) if sz > MAX_METADATA_VALUE_SIZE: del value[k] - log.warning(f'Refusing to bind metadata key {k} due to size ({sz})') + log.warning(f"Refusing to bind metadata key {k} due to size ({sz})") value = json_encoder.encode(value).encode() return value @@ -368,6 +362,7 @@ class UUIDType(TypeDecorator): CHAR(32), storing as stringified hex values. """ + impl = CHAR cache_ok = True @@ -396,5 +391,5 @@ class TrimmedString(TypeDecorator): def process_bind_param(self, value, dialect): """Automatically truncate string values""" if self.impl.length and value is not None: - value = unicodify(value)[0:self.impl.length] + value = unicodify(value)[0 : self.impl.length] return value diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 161c5513378..13085c84546 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -4,16 +4,30 @@ import os import re from abc import abstractmethod from json import dumps -from typing import Any, cast, Dict, List, Set, Union +from typing import ( + Any, + cast, + Dict, + List, + Set, + Union, +) from galaxy import model from galaxy.exceptions import ItemAccessibilityException from galaxy.job_execution.actions.post import ActionBox -from galaxy.model import LibraryDatasetDatasetAssociation, WorkflowRequestInputParameter +from galaxy.model import ( + LibraryDatasetDatasetAssociation, + WorkflowRequestInputParameter, +) from galaxy.model.dataset_collections.builder import CollectionBuilder from galaxy.model.none_like import NoneDataset from galaxy.tools.parameters import update_dataset_ids -from galaxy.tools.parameters.basic import DataCollectionToolParameter, DataToolParameter, RuntimeValue +from galaxy.tools.parameters.basic import ( + DataCollectionToolParameter, + DataToolParameter, + RuntimeValue, +) from galaxy.tools.parameters.wrapped import WrappedParameters from galaxy.util import ExecutionTimer from galaxy.util.template import fill_template @@ -23,7 +37,7 @@ log = logging.getLogger(__name__) class ToolExecutionCache: - """ An object mean to cache calculation caused by repeatedly evaluting + """An object mean to cache calculation caused by repeatedly evaluting the same tool by the same user with slightly different parameters. """ @@ -35,11 +49,15 @@ class ToolExecutionCache: def get_chrom_info(self, tool_id, input_dbkey): genome_builds = self.trans.app.genome_builds - custom_build_hack_get_len_from_fasta_conversion = tool_id != 'CONVERTER_fasta_to_len' + custom_build_hack_get_len_from_fasta_conversion = tool_id != "CONVERTER_fasta_to_len" if custom_build_hack_get_len_from_fasta_conversion and input_dbkey in self.chrom_info: return self.chrom_info[input_dbkey] - chrom_info_pair = genome_builds.get_chrom_info(input_dbkey, trans=self.trans, custom_build_hack_get_len_from_fasta_conversion=custom_build_hack_get_len_from_fasta_conversion) + chrom_info_pair = genome_builds.get_chrom_info( + input_dbkey, + trans=self.trans, + custom_build_hack_get_len_from_fasta_conversion=custom_build_hack_get_len_from_fasta_conversion, + ) if custom_build_hack_get_len_from_fasta_conversion: self.chrom_info[input_dbkey] = chrom_info_pair @@ -59,9 +77,19 @@ class ToolAction: class DefaultToolAction(ToolAction): """Default tool action is to run an external command""" + produces_real_jobs = True - def _collect_input_datasets(self, tool, param_values, trans, history, current_user_roles=None, dataset_collection_elements=None, collection_info=None): + def _collect_input_datasets( + self, + tool, + param_values, + trans, + history, + current_user_roles=None, + dataset_collection_elements=None, + collection_info=None, + ): """ Collect any dataset inputs from incoming. Returns a mapping from parameter name to Dataset instance for each tool parameter that is @@ -78,7 +106,6 @@ class DefaultToolAction(ToolAction): all_permissions[action].add(role_id) def visitor(input, value, prefix, parent=None, **kwargs): - def process_dataset(data, formats=None): if not data or isinstance(data, RuntimeValue): return None @@ -98,17 +125,22 @@ class DefaultToolAction(ToolAction): if collection_info and collection_info.is_mapped_over(input_name): action_tuples = collection_info.map_over_action_tuples(input_name) if not trans.app.security_agent.can_access_datasets(current_user_roles, action_tuples): - raise ItemAccessibilityException("User does not have permission to use a dataset provided for input.") + raise ItemAccessibilityException( + "User does not have permission to use a dataset provided for input." + ) for action, role_id in action_tuples: record_permission(action, role_id) else: if not trans.app.security_agent.can_access_dataset(current_user_roles, data.dataset): - raise ItemAccessibilityException(f"User does not have permission to use a dataset ({data.id}) provided for input.") + raise ItemAccessibilityException( + f"User does not have permission to use a dataset ({data.id}) provided for input." + ) permissions = trans.app.security_agent.get_permissions(data.dataset) for action, roles in permissions.items(): for role in roles: record_permission(action.action, model.cached_id(role)) return data + if isinstance(input, DataToolParameter): if isinstance(value, list): # If there are multiple inputs with the same name, they @@ -121,22 +153,30 @@ class DefaultToolAction(ToolAction): input_datasets[prefix + input.name + str(i + 1)] = processed_dataset conversions = [] for conversion_name, conversion_extensions, conversion_datatypes in input.conversions: - new_data = process_dataset(input_datasets[prefix + input.name + str(i + 1)], conversion_datatypes) + new_data = process_dataset( + input_datasets[prefix + input.name + str(i + 1)], conversion_datatypes + ) if not new_data or new_data.datatype.matches_any(conversion_datatypes): input_datasets[prefix + conversion_name + str(i + 1)] = new_data conversions.append((conversion_name, new_data)) else: - raise Exception(f'A path for explicit datatype conversion has not been found: {input_datasets[prefix + input.name + str(i + 1)].extension} --/--> {conversion_extensions}') + raise Exception( + f"A path for explicit datatype conversion has not been found: {input_datasets[prefix + input.name + str(i + 1)].extension} --/--> {conversion_extensions}" + ) if parent: parent[input.name][i] = input_datasets[prefix + input.name + str(i + 1)] for conversion_name, conversion_data in conversions: # allow explicit conversion to be stored in job_parameter table - parent[conversion_name][i] = conversion_data.id # a more robust way to determine JSONable value is desired + parent[conversion_name][ + i + ] = conversion_data.id # a more robust way to determine JSONable value is desired else: param_values[input.name][i] = input_datasets[prefix + input.name + str(i + 1)] for conversion_name, conversion_data in conversions: # allow explicit conversion to be stored in job_parameter table - param_values[conversion_name][i] = conversion_data.id # a more robust way to determine JSONable value is desired + param_values[conversion_name][ + i + ] = conversion_data.id # a more robust way to determine JSONable value is desired else: input_datasets[prefix + input.name] = process_dataset(value) conversions = [] @@ -146,21 +186,25 @@ class DefaultToolAction(ToolAction): input_datasets[prefix + conversion_name] = new_data conversions.append((conversion_name, new_data)) else: - raise Exception(f'A path for explicit datatype conversion has not been found: {input_datasets[prefix + input.name].extension} --/--> {conversion_extensions}') + raise Exception( + f"A path for explicit datatype conversion has not been found: {input_datasets[prefix + input.name].extension} --/--> {conversion_extensions}" + ) target_dict = parent if not target_dict: target_dict = param_values target_dict[input.name] = input_datasets[prefix + input.name] for conversion_name, conversion_data in conversions: # allow explicit conversion to be stored in job_parameter table - target_dict[conversion_name] = conversion_data.id # a more robust way to determine JSONable value is desired + target_dict[ + conversion_name + ] = conversion_data.id # a more robust way to determine JSONable value is desired elif isinstance(input, DataCollectionToolParameter): if not value: return collection = None child_collection = False - if hasattr(value, 'child_collection'): + if hasattr(value, "child_collection"): # if we are mapping a collection over a tool, we only require the child_collection child_collection = True collection = value.child_collection @@ -170,7 +214,9 @@ class DefaultToolAction(ToolAction): action_tuples = collection.dataset_action_tuples if not trans.app.security_agent.can_access_datasets(current_user_roles, action_tuples): - raise ItemAccessibilityException("User does not have permission to use a dataset provided for input.") + raise ItemAccessibilityException( + "User does not have permission to use a dataset provided for input." + ) for action, role_id in action_tuples: record_permission(action, role_id) @@ -191,7 +237,11 @@ class DefaultToolAction(ToolAction): processed_dataset_dict[v] = processed_dataset input_datasets[prefix + input.name + str(i + 1)] = processed_dataset or v if conversion_required: - collection_type_description = trans.app.dataset_collection_manager.collection_type_descriptions.for_collection_type(collection.collection_type) + collection_type_description = ( + trans.app.dataset_collection_manager.collection_type_descriptions.for_collection_type( + collection.collection_type + ) + ) collection_builder = CollectionBuilder(collection_type_description) collection_builder.replace_elements_in_collection( template_collection=collection, @@ -220,7 +270,9 @@ class DefaultToolAction(ToolAction): if not isinstance(values, list): values = [value] for i, value in enumerate(values): - if isinstance(value, model.HistoryDatasetCollectionAssociation) or isinstance(value, model.DatasetCollectionElement): + if isinstance(value, model.HistoryDatasetCollectionAssociation) or isinstance( + value, model.DatasetCollectionElement + ): append_to_key(input_dataset_collections, prefixed_name, (value, True)) target_dict = parent if not target_dict: @@ -248,7 +300,7 @@ class DefaultToolAction(ToolAction): assert tool.allow_user_access(trans.user), f"User ({trans.user}) is not allowed to access this tool." def _collect_inputs(self, tool, trans, incoming, history, current_user_roles, collection_info): - """ Collect history as well as input datasets and collections. """ + """Collect history as well as input datasets and collections.""" # Set history. if not history: history = tool.get_default_history_by_trans(trans, create=True) @@ -257,7 +309,14 @@ class DefaultToolAction(ToolAction): # input datasets can process these normally. inp_dataset_collections = self.collect_input_dataset_collections(tool, incoming) # Collect any input datasets from the incoming parameters - inp_data, all_permissions = self._collect_input_datasets(tool, incoming, trans, history=history, current_user_roles=current_user_roles, collection_info=collection_info) + inp_data, all_permissions = self._collect_input_datasets( + tool, + incoming, + trans, + history=history, + current_user_roles=current_user_roles, + collection_info=collection_info, + ) preserved_tags = {} preserved_hdca_tags = {} @@ -278,22 +337,23 @@ class DefaultToolAction(ToolAction): preserved_tags.update(preserved_hdca_tags) return history, inp_data, inp_dataset_collections, preserved_tags, preserved_hdca_tags, all_permissions - def execute(self, - tool, - trans, - incoming=None, - return_job=False, - set_output_hid=True, - history=None, - job_params=None, - rerun_remap_job_id=None, - execution_cache=None, - dataset_collection_elements=None, - completed_job=None, - collection_info=None, - job_callback=None, - flush_job=True - ): + def execute( + self, + tool, + trans, + incoming=None, + return_job=False, + set_output_hid=True, + history=None, + job_params=None, + rerun_remap_job_id=None, + execution_cache=None, + dataset_collection_elements=None, + completed_job=None, + collection_info=None, + job_callback=None, + flush_job=True, + ): """ Executes a tool, creating job and tool outputs, associating them, and submitting the job to the job queue. If history is not specified, use @@ -306,14 +366,21 @@ class DefaultToolAction(ToolAction): if execution_cache is None: execution_cache = ToolExecutionCache(trans) current_user_roles = execution_cache.current_user_roles - history, inp_data, inp_dataset_collections, preserved_tags, preserved_hdca_tags, all_permissions = self._collect_inputs(tool, trans, incoming, history, current_user_roles, collection_info) + ( + history, + inp_data, + inp_dataset_collections, + preserved_tags, + preserved_hdca_tags, + all_permissions, + ) = self._collect_inputs(tool, trans, incoming, history, current_user_roles, collection_info) # Build name for output datasets based on tool name and input names on_text = self._get_on_text(inp_data) # format='input" previously would give you a random extension from # the input extensions, now it should just give "input" as the output # format. - input_ext = 'data' if tool.profile < 16.04 else "input" + input_ext = "data" if tool.profile < 16.04 else "input" input_dbkey = incoming.get("dbkey", "?") for name, data in reversed(list(inp_data.items())): if not data: @@ -328,7 +395,7 @@ class DefaultToolAction(ToolAction): if tool.profile < 16.04: input_ext = data.ext - if data.dbkey not in [None, '?']: + if data.dbkey not in [None, "?"]: input_dbkey = data.dbkey identifier = getattr(data, "element_identifier", None) @@ -377,7 +444,7 @@ class DefaultToolAction(ToolAction): # datasets first, then create the associations parent_to_child_pairs = [] child_dataset_names = set() - async_tool = tool.tool_type == 'data_source_async' + async_tool = tool.tool_type == "data_source_async" def handle_output(name, output, hidden=None): if output.parent: @@ -410,7 +477,9 @@ class DefaultToolAction(ToolAction): dataset = output_dataset.dataset.dataset break - data = app.model.HistoryDatasetAssociation(extension=ext, dataset=dataset, create_dataset=create_datasets, flush=False) + data = app.model.HistoryDatasetAssociation( + extension=ext, dataset=dataset, create_dataset=create_datasets, flush=False + ) if create_datasets: from_work_dir = output.from_work_dir if from_work_dir is not None: @@ -425,7 +494,9 @@ class DefaultToolAction(ToolAction): dataset_collection_elements[name].hda = data trans.sa_session.add(data) if not completed_job: - trans.app.security_agent.set_all_dataset_permissions(data.dataset, output_permissions, new=True, flush=False) + trans.app.security_agent.set_all_dataset_permissions( + data.dataset, output_permissions, new=True, flush=False + ) data.copy_tags_to(preserved_tags.values()) # This may not be neccesary with the new parent/child associations @@ -453,7 +524,9 @@ class DefaultToolAction(ToolAction): else: data.blurb = "queued" # Set output label - data.name = self.get_output_name(output, data, tool, on_text, trans, incoming, history, wrapped_params.params, job_params) + data.name = self.get_output_name( + output, data, tool, on_text, trans, incoming, history, wrapped_params.params, job_params + ) # Store output out_data[name] = data if output.actions: @@ -462,7 +535,9 @@ class DefaultToolAction(ToolAction): output_action_params.update(incoming) output.actions.apply_action(data, output_action_params) # Also set the default values of actions of type metadata - self.set_metadata_defaults(output, data, tool, on_text, trans, incoming, history, wrapped_params.params, job_params) + self.set_metadata_defaults( + output, data, tool, on_text, trans, incoming, history, wrapped_params.params, job_params + ) # Flush all datasets at once. return data @@ -474,9 +549,7 @@ class DefaultToolAction(ToolAction): # Output collection is mapped over and has already been copied from original job continue collections_manager = app.dataset_collection_manager - element_identifiers: List[ - Dict[str, Union[str, List[Dict[str, Union[str, List[Any]]]]]] - ] = [] + element_identifiers: List[Dict[str, Union[str, List[Dict[str, Union[str, List[Any]]]]]]] = [] # mypy doesn't yet support recursive type definitions known_outputs = output.known_outputs(input_collections, collections_manager.type_registry) # Just to echo TODO elsewhere - this should be restructured to allow @@ -486,33 +559,33 @@ class DefaultToolAction(ToolAction): current_element_identifiers = element_identifiers current_collection_type = output.structure.collection_type - for parent_id in (output_part_def.parent_ids or []): + for parent_id in output_part_def.parent_ids or []: # TODO: replace following line with formal abstractions for doing this. current_collection_type = ":".join(current_collection_type.split(":")[1:]) - name_to_index = {value["name"]: index for (index, value) in enumerate(current_element_identifiers)} + name_to_index = { + value["name"]: index for (index, value) in enumerate(current_element_identifiers) + } if parent_id not in name_to_index: if parent_id not in current_element_identifiers: index = len(current_element_identifiers) - current_element_identifiers.append(dict( - name=parent_id, - collection_type=current_collection_type, - src="new_collection", - element_identifiers=[], - )) + current_element_identifiers.append( + dict( + name=parent_id, + collection_type=current_collection_type, + src="new_collection", + element_identifiers=[], + ) + ) else: index = name_to_index[parent_id] current_element_identifiers = cast( List[ Dict[ str, - Union[ - str, List[Dict[str, Union[str, List[Any]]]] - ], + Union[str, List[Dict[str, Union[str, List[Any]]]]], ] ], - current_element_identifiers[index][ - "element_identifiers" - ], + current_element_identifiers[index]["element_identifiers"], ) effective_output_name = output_part_def.effective_output_name @@ -524,10 +597,12 @@ class DefaultToolAction(ToolAction): # Following hack causes dataset to no be added to history... child_dataset_names.add(effective_output_name) trans.sa_session.add(element) - current_element_identifiers.append({ - "__object__": element, - "name": output_part_def.element_identifier, - }) + current_element_identifiers.append( + { + "__object__": element, + "name": output_part_def.element_identifier, + } + ) if output.dynamic_structure: assert not element_identifiers # known_outputs must have been empty @@ -535,10 +610,7 @@ class DefaultToolAction(ToolAction): else: element_kwds = dict(element_identifiers=element_identifiers) output_collections.create_collection( - output=output, - name=name, - completed_job=completed_job, - **element_kwds + output=output, name=name, completed_job=completed_job, **element_kwds ) log.info(f"Handled collection output named {name} for tool {tool.id} {handle_output_timer}") else: @@ -546,12 +618,14 @@ class DefaultToolAction(ToolAction): log.info(f"Handled output named {name} for tool {tool.id} {handle_output_timer}") add_datasets_timer = tool.app.execution_timer_factory.get_timer( - 'internals.galaxy.tools.actions.add_datasets', - 'Added output datasets to history', + "internals.galaxy.tools.actions.add_datasets", + "Added output datasets to history", ) # Add all the top-level (non-child) datasets to the history unless otherwise specified for name, data in out_data.items(): - if name not in child_dataset_names and name not in incoming: # don't add children; or already existing datasets, i.e. async created + if ( + name not in child_dataset_names and name not in incoming + ): # don't add children; or already existing datasets, i.e. async created history.stage_addition(data) history.add_pending_items(set_output_hid=set_output_hid) @@ -585,25 +659,27 @@ class DefaultToolAction(ToolAction): session.flush() finally: session.expire_on_commit = True - self._remap_job_on_rerun(trans=trans, - galaxy_session=galaxy_session, - rerun_remap_job_id=rerun_remap_job_id, - current_job=job, - out_data=out_data) + self._remap_job_on_rerun( + trans=trans, + galaxy_session=galaxy_session, + rerun_remap_job_id=rerun_remap_job_id, + current_job=job, + out_data=out_data, + ) log.info(f"Setup for job {job.log_str()} complete, ready to be enqueued {job_setup_timer}") # Some tools are not really executable, but jobs are still created for them ( for record keeping ). # Examples include tools that redirect to other applications ( epigraph ). These special tools must # include something that can be retrieved from the params ( e.g., REDIRECT_URL ) to keep the job # from being queued. - if 'REDIRECT_URL' in incoming: + if "REDIRECT_URL" in incoming: # Get the dataset - there should only be 1 for name in inp_data.keys(): dataset = inp_data[name] redirect_url = tool.parse_redirect_url(dataset, incoming) # GALAXY_URL should be include in the tool params to enable the external application # to send back to the current Galaxy instance - GALAXY_URL = incoming.get('GALAXY_URL', None) + GALAXY_URL = incoming.get("GALAXY_URL", None) assert GALAXY_URL is not None, "GALAXY_URL parameter missing in tool config." redirect_url += f"&GALAXY_URL={GALAXY_URL}" # Job should not be queued, so set state to ok @@ -611,7 +687,9 @@ class DefaultToolAction(ToolAction): job.info = f"Redirected to: {redirect_url}" trans.sa_session.add(job) trans.sa_session.flush() - trans.response.send_redirect(url_for(controller='tool_runner', action='redirect', redirect_url=redirect_url)) + trans.response.send_redirect( + url_for(controller="tool_runner", action="redirect", redirect_url=redirect_url) + ) else: if flush_job: # Set HID and add to history. @@ -631,14 +709,20 @@ class DefaultToolAction(ToolAction): """ try: old_job = trans.sa_session.query(trans.app.model.Job).get(rerun_remap_job_id) - assert old_job is not None, f'({rerun_remap_job_id}/{current_job.id}): Old job id is invalid' - assert old_job.tool_id == current_job.tool_id, f'({old_job.id}/{current_job.id}): Old tool id ({old_job.tool_id}) does not match rerun tool id ({current_job.tool_id})' + assert old_job is not None, f"({rerun_remap_job_id}/{current_job.id}): Old job id is invalid" + assert ( + old_job.tool_id == current_job.tool_id + ), f"({old_job.id}/{current_job.id}): Old tool id ({old_job.tool_id}) does not match rerun tool id ({current_job.tool_id})" if trans.user is not None: - assert old_job.user_id == trans.user.id, f'({old_job.id}/{current_job.id}): Old user id ({old_job.user_id}) does not match rerun user id ({trans.user.id})' + assert ( + old_job.user_id == trans.user.id + ), f"({old_job.id}/{current_job.id}): Old user id ({old_job.user_id}) does not match rerun user id ({trans.user.id})" elif trans.user is None and type(galaxy_session) == trans.model.GalaxySession: - assert old_job.session_id == galaxy_session.id, f'({old_job.id}/{current_job.id}): Old session id ({old_job.session_id}) does not match rerun session id ({galaxy_session.id})' + assert ( + old_job.session_id == galaxy_session.id + ), f"({old_job.id}/{current_job.id}): Old session id ({old_job.session_id}) does not match rerun session id ({galaxy_session.id})" else: - raise Exception(f'({old_job.id}/{current_job.id}): Remapping via the API is not (yet) supported') + raise Exception(f"({old_job.id}/{current_job.id}): Remapping via the API is not (yet) supported") # Start by hiding current job outputs before taking over the old job's (implicit) outputs. current_job.hide_outputs(flush=False) # Duplicate PJAs before remap. @@ -654,13 +738,14 @@ class DefaultToolAction(ToolAction): if pja.action_type in ActionBox.immediate_actions: ActionBox.execute(trans.app, trans.sa_session, pja, current_job, replacement_dict) for p in old_job.parameters: - if p.name.endswith('|__identifier__'): + if p.name.endswith("|__identifier__"): current_job.parameters.append(p.copy()) remapped_hdas = self.__remap_data_inputs(old_job=old_job, current_job=current_job) for jtod in old_job.output_datasets: for (job_to_remap, jtid) in [(jtid.job, jtid) for jtid in jtod.dataset.dependent_jobs]: if (trans.user is not None and job_to_remap.user_id == trans.user.id) or ( - trans.user is None and job_to_remap.session_id == galaxy_session.id): + trans.user is None and job_to_remap.session_id == galaxy_session.id + ): self.__remap_parameters(job_to_remap, jtid, jtod, out_data) trans.sa_session.add(job_to_remap) trans.sa_session.add(jtid) @@ -680,7 +765,7 @@ class DefaultToolAction(ToolAction): for jtoidca in old_job.output_dataset_collections: jtoidca.dataset_collection.replace_failed_elements(remapped_hdas) except Exception: - log.exception('Cannot remap rerun dependencies.') + log.exception("Cannot remap rerun dependencies.") def __remap_data_inputs(self, old_job, current_job): """Record output datasets from old_job and build a dictionary that maps the old output HDAs to the new output HDAs.""" @@ -694,13 +779,13 @@ class DefaultToolAction(ToolAction): input_values = {p.name: json.loads(p.value) for p in job_to_remap.parameters if p.value is not None} old_dataset_id = jtod.dataset_id new_dataset_id = out_data[jtod.name].id - input_values = update_dataset_ids(input_values, {old_dataset_id: new_dataset_id}, src='hda') + input_values = update_dataset_ids(input_values, {old_dataset_id: new_dataset_id}, src="hda") for p in job_to_remap.parameters: if p.name in input_values: p.value = json.dumps(input_values[p.name]) jtid.dataset = out_data[jtod.name] jtid.dataset.hid = jtod.dataset.hid - log.info(f'Job {job_to_remap.id} input HDA {jtod.dataset.id} remapped to new HDA {jtid.dataset.id}') + log.info(f"Job {job_to_remap.id} input HDA {jtod.dataset.id} remapped to new HDA {jtid.dataset.id}") def _wrapped_params(self, trans, tool, incoming, input_datasets=None): wrapped_params = WrappedParameters(trans, tool, incoming, input_datasets=input_datasets) @@ -710,7 +795,7 @@ class DefaultToolAction(ToolAction): input_names = [] for data in reversed(list(inp_data.values())): if getattr(data, "hid", None): - input_names.append(f'data {data.hid}') + input_names.append(f"data {data.hid}") return on_text_for_names(input_names) @@ -767,9 +852,9 @@ class DefaultToolAction(ToolAction): target_dict[input.name] = [] for reduced_collection in reductions[prefixed_name]: if hasattr(reduced_collection, "child_collection"): - target_dict[input.name].append({'id': model.cached_id(reduced_collection), 'src': 'dce'}) + target_dict[input.name].append({"id": model.cached_id(reduced_collection), "src": "dce"}) else: - target_dict[input.name].append({'id': model.cached_id(reduced_collection), 'src': 'hdca'}) + target_dict[input.name].append({"id": model.cached_id(reduced_collection), "src": "hdca"}) if reductions: tool.visit_inputs(incoming, restore_reduction_visitor) @@ -794,13 +879,33 @@ class DefaultToolAction(ToolAction): # TODO: figure out why can't pass dataset_id here. job.add_input_dataset(name, dataset=dataset) - def get_output_name(self, output, dataset=None, tool=None, on_text=None, trans=None, incoming=None, history=None, params=None, job_params=None): + def get_output_name( + self, + output, + dataset=None, + tool=None, + on_text=None, + trans=None, + incoming=None, + history=None, + params=None, + job_params=None, + ): if output.label: - params['tool'] = tool - params['on_string'] = on_text + params["tool"] = tool + params["on_string"] = on_text return fill_template(output.label, context=params, python_template_version=tool.python_template_version) else: - return self._get_default_data_name(dataset, tool, on_text=on_text, trans=trans, incoming=incoming, history=history, params=params, job_params=job_params) + return self._get_default_data_name( + dataset, + tool, + on_text=on_text, + trans=trans, + incoming=incoming, + history=history, + params=params, + job_params=job_params, + ) def set_metadata_defaults(self, output, dataset, tool, on_text, trans, incoming, history, params, job_params): """ @@ -817,10 +922,14 @@ class DefaultToolAction(ToolAction): if output.actions: for action in output.actions.actions: if action.tag == "metadata" and action.default: - metadata_new_value = fill_template(action.default, context=params, python_template_version=tool.python_template_version).split(",") + metadata_new_value = fill_template( + action.default, context=params, python_template_version=tool.python_template_version + ).split(",") dataset.metadata.__setattr__(str(action.name), metadata_new_value) - def _get_default_data_name(self, dataset, tool, on_text=None, trans=None, incoming=None, history=None, params=None, job_params=None, **kwd): + def _get_default_data_name( + self, dataset, tool, on_text=None, trans=None, incoming=None, history=None, params=None, job_params=None, **kwd + ): name = tool.name if on_text: name += f" on {on_text}" @@ -828,14 +937,28 @@ class DefaultToolAction(ToolAction): class OutputCollections: - """ Keeps track of collections (DC or HDCA) created by actions. + """Keeps track of collections (DC or HDCA) created by actions. Actions do fairly different things depending on whether we are creating just part of an collection or a whole output collection (mapping_over_collection parameter). """ - def __init__(self, trans, history, tool, tool_action, input_collections, dataset_collection_elements, on_text, incoming, params, job_params, tags, hdca_tags): + def __init__( + self, + trans, + history, + tool, + tool_action, + input_collections, + dataset_collection_elements, + on_text, + incoming, + params, + job_params, + tags, + hdca_tags, + ): self.trans = trans self.tag_handler = trans.app.tag_handler.create_tag_handler_session() self.history = history @@ -852,7 +975,9 @@ class OutputCollections: self.tags = tags # all inherited tags self.hdca_tags = hdca_tags # only tags inherited from input HDCAs - def create_collection(self, output, name, collection_type=None, completed_job=None, propagate_hda_tags=True, **element_kwds): + def create_collection( + self, output, name, collection_type=None, completed_job=None, propagate_hda_tags=True, **element_kwds + ): input_collections = self.input_collections collections_manager = self.trans.app.dataset_collection_manager collection_type = collection_type or output.structure.collection_type @@ -867,11 +992,11 @@ class OutputCollections: # Using the collection_type_source string we get the DataCollectionToolParameter data_param = self.tool.inputs - groups = collection_type_source.split('|') + groups = collection_type_source.split("|") for group in groups: - values = group.split('_') + values = group.split("_") if values[-1].isdigit(): - key = ("_".join(values[0:-1])) + key = "_".join(values[0:-1]) # We don't care about the repeat index, we just need to find the correct DataCollectionToolParameter else: key = group @@ -879,13 +1004,16 @@ class OutputCollections: data_param = data_param.get(key) else: data_param = data_param.inputs.get(key) - collection_type_description = data_param._history_query(self.trans).can_map_over(input_collections[collection_type_source]) + collection_type_description = data_param._history_query(self.trans).can_map_over( + input_collections[collection_type_source] + ) if collection_type_description: collection_type = collection_type_description.collection_type else: collection_type = input_collections[collection_type_source].collection.collection_type if "elements" in element_kwds: + def check_elements(elements): if hasattr(elements, "items"): # else it is ELEMENTS_UNINITIALIZED object. for value in elements.values(): @@ -904,9 +1032,7 @@ class OutputCollections: if self.dataset_collection_elements is not None: dc = collections_manager.create_dataset_collection( - self.trans, - collection_type=collection_type, - **element_kwds + self.trans, collection_type=collection_type, **element_kwds ) if name in self.dataset_collection_elements: self.dataset_collection_elements[name].child_collection = dc @@ -935,7 +1061,7 @@ class OutputCollections: flush=False, completed_job=completed_job, output_name=name, - **element_kwds + **element_kwds, ) # name here is name of the output element - not name # of the hdca. @@ -957,11 +1083,11 @@ def on_text_for_names(input_names): if len(input_names) == 1: on_text = input_names[0] elif len(input_names) == 2: - on_text = '%s and %s' % tuple(input_names[0:2]) + on_text = "%s and %s" % tuple(input_names[0:2]) elif len(input_names) == 3: - on_text = '%s, %s, and %s' % tuple(input_names[0:3]) + on_text = "%s, %s, and %s" % tuple(input_names[0:3]) elif len(input_names) > 3: - on_text = '%s, %s, and others' % tuple(input_names[0:2]) + on_text = "%s, %s, and others" % tuple(input_names[0:2]) else: on_text = "" return on_text @@ -973,7 +1099,7 @@ def filter_output(tool, output, incoming): if not eval(filter.text.strip(), globals(), incoming): return True # do not create this dataset except Exception as e: - log.debug(f'Tool {tool.id} output {output.name}: dataset output filter ({filter.text}) failed: {e}') + log.debug(f"Tool {tool.id} output {output.name}: dataset output filter ({filter.text}) failed: {e}") return False @@ -987,8 +1113,16 @@ def get_ext_or_implicit_ext(hda): return hda.ext -def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext, python_template_version='3', execution_cache=None): - """ Determines the output format for a dataset based on an abstract +def determine_output_format( + output, + parameter_context, + input_datasets, + input_dataset_collections, + random_input_ext, + python_template_version="3", + execution_cache=None, +): + """Determines the output format for a dataset based on an abstract description of the output (galaxy.tool_util.parser.ToolOutput), the parameter wrappers, a map of the input datasets (name => HDA), and the last input extensions in the tool form. @@ -999,7 +1133,7 @@ def determine_output_format(output, parameter_context, input_datasets, input_dat # the type should match the input ext = output.format if ext == "input": - if input_datasets and random_input_ext in {'data', 'auto'}: + if input_datasets and random_input_ext in {"data", "auto"}: # Probably dealing with an implicitly converted dataset try: first_input_dataset = next(iter(input_datasets.values())) @@ -1036,9 +1170,13 @@ def determine_output_format(output, parameter_context, input_datasets, input_dat input_element = input_collection_collection[element_index] except KeyError: if execution_cache: - dataset_elements = execution_cache.cached_collection_elements.get(input_collection_collection.id) + dataset_elements = execution_cache.cached_collection_elements.get( + input_collection_collection.id + ) if dataset_elements is None: - dataset_elements = execution_cache.cached_collection_elements[input_collection_collection.id] = input_collection_collection.dataset_elements + dataset_elements = execution_cache.cached_collection_elements[ + input_collection_collection.id + ] = input_collection_collection.dataset_elements else: dataset_elements = input_collection_collection.dataset_elements for element in dataset_elements: @@ -1054,25 +1192,27 @@ def determine_output_format(output, parameter_context, input_datasets, input_dat if output.change_format is not None: new_format_set = False for change_elem in output.change_format: - for when_elem in change_elem.findall('when'): - check = when_elem.get('input', None) + for when_elem in change_elem.findall("when"): + check = when_elem.get("input", None) if check is not None: try: - if '$' not in check: + if "$" not in check: # allow a simple name or more complex specifications - check = '${%s}' % check - if fill_template(check, context=parameter_context, python_template_version=python_template_version) == when_elem.get('value', None): - ext = when_elem.get('format', ext) + check = "${%s}" % check + if fill_template( + check, context=parameter_context, python_template_version=python_template_version + ) == when_elem.get("value", None): + ext = when_elem.get("format", ext) except Exception: # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct continue else: - check = when_elem.get('input_dataset', None) + check = when_elem.get("input_dataset", None) if check is not None: check = input_datasets.get(check, None) # At this point check is a HistoryDatasetAssociation object. - check_format = when_elem.get('format', ext) - check_value = when_elem.get('value', None) - check_attribute = when_elem.get('attribute', None) + check_format = when_elem.get("format", ext) + check_value = when_elem.get("value", None) + check_attribute = when_elem.get("attribute", None) if check is not None and check_value is not None and check_attribute is not None: # See if the attribute to be checked belongs to the HistoryDatasetAssociation object. if hasattr(check, check_attribute): diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 29d8c6b008a..4f1a34fc6fc 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -5,7 +5,13 @@ import shlex import string import tempfile from datetime import datetime -from typing import Any, Callable, Dict, List, Optional +from typing import ( + Any, + Callable, + Dict, + List, + Optional, +) from galaxy import model from galaxy.job_execution.compute_environment import ComputeEnvironment @@ -24,7 +30,7 @@ from galaxy.tools.parameters.basic import ( from galaxy.tools.parameters.grouping import ( Conditional, Repeat, - Section + Section, ) from galaxy.tools.wrappers import ( DatasetCollectionWrapper, @@ -55,12 +61,9 @@ class ToolErrorLog: self.max_errors = 100 def add_error(self, file, phase, exception): - self.error_stack.insert(0, { - "file": file, - "time": str(datetime.now()), - "phase": phase, - "error": unicodify(exception) - }) + self.error_stack.insert( + 0, {"file": file, "time": str(datetime.now()), "phase": phase, "error": unicodify(exception)} + ) if len(self.error_stack) > self.max_errors: self.error_stack.pop() @@ -78,7 +81,7 @@ def global_tool_logs(func, config_file, action_str): class ToolEvaluator: - """ An abstraction linking together a tool and a job runtime to evaluate + """An abstraction linking together a tool and a job runtime to evaluate tool inputs in an isolated, testable manner. """ @@ -112,6 +115,7 @@ class ToolEvaluator: def validate_inputs(input, value, context, **kwargs): value = input.from_json(value, request_context, context) input.validate(value, request_context) + visit_input_values(self.tool.inputs, incoming, validate_inputs) # Restore input / output data lists @@ -139,8 +143,9 @@ class ToolEvaluator: # ( this used to be performed in the "exec_before_job" hook, but hooks are deprecated ). self.tool.exec_before_job(self.app, inp_data, out_data, self.param_dict) # Run the before queue ("exec_before_job") hook - self.tool.call_hook('exec_before_job', self.app, inp_data=inp_data, - out_data=out_data, tool=self.tool, param_dict=incoming) + self.tool.call_hook( + "exec_before_job", self.app, inp_data=inp_data, out_data=out_data, tool=self.tool, param_dict=incoming + ) def build_param_dict(self, incoming, input_datasets, output_datasets, output_collections): """ @@ -159,12 +164,14 @@ class ToolEvaluator: raise SyntaxError("Unbound variable input.") # Don't let $input hang Python evaluation process. param_dict["input"] = input - param_dict['__datatypes_config__'] = param_dict['GALAXY_DATATYPES_CONF_FILE'] = os.path.join(job_working_directory, 'registry.xml') - if self.job.tool_id == 'upload1': - param_dict['paramfile'] = os.path.join(job_working_directory, 'upload_params.json') + param_dict["__datatypes_config__"] = param_dict["GALAXY_DATATYPES_CONF_FILE"] = os.path.join( + job_working_directory, "registry.xml" + ) + if self.job.tool_id == "upload1": + param_dict["paramfile"] = os.path.join(job_working_directory, "upload_params.json") if self._history: - param_dict['__history_id__'] = self.app.security.encode_id(self._history.id) - param_dict['__galaxy_url__'] = self.compute_environment.galaxy_url() + param_dict["__history_id__"] = self.app.security.encode_id(self._history.id) + param_dict["__galaxy_url__"] = self.compute_environment.galaxy_url() param_dict.update(self.tool.template_macro_params) # All parameters go into the param_dict param_dict.update(incoming) @@ -183,7 +190,6 @@ class ToolEvaluator: return param_dict def __walk_inputs(self, inputs, input_values, func): - def do_walk(inputs, input_values): """ Wraps parameters as neccesary. @@ -206,19 +212,19 @@ class ToolEvaluator: do_walk(inputs, input_values) def __populate_wrappers(self, param_dict, input_datasets, job_working_directory): - def wrap_input(input_values, input): value = input_values[input.name] if isinstance(input, DataToolParameter) and input.multiple: dataset_instances = DatasetListWrapper.to_dataset_instances(value) - input_values[input.name] = \ - DatasetListWrapper(job_working_directory, - dataset_instances, - compute_environment=self.compute_environment, - datatypes_registry=self.app.datatypes_registry, - tool=self.tool, - name=input.name, - formats=input.formats) + input_values[input.name] = DatasetListWrapper( + job_working_directory, + dataset_instances, + compute_environment=self.compute_environment, + datatypes_registry=self.app.datatypes_registry, + tool=self.tool, + name=input.name, + formats=input.formats, + ) elif isinstance(input, DataToolParameter): dataset = input_values[input.name] @@ -226,35 +232,30 @@ class ToolEvaluator: datatypes_registry=self.app.datatypes_registry, tool=self, name=input.name, - compute_environment=self.compute_environment + compute_environment=self.compute_environment, ) element_identifier = element_identifier_mapper.identifier(dataset, param_dict) if element_identifier: wrapper_kwds["identifier"] = element_identifier - input_values[input.name] = \ - DatasetFilenameWrapper(dataset, **wrapper_kwds) + input_values[input.name] = DatasetFilenameWrapper(dataset, **wrapper_kwds) elif isinstance(input, DataCollectionToolParameter): dataset_collection = value wrapper_kwds = dict( datatypes_registry=self.app.datatypes_registry, compute_environment=self.compute_environment, tool=self, - name=input.name - ) - wrapper = DatasetCollectionWrapper( - job_working_directory, - dataset_collection, - **wrapper_kwds + name=input.name, ) + wrapper = DatasetCollectionWrapper(job_working_directory, dataset_collection, **wrapper_kwds) input_values[input.name] = wrapper elif isinstance(input, SelectToolParameter): if input.multiple: value = listify(value) input_values[input.name] = SelectToolParameterWrapper( - input, value, other_values=param_dict, compute_environment=self.compute_environment) + input, value, other_values=param_dict, compute_environment=self.compute_environment + ) else: - input_values[input.name] = InputValueWrapper( - input, value, param_dict) + input_values[input.name] = InputValueWrapper(input, value, param_dict) # HACK: only wrap if check_values is not false, this deals with external # tools where the inputs don't even get passed through. These @@ -309,15 +310,11 @@ class ToolEvaluator: wrapper_kwds = dict( datatypes_registry=self.app.datatypes_registry, compute_environment=self.compute_environment, - io_type='output', + io_type="output", tool=tool, - name=name - ) - wrapper = DatasetCollectionWrapper( - job_working_directory, - out_collection, - **wrapper_kwds + name=name, ) + wrapper = DatasetCollectionWrapper(job_working_directory, out_collection, **wrapper_kwds) param_dict[name] = wrapper # TODO: Handle nested collections... output_def = tool.output_collections[name] @@ -331,9 +328,11 @@ class ToolEvaluator: for name, hda in output_datasets.items(): # Write outputs to the working directory (for security purposes) # if desired. - param_dict[name] = DatasetFilenameWrapper(hda, compute_environment=self.compute_environment, io_type="output") - if '|__part__|' in name: - unqualified_name = name.split('|__part__|')[-1] + param_dict[name] = DatasetFilenameWrapper( + hda, compute_environment=self.compute_environment, io_type="output" + ) + if "|__part__|" in name: + unqualified_name = name.split("|__part__|")[-1] if unqualified_name not in param_dict: param_dict[unqualified_name] = param_dict[name] output_path = str(param_dict[name]) @@ -341,7 +340,7 @@ class ToolEvaluator: # - may already exist (e.g. symlink output) # - parent directory might not exist (e.g. Pulsar) if not os.path.exists(output_path) and os.path.exists(os.path.dirname(output_path)): - open(output_path, 'w').close() + open(output_path, "w").close() # Provide access to a path to store additional files # TODO: move compute path logic into compute environment, move setting files_path @@ -368,30 +367,33 @@ class ToolEvaluator: if table_name in self.app.tool_data_tables: return self.app.tool_data_tables[table_name].get_entry(query_attr, query_val, return_attr) - param_dict['__tool_directory__'] = self.compute_environment.tool_directory() - param_dict['__get_data_table_entry__'] = get_data_table_entry - param_dict['__local_working_directory__'] = self.local_working_directory + param_dict["__tool_directory__"] = self.compute_environment.tool_directory() + param_dict["__get_data_table_entry__"] = get_data_table_entry + param_dict["__local_working_directory__"] = self.local_working_directory # We add access to app here, this allows access to app.config, etc - param_dict['__app__'] = RawObjectWrapper(self.app) + param_dict["__app__"] = RawObjectWrapper(self.app) # More convienent access to app.config.new_file_path; we don't need to # wrap a string, but this method of generating additional datasets # should be considered DEPRECATED - param_dict['__new_file_path__'] = self.compute_environment.new_file_path() + param_dict["__new_file_path__"] = self.compute_environment.new_file_path() # The following points to location (xxx.loc) files which are pointers # to locally cached data - param_dict['__tool_data_path__'] = param_dict['GALAXY_DATA_INDEX_DIR'] = self.app.config.tool_data_path + param_dict["__tool_data_path__"] = param_dict["GALAXY_DATA_INDEX_DIR"] = self.app.config.tool_data_path # For the upload tool, we need to know the root directory and the # datatypes conf path, so we can load the datatypes registry - param_dict['__root_dir__'] = param_dict['GALAXY_ROOT_DIR'] = os.path.abspath(self.app.config.root) - param_dict['__admin_users__'] = self.app.config.admin_users - param_dict['__user__'] = RawObjectWrapper(param_dict.get('__user__', None)) + param_dict["__root_dir__"] = param_dict["GALAXY_ROOT_DIR"] = os.path.abspath(self.app.config.root) + param_dict["__admin_users__"] = self.app.config.admin_users + param_dict["__user__"] = RawObjectWrapper(param_dict.get("__user__", None)) def __populate_unstructured_path_rewrites(self, param_dict): - def rewrite_unstructured_paths(input_values, input): if isinstance(input, SelectToolParameter): input_values[input.name] = SelectToolParameterWrapper( - input, input_values[input.name], other_values=param_dict, compute_environment=self.compute_environment) + input, + input_values[input.name], + other_values=param_dict, + compute_environment=self.compute_environment, + ) if not self.tool.check_values and self.compute_environment: # The tools weren't "wrapped" yet, but need to be in order to get @@ -403,16 +405,18 @@ class ToolEvaluator: Populate InteractiveTools templated values. """ it = [] - for ep in getattr(self.tool, 'ports', []): + for ep in getattr(self.tool, "ports", []): ep_dict = {} - for key in 'port', 'name', 'url', 'requires_domain': + for key in "port", "name", "url", "requires_domain": val = ep.get(key, None) if val is not None and not isinstance(val, bool): - val = fill_template(val, context=self.param_dict, python_template_version=self.tool.python_template_version) + val = fill_template( + val, context=self.param_dict, python_template_version=self.tool.python_template_version + ) clean_val = [] - for line in val.split('\n'): + for line in val.split("\n"): clean_val.append(line.strip()) - val = '\n'.join(clean_val) + val = "\n".join(clean_val) val = val.replace("\n", " ").replace("\r", " ").strip() ep_dict[key] = val it.append(ep_dict) @@ -427,14 +431,16 @@ class ToolEvaluator: Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. """ # chromInfo is a filename, do not sanitize it. - skip = ['chromInfo'] + list(self.tool.template_macro_params.keys()) + skip = ["chromInfo"] + list(self.tool.template_macro_params.keys()) if not self.tool or not self.tool.options or self.tool.options.sanitize: for key, value in list(param_dict.items()): if key not in skip: # Remove key so that new wrapped object will occupy key slot del param_dict[key] # And replace with new wrapped key - param_dict[wrap_with_safe_string(key, no_wrap_classes=ToolParameterValueWrapper)] = wrap_with_safe_string(value, no_wrap_classes=ToolParameterValueWrapper) + param_dict[ + wrap_with_safe_string(key, no_wrap_classes=ToolParameterValueWrapper) + ] = wrap_with_safe_string(value, no_wrap_classes=ToolParameterValueWrapper) def build(self): """ @@ -444,7 +450,7 @@ class ToolEvaluator: """ config_file = self.tool.config_file global_tool_logs(self._build_config_files, config_file, "Building Config Files") - global_tool_logs(self._build_param_file, config_file, 'Building Param File') + global_tool_logs(self._build_param_file, config_file, "Building Param File") global_tool_logs(self._build_command_line, config_file, "Building Command Line") global_tool_logs(self._build_version_command, config_file, "Building Version Command Line") global_tool_logs(self._build_environment_variables, config_file, "Building Environment Variables") @@ -454,7 +460,7 @@ class ToolEvaluator: """ Build command line to invoke this tool given a populated param_dict """ - command = self.tool.command or '' + command = self.tool.command or "" param_dict = self.param_dict interpreter = self.tool.interpreter command_line = None @@ -462,12 +468,14 @@ class ToolEvaluator: return try: # Substituting parameters into the command - command_line = fill_template(command, context=param_dict, python_template_version=self.tool.python_template_version) + command_line = fill_template( + command, context=param_dict, python_template_version=self.tool.python_template_version + ) cleaned_command_line = [] # Remove leading and trailing whitespace from each line for readability. - for line in command_line.split('\n'): + for line in command_line.split("\n"): cleaned_command_line.append(line.strip()) - command_line = '\n'.join(cleaned_command_line) + command_line = "\n".join(cleaned_command_line) # Remove newlines from command line, and any leading/trailing white space command_line = command_line.replace("\n", " ").replace("\r", " ").strip() except Exception: @@ -486,7 +494,9 @@ class ToolEvaluator: version_string_cmd_raw = self.tool.version_string_cmd if version_string_cmd_raw: version_command_template = string.Template(version_string_cmd_raw) - version_command = version_command_template.safe_substitute({"__tool_directory__": self.compute_environment.tool_directory()}) + version_command = version_command_template.safe_substitute( + {"__tool_directory__": self.compute_environment.tool_directory()} + ) self.version_command_line = f"{version_command} > {self.compute_environment.version_path()} 2>&1;\n" def _build_config_files(self): @@ -522,6 +532,7 @@ class ToolEvaluator: if inject == "api_key": if self._user: from galaxy.managers import api_keys + environment_variable_template = api_keys.ApiKeyManager(self.app).get_or_create_api_key(self._user) else: environment_variable_template = "" @@ -530,10 +541,18 @@ class ToolEvaluator: is_template = True with tempfile.NamedTemporaryFile(dir=directory, prefix="tool_env_", delete=False) as temp: config_filename = temp.name - self.__write_workdir_file(config_filename, environment_variable_template, param_dict, is_template=is_template, strip=environment_variable_def.get("strip", False)) + self.__write_workdir_file( + config_filename, + environment_variable_template, + param_dict, + is_template=is_template, + strip=environment_variable_def.get("strip", False), + ) config_file_basename = os.path.basename(config_filename) # environment setup in job file template happens before `cd $working_directory` - environment_variable["value"] = f'`cat "{self.compute_environment.env_config_directory()}/{config_file_basename}"`' + environment_variable[ + "value" + ] = f'`cat "{self.compute_environment.env_config_directory()}/{config_file_basename}"`' environment_variable["raw"] = True environment_variable["job_directory_path"] = config_filename environment_variables.append(environment_variable) @@ -556,14 +575,14 @@ class ToolEvaluator: directory = self.local_working_directory command = self.tool.command if self.tool.profile < 16.04 and command and "$param_file" in command: - with tempfile.NamedTemporaryFile(mode='w', dir=directory, delete=False) as param: + with tempfile.NamedTemporaryFile(mode="w", dir=directory, delete=False) as param: for key, value in param_dict.items(): # parameters can be strings or lists of strings, coerce to list if not isinstance(value, list): value = [value] for elem in value: - param.write(f'{key}={elem}\n') - self.__register_extra_file('param_file', param.name) + param.write(f"{key}={elem}\n") + self.__register_extra_file("param_file", param.name) return param.name else: return None @@ -587,10 +606,12 @@ class ToolEvaluator: else: raise Exception(f"Unknown config file type {config_type}") - return json.dumps(wrapped_json.json_wrap(self.tool.inputs, - self.param_dict, - self.tool.profile, - handle_files=handle_files)), False + return ( + json.dumps( + wrapped_json.json_wrap(self.tool.inputs, self.param_dict, self.tool.profile, handle_files=handle_files) + ), + False, + ) def __write_workdir_file(self, config_filename, content, context, is_template=True, strip=False): parent_dir = os.path.dirname(config_filename) @@ -602,7 +623,7 @@ class ToolEvaluator: value = unicodify(content) if strip: value = value.strip() - with open(config_filename, "w", encoding='utf-8') as f: + with open(config_filename, "w", encoding="utf-8") as f: f.write(value) # For running jobs as the actual user, ensure the config file is globally readable os.chmod(config_filename, RW_R__R__) @@ -658,7 +679,7 @@ class RemoteToolEvaluator(ToolEvaluator): def build(self): config_file = self.tool.config_file global_tool_logs(self._build_config_files, config_file, "Building Config Files") - global_tool_logs(self._build_param_file, config_file, 'Building Param File') + global_tool_logs(self._build_param_file, config_file, "Building Param File") global_tool_logs(self._build_command_line, config_file, "Building Command Line") global_tool_logs(self._build_version_command, config_file, "Building Version Command Line") return self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 874fd2c2ef7..39feb8ba5bf 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -8,7 +8,14 @@ import logging import os import os.path import re -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import ( + Any, + Dict, + List, + Optional, + Tuple, + Union, +) from webob.compat import cgi_FieldStorage @@ -38,11 +45,9 @@ from galaxy.util.rules_dsl import RuleSet from . import ( dynamic_options, history_query, - validation -) -from .dataset_matcher import ( - get_dataset_matcher_factory, + validation, ) +from .dataset_matcher import get_dataset_matcher_factory from .sanitize import ToolParameterSanitizer log = logging.getLogger(__name__) @@ -54,7 +59,7 @@ class workflow_building_modes: USE_HISTORY = 1 -WORKFLOW_PARAMETER_REGULAR_EXPRESSION = re.compile(r'\$\{.+?\}') +WORKFLOW_PARAMETER_REGULAR_EXPRESSION = re.compile(r"\$\{.+?\}") class ImplicitConversionRequired(Exception): @@ -72,8 +77,9 @@ def contains_workflow_parameter(value, search=False): def is_runtime_value(value): - return isinstance(value, RuntimeValue) or (isinstance(value, dict) - and value.get("__class__") in ["RuntimeValue", "ConnectedValue"]) + return isinstance(value, RuntimeValue) or ( + isinstance(value, dict) and value.get("__class__") in ["RuntimeValue", "ConnectedValue"] + ) def is_runtime_context(trans, other_values): @@ -83,9 +89,9 @@ def is_runtime_context(trans, other_values): if is_runtime_value(context_value): return True for v in util.listify(context_value): - if isinstance(v, HistoryDatasetAssociation) and \ - ((hasattr(v, 'state') and v.state != Dataset.states.OK) - or hasattr(v, 'implicit_conversion')): + if isinstance(v, HistoryDatasetAssociation) and ( + (hasattr(v, "state") and v.state != Dataset.states.OK) or hasattr(v, "implicit_conversion") + ): return True return False @@ -114,7 +120,6 @@ def assert_throws_param_value_error(message): class ParameterValueError(ValueError): - def __init__(self, message_suffix, parameter_name, parameter_value=NO_PARAMETER_VALUE, is_dynamic=None): message = f"parameter '{parameter_name}': {message_suffix}" super().__init__(message) @@ -146,7 +151,8 @@ class ToolParameter(Dictifiable): >>> assert p.name == 'parameter_name' >>> assert sorted(p.to_dict(trans).items()) == [('argument', '--parameter-name'), ('help', ''), ('hidden', False), ('is_dynamic', False), ('label', ''), ('model_class', 'ToolParameter'), ('name', 'parameter_name'), ('optional', False), ('refresh_on_change', False), ('type', 'text'), ('value', None)] """ - dict_collection_visible_keys = ['name', 'argument', 'type', 'label', 'help', 'refresh_on_change'] + + dict_collection_visible_keys = ["name", "argument", "type", "label", "help", "refresh_on_change"] def __init__(self, tool, input_source, context=None): input_source = ensure_input_source(input_source) @@ -222,10 +228,10 @@ class ToolParameter(Dictifiable): # Handle Runtime and Unvalidated values if is_runtime_value(value): if isinstance(self, HiddenToolParameter): - raise ParameterValueError(message_suffix='Runtime Parameter not valid', parameter_name=self.name) + raise ParameterValueError(message_suffix="Runtime Parameter not valid", parameter_name=self.name) return runtime_to_object(value) - elif isinstance(value, dict) and value.get('__class__') == 'UnvalidatedValue': - return value['value'] + elif isinstance(value, dict) and value.get("__class__") == "UnvalidatedValue": + return value["value"] # Delegate to the 'to_python' method if ignore_errors: try: @@ -285,14 +291,16 @@ class ToolParameter(Dictifiable): validator.validate(value, trans) def to_dict(self, trans, other_values=None): - """ to_dict tool parameter. This can be overridden by subclasses. """ + """to_dict tool parameter. This can be overridden by subclasses.""" other_values = other_values or {} tool_dict = super().to_dict() - tool_dict['model_class'] = self.__class__.__name__ - tool_dict['optional'] = self.optional - tool_dict['hidden'] = self.hidden - tool_dict['is_dynamic'] = self.is_dynamic - tool_dict['value'] = self.value_to_basic(self.get_initial_value(trans, other_values), trans.app, use_security=True) + tool_dict["model_class"] = self.__class__.__name__ + tool_dict["optional"] = self.optional + tool_dict["hidden"] = self.hidden + tool_dict["is_dynamic"] = self.is_dynamic + tool_dict["value"] = self.value_to_basic( + self.get_initial_value(trans, other_values), trans.app, use_security=True + ) return tool_dict @classmethod @@ -300,7 +308,7 @@ class ToolParameter(Dictifiable): """Factory method to create parameter of correct type""" input_source = ensure_input_source(input_source) param_name = cls.parse_name(input_source) - param_type = input_source.get('type') + param_type = input_source.get("type") if not param_type: raise ValueError(f"parameter '{param_name}' requires a 'type'") elif param_type not in parameter_types: @@ -314,15 +322,14 @@ class ToolParameter(Dictifiable): class SimpleTextToolParameter(ToolParameter): - def __init__(self, tool, input_source): input_source = ensure_input_source(input_source) super().__init__(tool, input_source) - self.optional = input_source.get_bool('optional', False) + self.optional = input_source.get_bool("optional", False) if self.optional: self.value = None else: - self.value = '' + self.value = "" def to_json(self, value, app, use_security): """Convert a value to a string representation suitable for persisting""" @@ -349,21 +356,25 @@ class TextToolParameter(SimpleTextToolParameter): super().__init__(tool, input_source) self.datalist = [] for (title, value, _) in input_source.parse_static_options(): - self.datalist.append({'label': title, 'value': value}) - self.value = input_source.get('value') - self.area = input_source.get_bool('area', False) + self.datalist.append({"label": title, "value": value}) + self.value = input_source.get("value") + self.area = input_source.get_bool("area", False) def validate(self, value, trans=None): search = self.type == "text" - if not (trans and trans.workflow_building_mode is workflow_building_modes.ENABLED and contains_workflow_parameter(value, search=search)): + if not ( + trans + and trans.workflow_building_mode is workflow_building_modes.ENABLED + and contains_workflow_parameter(value, search=search) + ): return super().validate(value, trans) def to_dict(self, trans, other_values=None): d = super().to_dict(trans) other_values = other_values or {} - d['area'] = self.area - d['datalist'] = self.datalist - d['optional'] = self.optional + d["area"] = self.area + d["datalist"] = self.datalist + d["optional"] = self.optional return d @@ -382,7 +393,7 @@ class IntegerToolParameter(TextToolParameter): ... p.from_json("_string", trans) """ - dict_collection_visible_keys = ToolParameter.dict_collection_visible_keys + ['min', 'max'] + dict_collection_visible_keys = ToolParameter.dict_collection_visible_keys + ["min", "max"] def __init__(self, tool, input_source): super().__init__(tool, input_source) @@ -393,8 +404,8 @@ class IntegerToolParameter(TextToolParameter): raise ParameterValueError("the attribute 'value' must be an integer", self.name) elif self.value is None and not self.optional: raise ParameterValueError("the attribute 'value' must be set for non optional parameters", self.name, None) - self.min = input_source.get('min') - self.max = input_source.get('max') + self.min = input_source.get("min") + self.max = input_source.get("max") if self.min: try: self.min = int(self.min) @@ -420,7 +431,9 @@ class IntegerToolParameter(TextToolParameter): if trans.workflow_building_mode is workflow_building_modes.ENABLED: raise ParameterValueError("an integer or workflow parameter is required", self.name, value) else: - raise ParameterValueError("the attribute 'value' must be set for non optional parameters", self.name, value) + raise ParameterValueError( + "the attribute 'value' must be set for non optional parameters", self.name, value + ) def to_python(self, value, app): try: @@ -454,12 +467,12 @@ class FloatToolParameter(TextToolParameter): ... p.from_json("_string", trans) """ - dict_collection_visible_keys = ToolParameter.dict_collection_visible_keys + ['min', 'max'] + dict_collection_visible_keys = ToolParameter.dict_collection_visible_keys + ["min", "max"] def __init__(self, tool, input_source): super().__init__(tool, input_source) - self.min = input_source.get('min') - self.max = input_source.get('max') + self.min = input_source.get("min") + self.max = input_source.get("max") if self.value: try: float(self.value) @@ -492,7 +505,9 @@ class FloatToolParameter(TextToolParameter): if trans.workflow_building_mode is workflow_building_modes.ENABLED: raise ParameterValueError("an integer or workflow parameter is required", self.name, value) else: - raise ParameterValueError("the attribute 'value' must be set for non optional parameters", self.name, value) + raise ParameterValueError( + "the attribute 'value' must be set for non optional parameters", self.name, value + ) def to_python(self, value, app): try: @@ -536,11 +551,11 @@ class BooleanToolParameter(ToolParameter): def __init__(self, tool, input_source): input_source = ensure_input_source(input_source) super().__init__(tool, input_source) - self.truevalue = input_source.get('truevalue', 'true') - self.falsevalue = input_source.get('falsevalue', 'false') - nullable = input_source.get_bool('optional', False) + self.truevalue = input_source.get("truevalue", "true") + self.falsevalue = input_source.get("falsevalue", "false") + nullable = input_source.get_bool("optional", False) self.optional = nullable - self.checked = input_source.get_bool('checked', None if nullable else False) + self.checked = input_source.get_bool("checked", None if nullable else False) def from_json(self, value, trans=None, other_values=None): return self.to_python(value) @@ -567,9 +582,9 @@ class BooleanToolParameter(ToolParameter): def to_dict(self, trans, other_values=None): d = super().to_dict(trans) - d['truevalue'] = self.truevalue - d['falsevalue'] = self.falsevalue - d['optional'] = self.optional + d["truevalue"] = self.truevalue + d["falsevalue"] = self.falsevalue + d["optional"] = self.optional return d @property @@ -597,11 +612,11 @@ class FileToolParameter(ToolParameter): # Middleware or proxies may encode files in special ways (TODO: this # should be pluggable) if type(value) == dict: - if 'session_id' in value: + if "session_id" in value: # handle api upload session_id = value["session_id"] upload_store = trans.app.config.tus_upload_store or trans.app.config.new_file_path - if re.match(r'^[\w-]+$', session_id) is None: + if re.match(r"^[\w-]+$", session_id) is None: raise ValueError("Invalid session id format.") local_filename = os.path.abspath(os.path.join(upload_store, session_id)) if upload_store != trans.app.config.new_file_path and not os.path.exists(local_filename): @@ -610,9 +625,13 @@ class FileToolParameter(ToolParameter): else: # handle nginx upload upload_store = trans.app.config.nginx_upload_store - assert upload_store, "Request appears to have been processed by nginx_upload_module but Galaxy is not configured to recognize it." - local_filename = os.path.abspath(value['path']) - assert local_filename.startswith(upload_store), f"Filename provided by nginx ({local_filename}) is not in correct directory ({upload_store})." + assert ( + upload_store + ), "Request appears to have been processed by nginx_upload_module but Galaxy is not configured to recognize it." + local_filename = os.path.abspath(value["path"]) + assert local_filename.startswith( + upload_store + ), f"Filename provided by nginx ({local_filename}) is not in correct directory ({upload_store})." value = dict(filename=value["name"], local_filename=local_filename) return value @@ -623,14 +642,14 @@ class FileToolParameter(ToolParameter): return "multipart/form-data" def to_json(self, value, app, use_security): - if value in [None, '']: + if value in [None, ""]: return None elif isinstance(value, str): return value elif isinstance(value, dict): # or should we jsonify? try: - return value['local_filename'] + return value["local_filename"] except KeyError: return None elif isinstance(value, cgi_FieldStorage): @@ -662,9 +681,9 @@ class FTPFileToolParameter(ToolParameter): def __init__(self, tool, input_source): input_source = ensure_input_source(input_source) super().__init__(tool, input_source) - self.multiple = input_source.get_bool('multiple', True) + self.multiple = input_source.get_bool("multiple", True) self.optional = input_source.parse_optional(True) - self.user_ftp_dir = '' + self.user_ftp_dir = "" def get_initial_value(self, trans, other_values): if trans is not None: @@ -679,9 +698,9 @@ class FTPFileToolParameter(ToolParameter): return True def to_param_dict_string(self, value, other_values=None): - if value == '': - return 'None' - lst = [f'{self.user_ftp_dir}{dataset}' for dataset in value] + if value == "": + return "None" + lst = [f"{self.user_ftp_dir}{dataset}" for dataset in value] if self.multiple: return lst else: @@ -698,11 +717,11 @@ class FTPFileToolParameter(ToolParameter): value = [value] lst: List[str] = [] for val in value: - if val in [None, '']: + if val in [None, ""]: lst = [] break if isinstance(val, dict): - lst.append(val['name']) + lst.append(val["name"]) else: lst.append(val) if len(lst) == 0: @@ -715,7 +734,7 @@ class FTPFileToolParameter(ToolParameter): def to_dict(self, trans, other_values=None): d = super().to_dict(trans) - d['multiple'] = self.multiple + d["multiple"] = self.multiple return d @@ -733,7 +752,7 @@ class HiddenToolParameter(ToolParameter): def __init__(self, tool, input_source): super().__init__(tool, input_source) - self.value = input_source.get('value') + self.value = input_source.get("value") self.hidden = True def get_initial_value(self, trans, other_values): @@ -768,8 +787,8 @@ class ColorToolParameter(ToolParameter): def __init__(self, tool, input_source): input_source = ensure_input_source(input_source) super().__init__(tool, input_source) - self.value = input_source.get('value', '#000000') - self.rgb = input_source.get('rgb', False) + self.value = input_source.get("value", "#000000") + self.rgb = input_source.get("rgb", False) def get_initial_value(self, trans, other_values): if self.value is not None: @@ -778,7 +797,7 @@ class ColorToolParameter(ToolParameter): def to_param_dict_string(self, value, other_values=None): if self.rgb: try: - return str(tuple(int(value.lstrip('#')[i: i + 2], 16) for i in (0, 2, 4))) + return str(tuple(int(value.lstrip("#")[i : i + 2], 16) for i in (0, 2, 4))) except Exception: raise ParameterValueError(f"Failed to convert '{value}' to RGB.", self.name) return str(value) @@ -799,7 +818,7 @@ class BaseURLToolParameter(HiddenToolParameter): def __init__(self, tool, input_source): super().__init__(tool, input_source) - self.value = input_source.get('value', '') + self.value = input_source.get("value", "") def get_initial_value(self, trans, other_values): return self._get_value(trans) @@ -860,13 +879,13 @@ class SelectToolParameter(ToolParameter): def __init__(self, tool, input_source, context=None): input_source = ensure_input_source(input_source) super().__init__(tool, input_source) - self.multiple = input_source.get_bool('multiple', False) + self.multiple = input_source.get_bool("multiple", False) # Multiple selects are optional by default, single selection is the inverse. self.optional = input_source.parse_optional(self.multiple) - self.display = input_source.get('display', None) - self.separator = input_source.get('separator', ',') + self.display = input_source.get("display", None) + self.separator = input_source.get("separator", ",") self.legal_values = set() - self.dynamic_options = input_source.get('dynamic_options', None) + self.dynamic_options = input_source.get("dynamic_options", None) self.options = parse_dynamic_options(self, input_source) if self.options is not None: for validator in self.options.validators: @@ -875,10 +894,10 @@ class SelectToolParameter(ToolParameter): self.static_options = input_source.parse_static_options() for (_, value, _) in self.static_options: self.legal_values.add(value) - self.is_dynamic = ((self.dynamic_options is not None) or (self.options is not None)) + self.is_dynamic = (self.dynamic_options is not None) or (self.options is not None) def _get_dynamic_options_call_other_values(self, trans, other_values): - call_other_values = ExpressionContext({'__trans__': trans}) + call_other_values = ExpressionContext({"__trans__": trans}) if other_values: call_other_values.parent = other_values.parent call_other_values.update(other_values.dict) @@ -892,7 +911,12 @@ class SelectToolParameter(ToolParameter): try: return eval(self.dynamic_options, self.tool.code_namespace, call_other_values) except Exception as e: - log.debug("Error determining dynamic options for parameter '%s' in tool '%s':", self.name, self.tool.id, exc_info=e) + log.debug( + "Error determining dynamic options for parameter '%s' in tool '%s':", + self.name, + self.tool.id, + exc_info=e, + ) return [] else: return self.static_options @@ -926,7 +950,7 @@ class SelectToolParameter(ToolParameter): # we do not allow this to be the case in a dynamically # generated multiple select list being set in workflow building # mode we instead treat '' as 'No option Selected' (None) - if value == '': + if value == "": value = None else: if isinstance(value, str): @@ -938,31 +962,45 @@ class SelectToolParameter(ToolParameter): elif value is None: if self.optional: return None - raise ParameterValueError("an invalid option (None) was selected, please verify", self.name, None, is_dynamic=self.is_dynamic) + raise ParameterValueError( + "an invalid option (None) was selected, please verify", self.name, None, is_dynamic=self.is_dynamic + ) elif not legal_values: if self.optional and self.tool.profile < 18.09: # Covers optional parameters with default values that reference other optional parameters. # These will have a value but no legal_values. # See https://github.com/galaxyproject/tools-iuc/pull/1842#issuecomment-394083768 for context. return None - raise ParameterValueError("requires a value, but no legal values defined", self.name, is_dynamic=self.is_dynamic) + raise ParameterValueError( + "requires a value, but no legal values defined", self.name, is_dynamic=self.is_dynamic + ) if isinstance(value, list): if not self.multiple: - raise ParameterValueError("multiple values provided but parameter is not expecting multiple values", self.name, is_dynamic=self.is_dynamic) + raise ParameterValueError( + "multiple values provided but parameter is not expecting multiple values", + self.name, + is_dynamic=self.is_dynamic, + ) if set(value).issubset(legal_values): return value elif set(value).issubset(set(fallback_values.keys())): return [fallback_values[v] for v in value] else: - raise ParameterValueError(f"invalid options ({','.join(set(value) - set(legal_values))!r}) were selected (valid options: {','.join(legal_values)})", self.name, is_dynamic=self.is_dynamic) + raise ParameterValueError( + f"invalid options ({','.join(set(value) - set(legal_values))!r}) were selected (valid options: {','.join(legal_values)})", + self.name, + is_dynamic=self.is_dynamic, + ) else: - value_is_none = (value == "None" and "None" not in legal_values) + value_is_none = value == "None" and "None" not in legal_values if value_is_none or not value: if self.multiple: if self.optional: return [] else: - raise ParameterValueError("no option was selected for non optional parameter", self.name, is_dynamic=self.is_dynamic) + raise ParameterValueError( + "no option was selected for non optional parameter", self.name, is_dynamic=self.is_dynamic + ) if is_runtime_value(value): return None if value in legal_values: @@ -972,14 +1010,23 @@ class SelectToolParameter(ToolParameter): elif not require_legal_value: return value else: - raise ParameterValueError(f"an invalid option ({value!r}) was selected (valid options: {','.join(legal_values)})", self.name, value, is_dynamic=self.is_dynamic) + raise ParameterValueError( + f"an invalid option ({value!r}) was selected (valid options: {','.join(legal_values)})", + self.name, + value, + is_dynamic=self.is_dynamic, + ) def to_param_dict_string(self, value, other_values=None): if value in (None, []): return "None" if isinstance(value, list): if not self.multiple: - raise ParameterValueError("multiple values provided but parameter is not expecting multiple values", self.name, is_dynamic=self.is_dynamic) + raise ParameterValueError( + "multiple values provided but parameter is not expecting multiple values", + self.name, + is_dynamic=self.is_dynamic, + ) value = list(map(str, value)) else: value = str(value) @@ -1048,10 +1095,10 @@ class SelectToolParameter(ToolParameter): # Get options, value. options = self.get_options(trans, other_values) - d['options'] = options - d['display'] = self.display - d['multiple'] = self.multiple - d['textable'] = is_runtime_context(trans, other_values) + d["options"] = options + d["display"] = self.display + d["multiple"] = self.multiple + d["textable"] = is_runtime_context(trans, other_values) return d def validate(self, value, trans=None): @@ -1114,12 +1161,14 @@ class GenomeBuildParameter(SelectToolParameter): # Found selected option. value = option[1] - d.update({ - 'options': options, - 'value': value, - 'display': self.display, - 'multiple': self.multiple, - }) + d.update( + { + "options": options, + "value": value, + "display": self.display, + "multiple": self.multiple, + } + ) return d @@ -1161,9 +1210,9 @@ class SelectTagParameter(SelectToolParameter): # split on newline and , if isinstance(value, list) or isinstance(value, str): if not isinstance(value, list): - value = value.split('\n') + value = value.split("\n") for tag_str in value: - for tag in str(tag_str).split(','): + for tag in str(tag_str).split(","): tag = tag.strip() if tag: tag_list.append(tag) @@ -1191,14 +1240,14 @@ class SelectTagParameter(SelectToolParameter): return [] tags = set() for history_item in util.listify(history_items): - if hasattr(history_item, 'dataset_instances'): + if hasattr(history_item, "dataset_instances"): for dataset in history_item.dataset_instances: for tag in dataset.tags: - if tag.user_tname == 'group': + if tag.user_tname == "group": tags.add(tag.user_value) else: for tag in history_item.tags: - if tag.user_tname == 'group': + if tag.user_tname == "group": tags.add(tag.user_value) return list(tags) @@ -1227,7 +1276,7 @@ class SelectTagParameter(SelectToolParameter): def to_dict(self, trans, other_values=None): other_values = other_values or {} d = super().to_dict(trans, other_values=other_values) - d['data_ref'] = self.data_ref + d["data_ref"] = self.data_ref return d @@ -1287,9 +1336,9 @@ class ColumnListParameter(SelectToolParameter): if isinstance(value, list) or isinstance(value, str): column_list = [] if not isinstance(value, list): - value = value.split('\n') + value = value.split("\n") for column in value: - for column2 in str(column).split(','): + for column2 in str(column).split(","): column2 = column2.strip() if column2: column_list.append(column2) @@ -1305,14 +1354,14 @@ class ColumnListParameter(SelectToolParameter): else: value = None if not value and self.accept_default: - value = self.default_value or '1' + value = self.default_value or "1" return [value] if self.multiple else value return super().from_json(value, trans, other_values) @staticmethod def _strip_c(column): if isinstance(column, str): - if column.startswith('c') and len(column) > 1 and all(c.isdigit() for c in column[1:]): + if column.startswith("c") and len(column) > 1 and all(c.isdigit() for c in column[1:]): column = column.strip().lower()[1:] return column @@ -1332,23 +1381,27 @@ class ColumnListParameter(SelectToolParameter): if isinstance(dataset, HistoryDatasetCollectionAssociation): dataset = dataset.to_hda_representative() if isinstance(dataset, HistoryDatasetAssociation) and self.ref_input and self.ref_input.formats: - direct_match, target_ext, converted_dataset = dataset.find_conversion_destination(self.ref_input.formats) + direct_match, target_ext, converted_dataset = dataset.find_conversion_destination( + self.ref_input.formats + ) if not direct_match and target_ext: if not converted_dataset: raise ImplicitConversionRequired else: dataset = converted_dataset # Columns can only be identified if the dataset is ready and metadata is available - if not hasattr(dataset, 'metadata') or \ - not hasattr(dataset.metadata, 'columns') or \ - not dataset.metadata.columns: + if ( + not hasattr(dataset, "metadata") + or not hasattr(dataset.metadata, "columns") + or not dataset.metadata.columns + ): return [] # Build up possible columns for this dataset this_column_list = [] if self.numerical: # If numerical was requested, filter columns based on metadata for i, col in enumerate(dataset.metadata.column_types): - if col == 'int' or col == 'float': + if col == "int" or col == "float": this_column_list.append(str(i + 1)) else: this_column_list = [str(i) for i in range(1, dataset.metadata.columns + 1)] @@ -1369,12 +1422,12 @@ class ColumnListParameter(SelectToolParameter): try: with open(dataset.get_file_name()) as f: head = f.readline() - cnames = head.rstrip("\n\r ").split('\t') - column_list = [('%d' % (i + 1), 'c%d: %s' % (i + 1, x)) for i, x in enumerate(cnames)] + cnames = head.rstrip("\n\r ").split("\t") + column_list = [("%d" % (i + 1), "c%d: %s" % (i + 1, x)) for i, x in enumerate(cnames)] if self.numerical: # If numerical was requested, filter columns based on metadata - if hasattr(dataset, 'metadata') and hasattr(dataset.metadata, 'column_types'): + if hasattr(dataset, "metadata") and hasattr(dataset.metadata, "column_types"): if len(dataset.metadata.column_types) >= len(cnames): - numerics = [i for i, x in enumerate(dataset.metadata.column_types) if x in ['int', 'float']] + numerics = [i for i, x in enumerate(dataset.metadata.column_types) if x in ["int", "float"]] column_list = [column_list[i] for i in numerics] except Exception: column_list = self.get_column_list(trans, other_values) @@ -1427,8 +1480,8 @@ class ColumnListParameter(SelectToolParameter): def to_dict(self, trans, other_values=None): other_values = other_values or {} d = super().to_dict(trans, other_values=other_values) - d['data_ref'] = self.data_ref - d['numerical'] = self.numerical + d["data_ref"] = self.data_ref + d["numerical"] = self.numerical return d @@ -1479,47 +1532,57 @@ class DrillDownSelectToolParameter(SelectToolParameter): def __init__(self, tool, input_source, context=None): def recurse_option_elems(cur_options, option_elems): for option_elem in option_elems: - selected = string_as_bool(option_elem.get('selected', False)) - cur_options.append({'name': option_elem.get('name'), 'value': option_elem.get('value'), 'options': [], 'selected': selected}) - recurse_option_elems(cur_options[-1]['options'], option_elem.findall('option')) + selected = string_as_bool(option_elem.get("selected", False)) + cur_options.append( + { + "name": option_elem.get("name"), + "value": option_elem.get("value"), + "options": [], + "selected": selected, + } + ) + recurse_option_elems(cur_options[-1]["options"], option_elem.findall("option")) input_source = ensure_input_source(input_source) ToolParameter.__init__(self, tool, input_source) # TODO: abstract XML out of here - so non-XML InputSources can # specify DrillDown parameters. elem = input_source.elem() - self.multiple = string_as_bool(elem.get('multiple', False)) - self.display = elem.get('display', None) - self.hierarchy = elem.get('hierarchy', 'exact') # exact or recurse - self.separator = elem.get('separator', ',') - from_file = elem.get('from_file', None) + self.multiple = string_as_bool(elem.get("multiple", False)) + self.display = elem.get("display", None) + self.hierarchy = elem.get("hierarchy", "exact") # exact or recurse + self.separator = elem.get("separator", ",") + from_file = elem.get("from_file", None) if from_file: if not os.path.isabs(from_file): from_file = os.path.join(tool.app.config.tool_data_path, from_file) elem = XML(f"{open(from_file).read()}") - self.dynamic_options = elem.get('dynamic_options', None) + self.dynamic_options = elem.get("dynamic_options", None) if self.dynamic_options: self.is_dynamic = True self.options = [] self.filtered: Dict[str, Any] = {} - if elem.find('filter'): + if elem.find("filter"): self.is_dynamic = True - for filter in elem.findall('filter'): + for filter in elem.findall("filter"): # currently only filtering by metadata key matching input file is allowed - if filter.get('type') == 'data_meta': - if filter.get('data_ref') not in self.filtered: - self.filtered[filter.get('data_ref')] = {} - if filter.get('meta_key') not in self.filtered[filter.get('data_ref')]: - self.filtered[filter.get('data_ref')][filter.get('meta_key')] = {} - if filter.get('value') not in self.filtered[filter.get('data_ref')][filter.get('meta_key')]: - self.filtered[filter.get('data_ref')][filter.get('meta_key')][filter.get('value')] = [] - recurse_option_elems(self.filtered[filter.get('data_ref')][filter.get('meta_key')][filter.get('value')], filter.find('options').findall('option')) + if filter.get("type") == "data_meta": + if filter.get("data_ref") not in self.filtered: + self.filtered[filter.get("data_ref")] = {} + if filter.get("meta_key") not in self.filtered[filter.get("data_ref")]: + self.filtered[filter.get("data_ref")][filter.get("meta_key")] = {} + if filter.get("value") not in self.filtered[filter.get("data_ref")][filter.get("meta_key")]: + self.filtered[filter.get("data_ref")][filter.get("meta_key")][filter.get("value")] = [] + recurse_option_elems( + self.filtered[filter.get("data_ref")][filter.get("meta_key")][filter.get("value")], + filter.find("options").findall("option"), + ) elif not self.dynamic_options: - recurse_option_elems(self.options, elem.find('options').findall('option')) + recurse_option_elems(self.options, elem.find("options").findall("option")) def _get_options_from_code(self, trans=None, value=None, other_values=None): assert self.dynamic_options, Exception("dynamic_options was not specifed") - call_other_values = ExpressionContext({'__trans__': trans, '__value__': value}) + call_other_values = ExpressionContext({"__trans__": trans, "__value__": value}) if other_values: call_other_values.parent = other_values.parent call_other_values.update(other_values.dict) @@ -1537,12 +1600,16 @@ class DrillDownSelectToolParameter(SelectToolParameter): options = [] for filter_key, filter_value in self.filtered.items(): dataset = other_values.get(filter_key) - if dataset.__class__.__name__.endswith("DatasetFilenameWrapper"): # this is a bad way to check for this, but problems importing class (due to circular imports?) + if dataset.__class__.__name__.endswith( + "DatasetFilenameWrapper" + ): # this is a bad way to check for this, but problems importing class (due to circular imports?) dataset = dataset.dataset if dataset: for meta_key, meta_dict in filter_value.items(): - if hasattr(dataset, 'metadata') and hasattr(dataset.metadata, 'spec'): - check_meta_val = dataset.metadata.spec[meta_key].param.to_string(dataset.metadata.get(meta_key)) + if hasattr(dataset, "metadata") and hasattr(dataset.metadata, "spec"): + check_meta_val = dataset.metadata.spec[meta_key].param.to_string( + dataset.metadata.get(meta_key) + ) if check_meta_val in meta_dict: options.extend(meta_dict[check_meta_val]) return options @@ -1551,8 +1618,9 @@ class DrillDownSelectToolParameter(SelectToolParameter): def get_legal_values(self, trans, other_values, value): def recurse_options(legal_values, options): for option in options: - legal_values.append(option['value']) - recurse_options(legal_values, option['options']) + legal_values.append(option["value"]) + recurse_options(legal_values, option["options"]) + legal_values: List[str] = [] recurse_options(legal_values, self.get_options(trans=trans, other_values=other_values)) return legal_values @@ -1562,7 +1630,7 @@ class DrillDownSelectToolParameter(SelectToolParameter): legal_values = self.get_legal_values(trans, other_values, value) if not legal_values and trans.workflow_building_mode: if self.multiple: - if value == '': # No option selected + if value == "": # No option selected value = None else: value = value.split("\n") @@ -1576,11 +1644,17 @@ class DrillDownSelectToolParameter(SelectToolParameter): if not isinstance(value, list): value = [value] if len(value) > 1 and not self.multiple: - raise ParameterValueError("multiple values provided but parameter is not expecting multiple values", self.name) + raise ParameterValueError( + "multiple values provided but parameter is not expecting multiple values", self.name + ) rval = [] for val in value: if val not in legal_values: - raise ParameterValueError(f"an invalid option ({val!r}) was selected (valid options: {','.join(legal_values)})", self.name, val) + raise ParameterValueError( + f"an invalid option ({val!r}) was selected (valid options: {','.join(legal_values)})", + self.name, + val, + ) rval.append(val) return rval @@ -1590,18 +1664,18 @@ class DrillDownSelectToolParameter(SelectToolParameter): def get_options_list(value): def get_base_option(value, options): for option in options: - if value == option['value']: + if value == option["value"]: return option - rval = get_base_option(value, option['options']) + rval = get_base_option(value, option["options"]) if rval: return rval return None # not found def recurse_option(option_list, option): - if not option['options']: - option_list.append(option['value']) + if not option["options"]: + option_list.append(option["value"]) else: - for opt in option['options']: + for opt in option["options"]: recurse_option(option_list, opt) rval: List[str] = [] @@ -1618,7 +1692,9 @@ class DrillDownSelectToolParameter(SelectToolParameter): options = get_options_list(val) rval.extend(options) if len(rval) > 1 and not self.multiple: - raise ParameterValueError("multiple values provided but parameter is not expecting multiple values", self.name) + raise ParameterValueError( + "multiple values provided but parameter is not expecting multiple values", self.name + ) rval = self.separator.join(rval) if self.tool is None or self.tool.options.sanitize: if self.sanitizer: @@ -1630,9 +1706,10 @@ class DrillDownSelectToolParameter(SelectToolParameter): def get_initial_value(self, trans, other_values): def recurse_options(initial_values, options): for option in options: - if option['selected']: - initial_values.append(option['value']) - recurse_options(initial_values, option['options']) + if option["selected"]: + initial_values.append(option["value"]) + recurse_options(initial_values, option["options"]) + # More working around dynamic options for workflow options = self.get_options(trans=trans, other_values=other_values) if not options: @@ -1646,12 +1723,13 @@ class DrillDownSelectToolParameter(SelectToolParameter): def to_text(self, value): def get_option_display(value, options): for option in options: - if value == option['value']: - return option['name'] - rval = get_option_display(value, option['options']) + if value == option["value"]: + return option["name"] + rval = get_option_display(value, option["options"]) if rval: return rval return None # not found + if not value: value = [] elif not isinstance(value, list): @@ -1684,9 +1762,9 @@ class DrillDownSelectToolParameter(SelectToolParameter): other_values = other_values or {} # skip SelectToolParameter (the immediate parent) bc we need to get options in a different way here d = ToolParameter.to_dict(self, trans) - d['options'] = self.get_options(trans=trans, other_values=other_values) - d['display'] = self.display - d['multiple'] = self.multiple + d["options"] = self.get_options(trans=trans, other_values=other_values) + d["display"] = self.display + d["multiple"] = self.multiple return d @@ -1696,8 +1774,8 @@ class BaseDataToolParameter(ToolParameter): def __init__(self, tool, input_source, trans): super().__init__(tool, input_source) - self.min = input_source.get('min') - self.max = input_source.get('max') + self.min = input_source.get("min") + self.max = input_source.get("max") if self.min: try: self.min = int(self.min) @@ -1718,16 +1796,19 @@ class BaseDataToolParameter(ToolParameter): else: # This occurs for things such as unit tests import galaxy.datatypes.registry + self.datatypes_registry = galaxy.datatypes.registry.Registry() self.datatypes_registry.load_datatypes() else: - self.datatypes_registry = self.tool.app.datatypes_registry # can be None if self.tool.app is a ValidationContext + self.datatypes_registry = ( + self.tool.app.datatypes_registry + ) # can be None if self.tool.app is a ValidationContext def _parse_formats(self, trans, input_source): """ Build list of classes for supported data formats """ - self.extensions = input_source.get('format', 'data').split(",") + self.extensions = input_source.get("format", "data").split(",") formats = [] if self.datatypes_registry: # This may be None when self.tool.app is a ValidationContext normalized_extensions = [extension.strip().lower() for extension in self.extensions] @@ -1736,7 +1817,9 @@ class BaseDataToolParameter(ToolParameter): if datatype is not None: formats.append(datatype) else: - log.warning(f"Datatype class not found for extension '{extension}', which is used in the 'format' attribute of parameter '{self.name}'") + log.warning( + f"Datatype class not found for extension '{extension}', which is used in the 'format' attribute of parameter '{self.name}'" + ) self.formats = formats def _parse_options(self, input_source): @@ -1747,12 +1830,12 @@ class BaseDataToolParameter(ToolParameter): self.options = parse_dynamic_options(self, input_source) if self.options: # TODO: Abstract away XML handling here. - options_elem = input_source.elem().find('options') - self.options_filter_attribute = options_elem.get('options_filter_attribute', None) + options_elem = input_source.elem().find("options") + self.options_filter_attribute = options_elem.get("options_filter_attribute", None) self.is_dynamic = self.options is not None def get_initial_value(self, trans, other_values): - if trans.workflow_building_mode is workflow_building_modes.ENABLED or trans.app.name == 'tool_shed': + if trans.workflow_building_mode is workflow_building_modes.ENABLED or trans.app.name == "tool_shed": return RuntimeValue() if self.optional: return None @@ -1774,73 +1857,81 @@ class BaseDataToolParameter(ToolParameter): def to_json(self, value, app, use_security): def single_to_json(value): src = None - if isinstance(value, dict) and 'src' in value and 'id' in value: + if isinstance(value, dict) and "src" in value and "id" in value: return value elif isinstance(value, DatasetCollectionElement): - src = 'dce' + src = "dce" elif isinstance(value, HistoryDatasetCollectionAssociation): - src = 'hdca' + src = "hdca" elif isinstance(value, LibraryDatasetDatasetAssociation): - src = 'ldda' - elif isinstance(value, HistoryDatasetAssociation) or hasattr(value, 'id'): + src = "ldda" + elif isinstance(value, HistoryDatasetAssociation) or hasattr(value, "id"): # hasattr 'id' fires a query on persistent objects after a flush so better # to do the isinstance check. Not sure we need the hasattr check anymore - it'd be # nice to drop it. - src = 'hda' + src = "hda" if src is not None: object_id = cached_id(value) - return {'id': app.security.encode_id(object_id) if use_security else object_id, 'src': src} + return {"id": app.security.encode_id(object_id) if use_security else object_id, "src": src} - if value not in [None, '', 'None']: + if value not in [None, "", "None"]: if isinstance(value, list) and len(value) > 0: values = [single_to_json(v) for v in value] else: values = [single_to_json(value)] - return {'values': values} + return {"values": values} return None def to_python(self, value, app): def single_to_python(value): - if isinstance(value, dict) and 'src' in value: - id = value['id'] if isinstance(value['id'], int) else app.security.decode_id(value['id']) - if value['src'] == 'dce': + if isinstance(value, dict) and "src" in value: + id = value["id"] if isinstance(value["id"], int) else app.security.decode_id(value["id"]) + if value["src"] == "dce": return app.model.context.query(DatasetCollectionElement).get(id) - elif value['src'] == 'hdca': + elif value["src"] == "hdca": return app.model.context.query(HistoryDatasetCollectionAssociation).get(id) - elif value['src'] == 'ldda': + elif value["src"] == "ldda": return app.model.context.query(LibraryDatasetDatasetAssociation).get(id) else: return app.model.context.query(HistoryDatasetAssociation).get(id) - if isinstance(value, dict) and 'values' in value: - if hasattr(self, 'multiple') and self.multiple is True: - return [single_to_python(v) for v in value['values']] - elif len(value['values']) > 0: - return single_to_python(value['values'][0]) + if isinstance(value, dict) and "values" in value: + if hasattr(self, "multiple") and self.multiple is True: + return [single_to_python(v) for v in value["values"]] + elif len(value["values"]) > 0: + return single_to_python(value["values"][0]) # Handle legacy string values potentially stored in databases - none_values = [None, '', 'None'] + none_values = [None, "", "None"] if value in none_values: return None - if isinstance(value, str) and value.find(',') > -1: - return [app.model.context.query(HistoryDatasetAssociation).get(int(v)) for v in value.split(',') if v not in none_values] + if isinstance(value, str) and value.find(",") > -1: + return [ + app.model.context.query(HistoryDatasetAssociation).get(int(v)) + for v in value.split(",") + if v not in none_values + ] elif str(value).startswith("__collection_reduce__|"): - decoded_id = str(value)[len("__collection_reduce__|"):] + decoded_id = str(value)[len("__collection_reduce__|") :] if not decoded_id.isdigit(): decoded_id = app.security.decode_id(decoded_id) return app.model.context.query(HistoryDatasetCollectionAssociation).get(int(decoded_id)) elif str(value).startswith("dce:"): - return app.model.context.query(DatasetCollectionElement).get(int(value[len("dce:"):])) + return app.model.context.query(DatasetCollectionElement).get(int(value[len("dce:") :])) elif str(value).startswith("hdca:"): - return app.model.context.query(HistoryDatasetCollectionAssociation).get(int(value[len("hdca:"):])) + return app.model.context.query(HistoryDatasetCollectionAssociation).get(int(value[len("hdca:") :])) else: return app.model.context.query(HistoryDatasetAssociation).get(int(value)) def validate(self, value, trans=None): - def do_validate(v): for validator in self.validators: - if validator.requires_dataset_metadata and v and hasattr(v, 'dataset') and v.dataset.state != Dataset.states.OK: + if ( + validator.requires_dataset_metadata + and v + and hasattr(v, "dataset") + and v.dataset.state != Dataset.states.OK + ): return else: validator.validate(v, trans) @@ -1893,26 +1984,38 @@ class DataToolParameter(BaseDataToolParameter): super().__init__(tool, input_source, trans) self.load_contents = int(input_source.get("load_contents", 0)) # Add metadata validator - if not input_source.get_bool('no_validation', False): + if not input_source.get_bool("no_validation", False): self.validators.append(validation.MetadataValidator()) self._parse_formats(trans, input_source) tag = input_source.get("tag") - self.multiple = input_source.get_bool('multiple', False) + self.multiple = input_source.get_bool("multiple", False) if not self.multiple and (self.min is not None): - raise ParameterValueError("cannot specify 'min' property on single data parameter. Set multiple=\"true\" to enable this option", self.name) + raise ParameterValueError( + "cannot specify 'min' property on single data parameter. Set multiple=\"true\" to enable this option", + self.name, + ) if not self.multiple and (self.max is not None): - raise ParameterValueError("cannot specify 'max' property on single data parameter. Set multiple=\"true\" to enable this option", self.name) + raise ParameterValueError( + "cannot specify 'max' property on single data parameter. Set multiple=\"true\" to enable this option", + self.name, + ) self.tag = tag self.is_dynamic = True self._parse_options(input_source) # Load conversions required for the dataset input self.conversions = [] for name, conv_extension in input_source.parse_conversion_tuples(): - assert None not in [name, conv_extension], f'A name ({name}) and type ({conv_extension}) are required for explicit conversion' + assert None not in [ + name, + conv_extension, + ], f"A name ({name}) and type ({conv_extension}) are required for explicit conversion" if self.datatypes_registry: conv_type = self.datatypes_registry.get_datatype_by_extension(conv_extension.lower()) if conv_type is None: - raise ParameterValueError(f"datatype class not found for extension '{conv_type}', which is used as 'type' attribute in conversion of data parameter", self.name) + raise ParameterValueError( + f"datatype class not found for extension '{conv_type}', which is used as 'type' attribute in conversion of data parameter", + self.name, + ) self.conversions.append((name, conv_extension, [conv_type])) def from_json(self, value, trans, other_values=None): @@ -1921,9 +2024,9 @@ class DataToolParameter(BaseDataToolParameter): return None if not value and not self.optional: raise ParameterValueError("specify a dataset of the required format / build for parameter", self.name) - if value in [None, "None", '']: + if value in [None, "None", ""]: return None - if isinstance(value, dict) and 'values' in value: + if isinstance(value, dict) and "values" in value: value = self.to_python(value, trans.app) if isinstance(value, str) and value.find(",") > 0: value = [int(value_part) for value_part in value.split(",")] @@ -1931,25 +2034,28 @@ class DataToolParameter(BaseDataToolParameter): if isinstance(value, list): found_hdca = False for single_value in value: - if isinstance(single_value, dict) and 'src' in single_value and 'id' in single_value: - if single_value['src'] == 'hda': - decoded_id = trans.security.decode_id(single_value['id']) + if isinstance(single_value, dict) and "src" in single_value and "id" in single_value: + if single_value["src"] == "hda": + decoded_id = trans.security.decode_id(single_value["id"]) rval.append(trans.sa_session.query(HistoryDatasetAssociation).get(decoded_id)) - elif single_value['src'] == 'hdca': + elif single_value["src"] == "hdca": found_hdca = True - decoded_id = trans.security.decode_id(single_value['id']) + decoded_id = trans.security.decode_id(single_value["id"]) rval.append(trans.sa_session.query(HistoryDatasetCollectionAssociation).get(decoded_id)) - elif single_value['src'] == 'ldda': - decoded_id = trans.security.decode_id(single_value['id']) + elif single_value["src"] == "ldda": + decoded_id = trans.security.decode_id(single_value["id"]) rval.append(trans.sa_session.query(LibraryDatasetDatasetAssociation).get(decoded_id)) else: raise ValueError(f"Unknown input source {single_value['src']} passed to job submission API.") - elif isinstance(single_value, ( + elif isinstance( + single_value, + ( HistoryDatasetCollectionAssociation, DatasetCollectionElement, HistoryDatasetAssociation, - LibraryDatasetDatasetAssociation - )): + LibraryDatasetDatasetAssociation, + ), + ): rval.append(single_value) else: if len(str(single_value)) == 16: @@ -1961,20 +2067,23 @@ class DataToolParameter(BaseDataToolParameter): if found_hdca: for val in rval: if not isinstance(val, HistoryDatasetCollectionAssociation): - raise ParameterValueError("if collections are supplied to multiple data input parameter, only collections may be used", self.name) + raise ParameterValueError( + "if collections are supplied to multiple data input parameter, only collections may be used", + self.name, + ) elif isinstance(value, (HistoryDatasetAssociation, LibraryDatasetDatasetAssociation)): rval.append(value) - elif isinstance(value, dict) and 'src' in value and 'id' in value: - if value['src'] == 'hda': - decoded_id = trans.security.decode_id(value['id']) + elif isinstance(value, dict) and "src" in value and "id" in value: + if value["src"] == "hda": + decoded_id = trans.security.decode_id(value["id"]) rval.append(trans.sa_session.query(HistoryDatasetAssociation).get(decoded_id)) - elif value['src'] == 'hdca': - decoded_id = trans.security.decode_id(value['id']) + elif value["src"] == "hdca": + decoded_id = trans.security.decode_id(value["id"]) rval.append(trans.sa_session.query(HistoryDatasetCollectionAssociation).get(decoded_id)) else: raise ValueError(f"Unknown input source {value['src']} passed to job submission API.") elif str(value).startswith("__collection_reduce__|"): - encoded_ids = [v[len("__collection_reduce__|"):] for v in str(value).split(",")] + encoded_ids = [v[len("__collection_reduce__|") :] for v in str(value).split(",")] decoded_ids = map(trans.security.decode_id, encoded_ids) rval = [] for decoded_id in decoded_ids: @@ -1991,7 +2100,9 @@ class DataToolParameter(BaseDataToolParameter): if hasattr(v, "deleted") and v.deleted: raise ParameterValueError("the previously selected dataset has been deleted.", self.name) elif hasattr(v, "dataset") and v.dataset.state in [Dataset.states.ERROR, Dataset.states.DISCARDED]: - raise ParameterValueError("the previously selected dataset has entered an unusable state", self.name) + raise ParameterValueError( + "the previously selected dataset has entered an unusable state", self.name + ) elif hasattr(v, "dataset"): match = dataset_matcher.hda_match(v) if match and match.implicit_conversion: @@ -2030,7 +2141,12 @@ class DataToolParameter(BaseDataToolParameter): return [] def converter_safe(self, other_values, trans): - if self.tool is None or self.tool.has_multiple_pages or not hasattr(trans, 'workflow_building_mode') or trans.workflow_building_mode: + if ( + self.tool is None + or self.tool.has_multiple_pages + or not hasattr(trans, "workflow_building_mode") + or trans.workflow_building_mode + ): return False if other_values is None: return True # we don't know other values, so we can't check, assume ok @@ -2038,8 +2154,15 @@ class DataToolParameter(BaseDataToolParameter): def visitor(prefix, input, value, parent=None): if isinstance(input, SelectToolParameter) and self.name in input.get_dependencies(): - if input.is_dynamic and (input.dynamic_options or (not input.dynamic_options and not input.options) or not input.options.converter_safe): - converter_safe[0] = False # This option does not allow for conversion, i.e. uses contents of dataset file to generate options + if input.is_dynamic and ( + input.dynamic_options + or (not input.dynamic_options and not input.options) + or not input.options.converter_safe + ): + converter_safe[ + 0 + ] = False # This option does not allow for conversion, i.e. uses contents of dataset file to generate options + self.tool.visit_inputs(other_values, visitor) return False not in converter_safe @@ -2056,7 +2179,7 @@ class DataToolParameter(BaseDataToolParameter): else: call_attribute = False ref = value - for attribute in options_filter_attribute.split('.'): + for attribute in options_filter_attribute.split("."): ref = getattr(ref, attribute) if call_attribute: ref = ref() @@ -2067,20 +2190,22 @@ class DataToolParameter(BaseDataToolParameter): # create dictionary and fill default parameters d = super().to_dict(trans) extensions = self.extensions - all_edam_formats = self.datatypes_registry.edam_formats if hasattr(self.datatypes_registry, 'edam_formats') else {} - all_edam_data = self.datatypes_registry.edam_data if hasattr(self.datatypes_registry, 'edam_formats') else {} + all_edam_formats = ( + self.datatypes_registry.edam_formats if hasattr(self.datatypes_registry, "edam_formats") else {} + ) + all_edam_data = self.datatypes_registry.edam_data if hasattr(self.datatypes_registry, "edam_formats") else {} edam_formats = [all_edam_formats.get(ext, None) for ext in extensions] edam_data = [all_edam_data.get(ext, None) for ext in extensions] - d['extensions'] = extensions - d['edam'] = {'edam_formats': edam_formats, 'edam_data': edam_data} - d['multiple'] = self.multiple + d["extensions"] = extensions + d["edam"] = {"edam_formats": edam_formats, "edam_data": edam_data} + d["multiple"] = self.multiple if self.multiple: # For consistency, should these just always be in the dict? - d['min'] = self.min - d['max'] = self.max - d['options'] = {'hda': [], 'hdca': []} - d['tag'] = self.tag + d["min"] = self.min + d["max"] = self.max + d["options"] = {"hda": [], "hdca": []} + d["tag"] = self.tag # return dictionary without options if context is unavailable history = trans.history @@ -2095,12 +2220,12 @@ class DataToolParameter(BaseDataToolParameter): # build and append a new select option def append(list, hda, name, src, keep=False, subcollection_type=None): value = { - 'id': trans.security.encode_id(hda.id), - 'hid': hda.hid if hda.hid is not None else -1, - 'name': name, - 'tags': [t.user_tname if not t.value else f"{t.user_tname}:{t.value}" for t in hda.tags], - 'src': src, - 'keep': keep + "id": trans.security.encode_id(hda.id), + "hid": hda.hid if hda.hid is not None else -1, + "name": name, + "tags": [t.user_tname if not t.value else f"{t.user_tname}:{t.value}" for t in hda.tags], + "src": src, + "keep": keep, } if subcollection_type: value["map_over_type"] = subcollection_type @@ -2114,17 +2239,17 @@ class DataToolParameter(BaseDataToolParameter): if match: m = match.hda hda_list = [h for h in hda_list if h != m and h != hda] - m_name = f'{match.original_hda.name} (as {match.target_ext})' if match.implicit_conversion else m.name - append(d['options']['hda'], m, m_name, 'hda') + m_name = f"{match.original_hda.name} (as {match.target_ext})" if match.implicit_conversion else m.name + append(d["options"]["hda"], m, m_name, "hda") for hda in hda_list: - if hasattr(hda, 'hid'): + if hasattr(hda, "hid"): if hda.deleted: - hda_state = 'deleted' + hda_state = "deleted" elif not hda.visible: - hda_state = 'hidden' + hda_state = "hidden" else: - hda_state = 'unavailable' - append(d['options']['hda'], hda, f'({hda_state}) {hda.name}', 'hda', True) + hda_state = "unavailable" + append(d["options"]["hda"], hda, f"({hda_state}) {hda.name}", "hda", True) # add dataset collections dataset_collection_matcher = dataset_matcher_factory.dataset_collection_matcher(dataset_matcher) @@ -2132,7 +2257,7 @@ class DataToolParameter(BaseDataToolParameter): match = dataset_collection_matcher.hdca_match(hdca) if match: subcollection_type = None - if multiple and hdca.collection.collection_type != 'list': + if multiple and hdca.collection.collection_type != "list": collection_type_description = self._history_query(trans).can_map_over(hdca) if collection_type_description: subcollection_type = collection_type_description.collection_type @@ -2142,12 +2267,12 @@ class DataToolParameter(BaseDataToolParameter): name = hdca.name if match.implicit_conversion: name = f"{name} (with implicit datatype conversion)" - append(d['options']['hdca'], hdca, name, 'hdca', subcollection_type=subcollection_type) + append(d["options"]["hdca"], hdca, name, "hdca", subcollection_type=subcollection_type) continue # sort both lists - d['options']['hda'] = sorted(d['options']['hda'], key=lambda k: k.get('hid', -1), reverse=True) - d['options']['hdca'] = sorted(d['options']['hdca'], key=lambda k: k.get('hid', -1), reverse=True) + d["options"]["hda"] = sorted(d["options"]["hda"], key=lambda k: k.get("hid", -1), reverse=True) + d["options"]["hdca"] = sorted(d["options"]["hdca"], key=lambda k: k.get("hid", -1), reverse=True) # return final dictionary return d @@ -2160,8 +2285,7 @@ class DataToolParameter(BaseDataToolParameter): class DataCollectionToolParameter(BaseDataToolParameter): - """ - """ + """ """ def __init__(self, tool, input_source, trans=None): input_source = ensure_input_source(input_source) @@ -2186,7 +2310,9 @@ class DataCollectionToolParameter(BaseDataToolParameter): return history_query.HistoryQuery.from_parameter(self, dataset_collection_type_descriptions) def match_collections(self, trans, history, dataset_collection_matcher): - dataset_collections = trans.app.dataset_collection_manager.history_dataset_collections(history, self._history_query(trans)) + dataset_collections = trans.app.dataset_collection_manager.history_dataset_collections( + history, self._history_query(trans) + ) for dataset_collection_instance in dataset_collections: match = dataset_collection_matcher.hdca_match(dataset_collection_instance) @@ -2212,7 +2338,7 @@ class DataCollectionToolParameter(BaseDataToolParameter): raise ParameterValueError("specify a dataset collection of the correct type", self.name) if value in [None, "None"]: return None - if isinstance(value, dict) and 'values' in value: + if isinstance(value, dict) and "values" in value: value = self.to_python(value, trans.app) if isinstance(value, str) and value.find(",") > 0: value = [int(value_part) for value_part in value.split(",")] @@ -2223,22 +2349,28 @@ class DataCollectionToolParameter(BaseDataToolParameter): # a DatasetCollectionElement instead of a # HistoryDatasetCollectionAssociation. rval = value - elif isinstance(value, dict) and 'src' in value and 'id' in value: - if value['src'] == 'hdca': - rval = trans.sa_session.query(HistoryDatasetCollectionAssociation).get(trans.security.decode_id(value['id'])) + elif isinstance(value, dict) and "src" in value and "id" in value: + if value["src"] == "hdca": + rval = trans.sa_session.query(HistoryDatasetCollectionAssociation).get( + trans.security.decode_id(value["id"]) + ) elif isinstance(value, list): if len(value) > 0: value = value[0] - if isinstance(value, dict) and 'src' in value and 'id' in value: - if value['src'] == 'hdca': - rval = trans.sa_session.query(HistoryDatasetCollectionAssociation).get(trans.security.decode_id(value['id'])) - elif value['src'] == 'dce': - rval = trans.sa_session.query(DatasetCollectionElement).get(trans.security.decode_id(value['id'])) + if isinstance(value, dict) and "src" in value and "id" in value: + if value["src"] == "hdca": + rval = trans.sa_session.query(HistoryDatasetCollectionAssociation).get( + trans.security.decode_id(value["id"]) + ) + elif value["src"] == "dce": + rval = trans.sa_session.query(DatasetCollectionElement).get( + trans.security.decode_id(value["id"]) + ) elif isinstance(value, str): if value.startswith("dce:"): - rval = trans.sa_session.query(DatasetCollectionElement).get(value[len("dce:"):]) + rval = trans.sa_session.query(DatasetCollectionElement).get(value[len("dce:") :]) elif value.startswith("hdca:"): - rval = trans.sa_session.query(HistoryDatasetCollectionAssociation).get(value[len("hdca:"):]) + rval = trans.sa_session.query(HistoryDatasetCollectionAssociation).get(value[len("hdca:") :]) else: rval = trans.sa_session.query(HistoryDatasetCollectionAssociation).get(value) if rval and isinstance(rval, HistoryDatasetCollectionAssociation): @@ -2261,10 +2393,10 @@ class DataCollectionToolParameter(BaseDataToolParameter): # create dictionary and fill default parameters other_values = other_values or {} d = super().to_dict(trans) - d['extensions'] = self.extensions - d['multiple'] = self.multiple - d['options'] = {'hda': [], 'hdca': [], 'dce': []} - d['tag'] = self.tag + d["extensions"] = self.extensions + d["multiple"] = self.multiple + d["options"] = {"hda": [], "hdca": [], "dce": []} + d["tag"] = self.tag # return dictionary without options if context is unavailable history = trans.history @@ -2279,26 +2411,30 @@ class DataCollectionToolParameter(BaseDataToolParameter): # append DCE if isinstance(other_values.get(self.name), DatasetCollectionElement): dce = other_values[self.name] - d['options']['dce'].append({ - 'id': trans.security.encode_id(dce.id), - 'hid': None, - 'name': dce.element_identifier, - 'src': 'dce', - 'tags': [] - }) + d["options"]["dce"].append( + { + "id": trans.security.encode_id(dce.id), + "hid": None, + "name": dce.element_identifier, + "src": "dce", + "tags": [], + } + ) # append directly matched collections for hdca, implicit_conversion in self.match_collections(trans, history, dataset_collection_matcher): name = hdca.name if implicit_conversion: name = f"{name} (with implicit datatype conversion)" - d['options']['hdca'].append({ - 'id': trans.security.encode_id(hdca.id), - 'hid': hdca.hid, - 'name': name, - 'src': 'hdca', - 'tags': [t.user_tname if not t.value else f"{t.user_tname}:{t.value}" for t in hdca.tags] - }) + d["options"]["hdca"].append( + { + "id": trans.security.encode_id(hdca.id), + "hid": hdca.hid, + "name": name, + "src": "hdca", + "tags": [t.user_tname if not t.value else f"{t.user_tname}:{t.value}" for t in hdca.tags], + } + ) # append matching subcollections for hdca, implicit_conversion in self.match_multirun_collections(trans, history, dataset_collection_matcher): @@ -2306,17 +2442,19 @@ class DataCollectionToolParameter(BaseDataToolParameter): name = hdca.name if implicit_conversion: name = f"{name} (with implicit datatype conversion)" - d['options']['hdca'].append({ - 'id': trans.security.encode_id(hdca.id), - 'hid': hdca.hid, - 'name': name, - 'src': 'hdca', - 'tags': [t.user_tname if not t.value else f"{t.user_tname}:{t.value}" for t in hdca.tags], - 'map_over_type': subcollection_type - }) + d["options"]["hdca"].append( + { + "id": trans.security.encode_id(hdca.id), + "hid": hdca.hid, + "name": name, + "src": "hdca", + "tags": [t.user_tname if not t.value else f"{t.user_tname}:{t.value}" for t in hdca.tags], + "map_over_type": subcollection_type, + } + ) # sort both lists - d['options']['hdca'] = sorted(d['options']['hdca'], key=lambda k: k.get('hid', -1), reverse=True) + d["options"]["hdca"] = sorted(d["options"]["hdca"], key=lambda k: k.get("hid", -1), reverse=True) # return final dictionary return d @@ -2343,7 +2481,7 @@ class LibraryDatasetToolParameter(ToolParameter): def __init__(self, tool, input_source, context=None): input_source = ensure_input_source(input_source) super().__init__(tool, input_source) - self.multiple = input_source.get_bool('multiple', True) + self.multiple = input_source.get_bool("multiple", True) def from_json(self, value, trans, other_values=None): other_values = other_values or {} @@ -2351,7 +2489,7 @@ class LibraryDatasetToolParameter(ToolParameter): def to_param_dict_string(self, value, other_values=None): if value is None: - return 'None' + return "None" elif self.multiple: return [dataset.get_file_name() for dataset in value] else: @@ -2369,17 +2507,13 @@ class LibraryDatasetToolParameter(ToolParameter): lda_id = app.security.encode_id(item.id) if use_security else item.id lda_name = item.name elif isinstance(item, dict): - lda_id = item.get('id') - lda_name = item.get('name') + lda_id = item.get("id") + lda_name = item.get("name") else: lst = [] break if lda_id is not None: - lst.append({ - 'id': lda_id, - 'name': lda_name, - 'src': 'ldda' - }) + lst.append({"id": lda_id, "name": lda_name, "src": "ldda"}) if len(lst) == 0: return None else: @@ -2402,17 +2536,21 @@ class LibraryDatasetToolParameter(ToolParameter): else: lda_id = None if isinstance(item, dict): - lda_id = item.get('id') + lda_id = item.get("id") elif isinstance(item, str): lda_id = item else: lst = [] break - lda = app.model.context.query(LibraryDatasetDatasetAssociation).get(lda_id if isinstance(lda_id, int) else app.security.decode_id(lda_id)) + lda = app.model.context.query(LibraryDatasetDatasetAssociation).get( + lda_id if isinstance(lda_id, int) else app.security.decode_id(lda_id) + ) if lda is not None: lst.append(lda) elif validate: - raise ParameterValueError("one of the selected library datasets is invalid or not available anymore", self.name) + raise ParameterValueError( + "one of the selected library datasets is invalid or not available anymore", self.name + ) if len(lst) == 0: if not self.optional and validate: raise ParameterValueError("invalid library dataset selected", self.name) @@ -2422,7 +2560,7 @@ class LibraryDatasetToolParameter(ToolParameter): def to_dict(self, trans, other_values=None): d = super().to_dict(trans) - d['multiple'] = self.multiple + d["multiple"] = self.multiple return d @@ -2530,19 +2668,23 @@ parameter_types = dict( library_data=LibraryDatasetToolParameter, rules=RulesListToolParameter, directory_uri=DirectoryUriToolParameter, - drill_down=DrillDownSelectToolParameter + drill_down=DrillDownSelectToolParameter, ) def runtime_to_json(runtime_value): - if isinstance(runtime_value, ConnectedValue) or (isinstance(runtime_value, dict) and runtime_value["__class__"] == "ConnectedValue"): + if isinstance(runtime_value, ConnectedValue) or ( + isinstance(runtime_value, dict) and runtime_value["__class__"] == "ConnectedValue" + ): return {"__class__": "ConnectedValue"} else: return {"__class__": "RuntimeValue"} def runtime_to_object(runtime_value): - if isinstance(runtime_value, ConnectedValue) or (isinstance(runtime_value, dict) and runtime_value["__class__"] == "ConnectedValue"): + if isinstance(runtime_value, ConnectedValue) or ( + isinstance(runtime_value, dict) and runtime_value["__class__"] == "ConnectedValue" + ): return ConnectedValue() else: return RuntimeValue() diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py index 6eaac75fad4..af9fbd2f1f0 100644 --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -12,11 +12,11 @@ from galaxy.model import ( HistoryDatasetAssociation, HistoryDatasetCollectionAssociation, MetadataFile, - User + User, ) from galaxy.tools.wrappers import ( DatasetFilenameWrapper, - DatasetListWrapper + DatasetListWrapper, ) from galaxy.util import string_as_bool from . import validation @@ -28,10 +28,11 @@ class Filter: """ A filter takes the current options list and modifies it. """ + @classmethod def from_element(cls, d_option, elem): """Loads the proper filter by the type attribute of elem""" - type = elem.get('type', None) + type = elem.get("type", None) assert type is not None, "Required 'type' attribute missing from filter" return filter_types[type.strip()](d_option, elem) @@ -69,7 +70,7 @@ class StaticValueFilter(Filter): column = elem.get("column", None) assert column is not None, "Required 'column' attribute missing from filter, when loading from file" self.column = d_option.column_spec_to_index(column) - self.keep = string_as_bool(elem.get("keep", 'True')) + self.keep = string_as_bool(elem.get("keep", "True")) def filter_options(self, options, trans, other_values): rval = [] @@ -105,7 +106,7 @@ class RegexpFilter(Filter): column = elem.get("column", None) assert column is not None, "Required 'column' attribute missing from filter, when loading from file" self.column = d_option.column_spec_to_index(column) - self.keep = string_as_bool(elem.get("keep", 'True')) + self.keep = string_as_bool(elem.get("keep", "True")) def filter_options(self, options, trans, other_values): rval = [] @@ -152,7 +153,9 @@ class DataMetaFilter(Filter): assert self.key is not None, "Required 'key' attribute missing from filter" self.column = elem.get("column", None) if self.column is None: - assert self.dynamic_option.file_fields is None and self.dynamic_option.dataset_ref_name is None, "Required 'column' attribute missing from filter, when loading from file" + assert ( + self.dynamic_option.file_fields is None and self.dynamic_option.dataset_ref_name is None + ), "Required 'column' attribute missing from filter, when loading from file" else: self.column = d_option.column_spec_to_index(self.column) self.multiple = string_as_bool(elem.get("multiple", "False")) @@ -220,11 +223,7 @@ class DataMetaFilter(Filter): return rval else: if not self.dynamic_option.columns: - self.dynamic_option.columns = { - "name": 0, - "value": 1, - "selected": 2 - } + self.dynamic_option.columns = {"name": 0, "value": 1, "selected": 2} self.dynamic_option.largest_index = 2 for value in meta_value: options.append((value, value, False)) @@ -257,10 +256,10 @@ class ParamValueFilter(Filter): column = elem.get("column", None) assert column is not None, "Required 'column' attribute missing from filter" self.column = d_option.column_spec_to_index(column) - self.keep = string_as_bool(elem.get("keep", 'True')) + self.keep = string_as_bool(elem.get("keep", "True")) self.ref_attribute = elem.get("ref_attribute", None) if self.ref_attribute: - self.ref_attribute = self.ref_attribute.split('.') + self.ref_attribute = self.ref_attribute.split(".") else: self.ref_attribute = [] @@ -353,7 +352,7 @@ class MultipleSplitterFilter(Filter): for fields in options: for column in self.columns: for field in fields[column].split(self.separator): - rval.append(fields[0:column] + [field] + fields[column + 1:]) + rval.append(fields[0:column] + [field] + fields[column + 1 :]) return rval @@ -424,8 +423,8 @@ class AdditionalValueFilter(Filter): add_value = [] for _ in range(self.dynamic_option.largest_index + 1): add_value.append("") - value_col = self.dynamic_option.columns.get('value', 0) - name_col = self.dynamic_option.columns.get('name', value_col) + value_col = self.dynamic_option.columns.get("value", 0) + name_col = self.dynamic_option.columns.get("name", value_col) # Set name first, then value, in case they are the same column add_value[name_col] = self.name add_value[value_col] = self.value @@ -459,7 +458,11 @@ class RemoveValueFilter(Filter): self.ref_name = elem.get("ref", None) self.meta_ref = elem.get("meta_ref", None) self.metadata_key = elem.get("key", None) - assert self.value is not None or self.ref_name is not None or (self.meta_ref is not None and self.metadata_key is not None), ValueError("Required 'value', or 'ref', or 'meta_ref' and 'key' attributes missing from filter") + assert ( + self.value is not None + or self.ref_name is not None + or (self.meta_ref is not None and self.metadata_key is not None) + ), ValueError("Required 'value', or 'ref', or 'meta_ref' and 'key' attributes missing from filter") self.multiple = string_as_bool(elem.get("multiple", "False")) self.separator = elem.get("separator", ",") @@ -488,11 +491,13 @@ class RemoveValueFilter(Filter): data_ref = other_values.get(self.meta_ref) if isinstance(data_ref, HistoryDatasetCollectionAssociation): data_ref = data_ref.to_hda_representative() - if not isinstance(data_ref, HistoryDatasetAssociation) and not isinstance(data_ref, DatasetFilenameWrapper): + if not isinstance(data_ref, HistoryDatasetAssociation) and not isinstance( + data_ref, DatasetFilenameWrapper + ): return options # cannot modify options value = data_ref.metadata.get(self.metadata_key, None) # Default to the second column (i.e. 1) since this used to work only on options produced by the data_meta filter - value_col = self.dynamic_option.columns.get('value', 1) + value_col = self.dynamic_option.columns.get("value", 1) return [option for option in options if not compare_value(option[value_col], value)] @@ -516,16 +521,18 @@ class SortByColumnFilter(Filter): return sorted(options, key=lambda x: x[self.column]) -filter_types = dict(data_meta=DataMetaFilter, - param_value=ParamValueFilter, - static_value=StaticValueFilter, - regexp=RegexpFilter, - unique_value=UniqueValueFilter, - multiple_splitter=MultipleSplitterFilter, - attribute_value_splitter=AttributeValueSplitterFilter, - add_value=AdditionalValueFilter, - remove_value=RemoveValueFilter, - sort_by=SortByColumnFilter) +filter_types = dict( + data_meta=DataMetaFilter, + param_value=ParamValueFilter, + static_value=StaticValueFilter, + regexp=RegexpFilter, + unique_value=UniqueValueFilter, + multiple_splitter=MultipleSplitterFilter, + attribute_value_splitter=AttributeValueSplitterFilter, + add_value=AdditionalValueFilter, + remove_value=RemoveValueFilter, + sort_by=SortByColumnFilter, +) class DynamicOptions: @@ -534,11 +541,12 @@ class DynamicOptions: def __init__(self, elem, tool_param): def load_from_parameter(from_parameter, transform_lines=None): obj = self.tool_param - for field in from_parameter.split('.'): + for field in from_parameter.split("."): obj = getattr(obj, field) if transform_lines: - obj = eval(transform_lines, {'self': self, 'obj': obj}) + obj = eval(transform_lines, {"self": self, "obj": obj}) return self.parse_file_fields(obj) + self.tool_param = tool_param self.columns = {} self.filters = [] @@ -552,14 +560,14 @@ class DynamicOptions: self.converter_safe = True # Parse the tag - self.separator = elem.get('separator', '\t') - self.line_startswith = elem.get('startswith', None) - data_file = elem.get('from_file', None) + self.separator = elem.get("separator", "\t") + self.line_startswith = elem.get("startswith", None) + data_file = elem.get("from_file", None) self.index_file = None self.missing_index_file = None - dataset_file = elem.get('from_dataset', None) - from_parameter = elem.get('from_parameter', None) - self.tool_data_table_name = elem.get('from_data_table', None) + dataset_file = elem.get("from_dataset", None) + from_parameter = elem.get("from_parameter", None) + self.tool_data_table_name = elem.get("from_data_table", None) # Options are defined from a data table loaded by the app self._tool_data_table = None self.elem = elem @@ -568,7 +576,9 @@ class DynamicOptions: # Options are defined by parsing tabular text data from a data file # on disk, a dataset, or the value of another parameter - if not self.tool_data_table_name and (data_file is not None or dataset_file is not None or from_parameter is not None): + if not self.tool_data_table_name and ( + data_file is not None or dataset_file is not None or from_parameter is not None + ): self.parse_column_definitions(elem) if data_file is not None: data_file = data_file.strip() @@ -581,20 +591,20 @@ class DynamicOptions: else: self.missing_index_file = data_file elif dataset_file is not None: - self.meta_file_key = elem.get('meta_file_key', None) + self.meta_file_key = elem.get("meta_file_key", None) self.dataset_ref_name = dataset_file self.has_dataset_dependencies = True self.converter_safe = False elif from_parameter is not None: - transform_lines = elem.get('transform_lines', None) + transform_lines = elem.get("transform_lines", None) self.file_fields = list(load_from_parameter(from_parameter, transform_lines)) # Load filters - for filter_elem in elem.findall('filter'): + for filter_elem in elem.findall("filter"): self.filters.append(Filter.from_element(self, filter_elem)) # Load Validators - for validator in elem.findall('validator'): + for validator in elem.findall("validator"): self.validators.append(validation.Validator.from_element(self.tool_param, validator)) if self.dataset_ref_name: @@ -628,24 +638,24 @@ class DynamicOptions: return None def parse_column_definitions(self, elem): - for column_elem in elem.findall('column'): - name = column_elem.get('name', None) + for column_elem in elem.findall("column"): + name = column_elem.get("name", None) assert name is not None, "Required 'name' attribute missing from column def" - index = column_elem.get('index', None) + index = column_elem.get("index", None) assert index is not None, "Required 'index' attribute missing from column def" index = int(index) self.columns[name] = index if index > self.largest_index: self.largest_index = index - assert 'value' in self.columns, "Required 'value' column missing from column def" - if 'name' not in self.columns: - self.columns['name'] = self.columns['value'] + assert "value" in self.columns, "Required 'value' column missing from column def" + if "name" not in self.columns: + self.columns["name"] = self.columns["value"] def parse_file_fields(self, reader): rval = [] field_count = None for line in reader: - if line.startswith('#') or (self.line_startswith and not line.startswith(self.line_startswith)): + if line.startswith("#") or (self.line_startswith and not line.startswith(self.line_startswith)): continue line = line.rstrip("\n\r") if line: @@ -659,8 +669,10 @@ class DynamicOptions: except AttributeError: name = "a configuration file" # Perhaps this should be an error, but even a warning is useful. - log.warning("Inconsistent number of fields (%i vs %i) in %s using separator %r, check line: %r" % - (field_count, len(fields), name, self.separator, line)) + log.warning( + "Inconsistent number of fields (%i vs %i) in %s using separator %r, check line: %r" + % (field_count, len(fields), name, self.separator, line) + ) rval.append(fields) return rval @@ -683,10 +695,14 @@ class DynamicOptions: try: datasets = _get_ref_data(other_values, self.dataset_ref_name) except KeyError: # no such dataset - log.warning(f"Parameter {self.tool_param.name}: could not create dynamic options from_dataset: {self.dataset_ref_name} unknown") + log.warning( + f"Parameter {self.tool_param.name}: could not create dynamic options from_dataset: {self.dataset_ref_name} unknown" + ) return [] except ValueError: # not a valid dataset - log.warning(f"Parameter {self.tool_param.name}: could not create dynamic options from_dataset: {self.dataset_ref_name} not a data or collection parameter") + log.warning( + f"Parameter {self.tool_param.name}: could not create dynamic options from_dataset: {self.dataset_ref_name} not a data or collection parameter" + ) return [] options = [] @@ -695,12 +711,14 @@ class DynamicOptions: if meta_file_key: dataset = getattr(dataset.metadata, meta_file_key, None) if not isinstance(dataset, MetadataFile): - log.warning(f"The meta_file_key `{meta_file_key}` was invalid or the referred object was not a valid file type metadata!") + log.warning( + f"The meta_file_key `{meta_file_key}` was invalid or the referred object was not a valid file type metadata!" + ) continue - if getattr(dataset, 'purged', False) or getattr(dataset, 'deleted', False): + if getattr(dataset, "purged", False) or getattr(dataset, "deleted", False): log.warning(f"The metadata file inferred from key `{meta_file_key}` was deleted!") continue - if not hasattr(dataset, 'file_name'): + if not hasattr(dataset, "file_name"): continue # Ensure parsing dynamic options does not consume more than a megabyte worth memory. path = dataset.file_name @@ -728,7 +746,7 @@ class DynamicOptions: Return a list of fields with column 'value' matching provided value. """ rval = [] - val_index = self.columns['value'] + val_index = self.columns["value"] for fields in self.get_fields(trans, other_values): if fields[val_index] == value: rval.append(fields) @@ -753,10 +771,15 @@ class DynamicOptions: def get_options(self, trans, other_values): rval = [] - if self.file_fields is not None or self.tool_data_table is not None or self.dataset_ref_name is not None or self.missing_index_file: + if ( + self.file_fields is not None + or self.tool_data_table is not None + or self.dataset_ref_name is not None + or self.missing_index_file + ): options = self.get_fields(trans, other_values) for fields in options: - rval.append((fields[self.columns['name']], fields[self.columns['value']], False)) + rval.append((fields[self.columns["name"]], fields[self.columns["value"]], False)) else: for filter in self.filters: rval = filter.filter_options(rval, trans, other_values) @@ -782,7 +805,16 @@ def _get_ref_data(other_values, ref_name): - a ValueError is raised if the element is not of the type DatasetFilenameWrapper, HistoryDatasetAssociation, DatasetListWrapper, HistoryDatasetCollectionAssociation, list """ ref = other_values[ref_name] - if not isinstance(ref, (DatasetFilenameWrapper, HistoryDatasetAssociation, DatasetListWrapper, HistoryDatasetCollectionAssociation, list)): + if not isinstance( + ref, + ( + DatasetFilenameWrapper, + HistoryDatasetAssociation, + DatasetListWrapper, + HistoryDatasetCollectionAssociation, + list, + ), + ): raise ValueError if isinstance(ref, (DatasetFilenameWrapper, HistoryDatasetAssociation)): ref = [ref] diff --git a/lib/galaxy/tools/remote_tool_eval.py b/lib/galaxy/tools/remote_tool_eval.py index 0235311023a..a9d6513954b 100644 --- a/lib/galaxy/tools/remote_tool_eval.py +++ b/lib/galaxy/tools/remote_tool_eval.py @@ -45,7 +45,8 @@ class ToolAppConfig(NamedTuple): class ToolApp(MinimalToolApp): """Dummy App that allows loading tools""" - name = 'tool_app' + + name = "tool_app" def __init__( self, @@ -70,22 +71,23 @@ def main(TMPDIR, WORKING_DIRECTORY, IMPORT_STORE_DIRECTORY): metadata_params = get_metadata_params(WORKING_DIRECTORY) datatypes_config = metadata_params["datatypes_config"] if not os.path.exists(datatypes_config): - datatypes_config = os.path.join(WORKING_DIRECTORY, 'configs', datatypes_config) + datatypes_config = os.path.join(WORKING_DIRECTORY, "configs", datatypes_config) datatypes_registry = validate_and_load_datatypes_config(datatypes_config) object_store = get_object_store(WORKING_DIRECTORY) import_store = store.imported_store_for_metadata(IMPORT_STORE_DIRECTORY) # TODO: clean up random places from which we read files in the working directory - job_io = JobIO.from_json(os.path.join(IMPORT_STORE_DIRECTORY, 'job_io.json'), sa_session=import_store.sa_session) + job_io = JobIO.from_json(os.path.join(IMPORT_STORE_DIRECTORY, "job_io.json"), sa_session=import_store.sa_session) tool_app_config = ToolAppConfig( - name='tool_app', + name="tool_app", tool_data_path=job_io.tool_data_path, galaxy_data_manager_data_path=job_io.galaxy_data_manager_data_path, nginx_upload_path=TMPDIR, len_file_path=job_io.len_file_path, builds_file_path=job_io.builds_file_path, root=TMPDIR, - is_admin_user=lambda _: job_io.user_context.is_admin) - with open(os.path.join(IMPORT_STORE_DIRECTORY, 'tool_data_tables.json')) as data_tables_json: + is_admin_user=lambda _: job_io.user_context.is_admin, + ) + with open(os.path.join(IMPORT_STORE_DIRECTORY, "tool_data_tables.json")) as data_tables_json: tdtm = ToolDataTableManager.from_dict(json.load(data_tables_json)) app = ToolApp( sa_session=import_store.sa_session, @@ -98,9 +100,11 @@ def main(TMPDIR, WORKING_DIRECTORY, IMPORT_STORE_DIRECTORY): # TODO: could try to serialize just a minimal tool variant instead of the whole thing ? tool_source = get_tool_source(tool_source_class=job_io.tool_source_class, raw_tool_source=job_io.tool_source) tool = create_tool_from_source(app, tool_source=tool_source, tool_dir=job_io.tool_dir) - tool_evaluator = evaluation.RemoteToolEvaluator(app=app, tool=tool, job=job_io.job, local_working_directory=WORKING_DIRECTORY) + tool_evaluator = evaluation.RemoteToolEvaluator( + app=app, tool=tool, job=job_io.job, local_working_directory=WORKING_DIRECTORY + ) tool_evaluator.set_compute_environment(compute_environment=SharedComputeEnvironment(job_io=job_io, job=job_io.job)) - with open(os.path.join(WORKING_DIRECTORY, 'tool_script.sh'), 'a') as out: + with open(os.path.join(WORKING_DIRECTORY, "tool_script.sh"), "a") as out: command_line, version_command_line, extra_filenames, environment_variables = tool_evaluator.build() out.write(f'{version_command_line or ""}{command_line}') @@ -109,17 +113,17 @@ if __name__ == "__main__": TMPDIR = tempfile.mkdtemp() WORKING_DIRECTORY = os.getcwd() WORKING_PARENT = os.path.join(WORKING_DIRECTORY, os.path.pardir) - if not os.path.isdir("working") and os.path.isdir(os.path.join(WORKING_PARENT, 'working')): + if not os.path.isdir("working") and os.path.isdir(os.path.join(WORKING_PARENT, "working")): # We're probably in pulsar WORKING_DIRECTORY = WORKING_PARENT - METADATA_DIRECTORY = os.path.join(WORKING_DIRECTORY, 'metadata') - IMPORT_STORE_DIRECTORY = os.path.join(METADATA_DIRECTORY, 'outputs_new') - EXPORT_STORE_DIRECTORY = os.path.join(METADATA_DIRECTORY, 'outputs_populated') + METADATA_DIRECTORY = os.path.join(WORKING_DIRECTORY, "metadata") + IMPORT_STORE_DIRECTORY = os.path.join(METADATA_DIRECTORY, "outputs_new") + EXPORT_STORE_DIRECTORY = os.path.join(METADATA_DIRECTORY, "outputs_populated") try: main(TMPDIR, WORKING_DIRECTORY, IMPORT_STORE_DIRECTORY) except Exception: os.makedirs(EXPORT_STORE_DIRECTORY, exist_ok=True) - with open(os.path.join(EXPORT_STORE_DIRECTORY, 'traceback.txt'), 'w') as out: + with open(os.path.join(EXPORT_STORE_DIRECTORY, "traceback.txt"), "w") as out: out.write(traceback.format_exc()) raise finally: diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index c45285715b1..ec2b61cd186 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -38,24 +38,28 @@ from urllib.parse import ( ) import requests +from boltons.iterutils import ( + default_enter, + remap, +) +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + try: import grp except ImportError: # For Pulsar on Windows (which does not use the function that uses grp) grp = None # type: ignore[assignment] -from boltons.iterutils import ( - default_enter, - remap, -) +try: + import uwsgi +except ImportError: + uwsgi = None LXML_AVAILABLE = True try: from lxml import etree except ImportError: LXML_AVAILABLE = False import xml.etree.ElementTree as etree # type: ignore[assignment,no-redef] -from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry - try: import docutils.core as docutils_core import docutils.writers.html4css1 as docutils_html4css1 @@ -63,14 +67,13 @@ except ImportError: docutils_core = None # type: ignore[assignment] docutils_html4css1 = None # type: ignore[assignment] -try: - import uwsgi -except ImportError: - uwsgi = None - from .custom_logging import get_logger from .inflection import Inflector -from .path import safe_contains, safe_makedirs, safe_relpath # noqa: F401 +from .path import ( # noqa: F401 + safe_contains, + safe_makedirs, + safe_relpath, +) inflector = Inflector() @@ -82,16 +85,16 @@ namedtuple = collections.namedtuple CHUNK_SIZE = 65536 # 64k DATABASE_MAX_STRING_SIZE = 32768 -DATABASE_MAX_STRING_SIZE_PRETTY = '32K' +DATABASE_MAX_STRING_SIZE_PRETTY = "32K" DEFAULT_SOCKET_TIMEOUT = 600 -gzip_magic = b'\x1f\x8b' -bz2_magic = b'BZh' -DEFAULT_ENCODING = os.environ.get('GALAXY_DEFAULT_ENCODING', 'utf-8') -NULL_CHAR = b'\x00' +gzip_magic = b"\x1f\x8b" +bz2_magic = b"BZh" +DEFAULT_ENCODING = os.environ.get("GALAXY_DEFAULT_ENCODING", "utf-8") +NULL_CHAR = b"\x00" BINARY_CHARS = [NULL_CHAR] -FILENAME_VALID_CHARS = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' +FILENAME_VALID_CHARS = ".,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" RW_R__R__ = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH RWXR_XR_X = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH @@ -109,23 +112,23 @@ def str_removeprefix(s: str, prefix: str): if sys.version_info >= (3, 9): return s.removeprefix(prefix) if s.startswith(prefix): - return s[len(prefix):] + return s[len(prefix) :] return s def remove_protocol_from_url(url): - """ Supplied URL may be null, if not ensure http:// or https:// + """Supplied URL may be null, if not ensure http:// or https:// etc... is stripped off. """ if url is None: return url # We have a URL - if url.find('://') > 0: - new_url = url.split('://')[1] + if url.find("://") > 0: + new_url = url.split("://")[1] else: new_url = url - return new_url.rstrip('/') + return new_url.rstrip("/") def is_binary(value): @@ -184,16 +187,16 @@ def directory_hash_id(id): # Drop the last three digits -- 1000 files per directory padded = padded[:-3] # Break into chunks of three - return [padded[i * 3:(i + 1) * 3] for i in range(len(padded) // 3)] + return [padded[i * 3 : (i + 1) * 3] for i in range(len(padded) // 3)] else: # assume it is a UUID return list(iter(s[0:3])) def get_charset_from_http_headers(headers, default=None): - rval = headers.get('content-type', None) - if rval and 'charset=' in rval: - rval = rval.split('charset=')[-1].split(';')[0].strip() + rval = headers.get("content-type", None) + if rval and "charset=" in rval: + rval = rval.split("charset=")[-1].split(";")[0].strip() if rval: return rval return default @@ -201,12 +204,14 @@ def get_charset_from_http_headers(headers, default=None): def synchronized(func): """This wrapper will serialize access to 'func' to a single thread. Use it as a decorator.""" + def caller(*params, **kparams): _lock.acquire(True) # Wait try: return func(*params, **kparams) finally: _lock.release() + return caller @@ -269,7 +274,7 @@ def parse_xml(fname, strip_whitespace=True, remove_comments=True): tree = etree.parse(fname, parser=parser) root = tree.getroot() if strip_whitespace: - for elem in root.iter('*'): + for elem in root.iter("*"): if elem.text is not None: elem.text = elem.text.strip() if elem.tail is not None: @@ -289,12 +294,12 @@ def parse_xml_string(xml_string, strip_whitespace=True): try: tree = etree.fromstring(xml_string) except ValueError as e: - if 'strings with encoding declaration are not supported' in unicodify(e): - tree = etree.fromstring(xml_string.encode('utf-8')) + if "strings with encoding declaration are not supported" in unicodify(e): + tree = etree.fromstring(xml_string.encode("utf-8")) else: raise e if strip_whitespace: - for elem in tree.iter('*'): + for elem in tree.iter("*"): if elem.text is not None: elem.text = elem.text.strip() if elem.tail is not None: @@ -312,18 +317,18 @@ def xml_to_string(elem, pretty=False): """ try: if elem is not None: - xml_str = etree.tostring(elem, encoding='unicode') + xml_str = etree.tostring(elem, encoding="unicode") else: - xml_str = '' + xml_str = "" except TypeError as e: # we assume this is a comment - if hasattr(elem, 'text'): + if hasattr(elem, "text"): return f"\n" else: raise e if xml_str and pretty: - pretty_string = xml.dom.minidom.parseString(xml_str).toprettyxml(indent=' ') - return "\n".join(line for line in pretty_string.split('\n') if not re.match(r'^[\s\\nb\']*$', line)) + pretty_string = xml.dom.minidom.parseString(xml_str).toprettyxml(indent=" ") + return "\n".join(line for line in pretty_string.split("\n") if not re.match(r"^[\s\\nb\']*$", line)) return xml_str @@ -366,7 +371,7 @@ def xml_element_to_dict(elem): if elem.text: text = elem.text.strip() if text and sub_elems or elem.attrib: - rval[elem.tag]['#text'] = text + rval[elem.tag]["#text"] = text else: rval[elem.tag] = text @@ -374,7 +379,7 @@ def xml_element_to_dict(elem): def pretty_print_xml(elem, level=0): - pad = ' ' + pad = " " i = "\n" + level * pad if len(elem): if not elem.text or not elem.text.strip(): @@ -412,14 +417,16 @@ def get_file_size(value, default=None): return default -def shrink_stream_by_size(value, size, join_by=b"..", left_larger=True, beginning_on_size_error=False, end_on_size_error=False): +def shrink_stream_by_size( + value, size, join_by=b"..", left_larger=True, beginning_on_size_error=False, end_on_size_error=False +): """ Shrinks bytes read from `value` to `size`. `value` needs to implement tell/seek, so files need to be opened in binary mode. Returns unicode text with invalid characters replaced. """ - rval = b'' + rval = b"" join_by = smart_str(join_by) if get_file_size(value) > size: start = value.tell() @@ -435,7 +442,9 @@ def shrink_stream_by_size(value, size, join_by=b"..", left_larger=True, beginnin rval = value.read(size) value.seek(start) return rval - raise ValueError('With the provided join_by value (%s), the minimum size value is %i.' % (join_by, min_size)) + raise ValueError( + "With the provided join_by value (%s), the minimum size value is %i." % (join_by, min_size) + ) left_index = right_index = int((size - len_join_by) / 2) if left_index + right_index + len_join_by < size: if left_larger: @@ -455,17 +464,17 @@ def shrink_stream_by_size(value, size, join_by=b"..", left_larger=True, beginnin def shrink_and_unicodify(stream): - stream = unicodify(stream, strip_null=True) or '' - if (len(stream) > DATABASE_MAX_STRING_SIZE): - stream = shrink_string_by_size(stream, - DATABASE_MAX_STRING_SIZE, - join_by="\n..\n", - left_larger=True, - beginning_on_size_error=True) + stream = unicodify(stream, strip_null=True) or "" + if len(stream) > DATABASE_MAX_STRING_SIZE: + stream = shrink_string_by_size( + stream, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True + ) return stream -def shrink_string_by_size(value, size, join_by="..", left_larger=True, beginning_on_size_error=False, end_on_size_error=False): +def shrink_string_by_size( + value, size, join_by="..", left_larger=True, beginning_on_size_error=False, end_on_size_error=False +): if len(value) > size: len_join_by = len(join_by) min_size = len_join_by + 2 @@ -474,7 +483,9 @@ def shrink_string_by_size(value, size, join_by="..", left_larger=True, beginning return value[:size] elif end_on_size_error: return value[-size:] - raise ValueError('With the provided join_by value (%s), the minimum size value is %i.' % (join_by, min_size)) + raise ValueError( + "With the provided join_by value (%s), the minimum size value is %i." % (join_by, min_size) + ) left_index = right_index = int((size - len_join_by) / 2) if left_index + right_index + len_join_by < size: if left_larger: @@ -513,7 +524,7 @@ def pretty_print_time_interval(time=False, precise=False, utc=False): day_diff = diff.days if day_diff < 0: - return '' + return "" if precise: if day_diff == 0: @@ -562,19 +573,21 @@ def pretty_print_json(json_data, is_json_string=False): valid_chars = set(string.ascii_letters + string.digits + " -=_.()/+*^,:?!") # characters that are allowed but need to be escaped -mapped_chars = {'>': '__gt__', - '<': '__lt__', - "'": '__sq__', - '"': '__dq__', - '[': '__ob__', - ']': '__cb__', - '{': '__oc__', - '}': '__cc__', - '@': '__at__', - '\n': '__cn__', - '\r': '__cr__', - '\t': '__tc__', - '#': '__pd__'} +mapped_chars = { + ">": "__gt__", + "<": "__lt__", + "'": "__sq__", + '"': "__dq__", + "[": "__ob__", + "]": "__cb__", + "{": "__oc__", + "}": "__cc__", + "@": "__at__", + "\n": "__cn__", + "\r": "__cr__", + "\t": "__tc__", + "#": "__pd__", +} def restore_text(text, character_map=mapped_chars): @@ -586,19 +599,24 @@ def restore_text(text, character_map=mapped_chars): return text -def sanitize_text(text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X'): +def sanitize_text(text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character="X"): """ Restricts the characters that are allowed in text; accepts both strings and lists of strings; non-string entities will be cast to strings. """ if isinstance(text, list): - return [sanitize_text(x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character) for x in text] + return [ + sanitize_text( + x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character + ) + for x in text + ] if not isinstance(text, str): text = smart_str(text) return _sanitize_text_helper(text, valid_characters=valid_characters, character_map=character_map) -def _sanitize_text_helper(text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X'): +def _sanitize_text_helper(text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character="X"): """Restricts the characters that are allowed in a string""" out = [] @@ -609,35 +627,48 @@ def _sanitize_text_helper(text, valid_characters=valid_chars, character_map=mapp out.append(character_map[c]) else: out.append(invalid_character) # makes debugging easier - return ''.join(out) + return "".join(out) -def sanitize_lists_to_string(values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X'): +def sanitize_lists_to_string(values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character="X"): if isinstance(values, list): rval = [] for value in values: - rval.append(sanitize_lists_to_string(value, - valid_characters=valid_characters, - character_map=character_map, - invalid_character=invalid_character)) + rval.append( + sanitize_lists_to_string( + value, + valid_characters=valid_characters, + character_map=character_map, + invalid_character=invalid_character, + ) + ) values = ",".join(rval) else: - values = sanitize_text(values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character) + values = sanitize_text( + values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character + ) return values -def sanitize_param(value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X'): +def sanitize_param(value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character="X"): """Clean incoming parameters (strings or lists)""" if isinstance(value, str): - return sanitize_text(value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character) + return sanitize_text( + value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character + ) elif isinstance(value, list): - return [sanitize_text(x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character) for x in value] + return [ + sanitize_text( + x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character + ) + for x in value + ] else: - raise Exception(f'Unknown parameter type ({type(value)})') + raise Exception(f"Unknown parameter type ({type(value)})") -valid_filename_chars = set(string.ascii_letters + string.digits + '_.') -invalid_filenames = ['', '.', '..'] +valid_filename_chars = set(string.ascii_letters + string.digits + "_.") +invalid_filenames = ["", ".", ".."] def sanitize_for_filename(text, default=None): @@ -650,8 +681,8 @@ def sanitize_for_filename(text, default=None): if c in valid_filename_chars: out.append(c) else: - out.append('_') - out = ''.join(out) + out.append("_") + out = "".join(out) if out in invalid_filenames: if default is None: return sanitize_for_filename(str(unique_id())) @@ -705,13 +736,15 @@ def mask_password_from_url(url): # This can manipulate the input other than just masking password, # so the previous string replace method is preferred when the # password doesn't appear twice in the url - split = split._replace(netloc=split.netloc.replace(f"{split.username}:{split.password}", f'{split.username}:********')) + split = split._replace( + netloc=split.netloc.replace(f"{split.username}:{split.password}", f"{split.username}:********") + ) url = urlunsplit(split) return url def ready_name_for_url(raw_name): - """ General method to convert a string (i.e. object name) to a URL-ready + """General method to convert a string (i.e. object name) to a URL-ready slug. >>> ready_name_for_url( "My Cool Object" ) @@ -727,7 +760,7 @@ def ready_name_for_url(raw_name): # Remove all non-alphanumeric characters. slug_base = re.sub(r"[^a-zA-Z0-9\-]", "", slug_base) # Remove trailing '-'. - if slug_base.endswith('-'): + if slug_base.endswith("-"): slug_base = slug_base[:-1] return slug_base @@ -768,7 +801,7 @@ def in_directory(file, directory, local_path_module=os.path): False """ if local_path_module != os.path: - _safe_contains = importlib.import_module(f'galaxy.util.path.{local_path_module.__name__}').safe_contains + _safe_contains = importlib.import_module(f"galaxy.util.path.{local_path_module.__name__}").safe_contains else: directory = os.path.realpath(directory) _safe_contains = safe_contains @@ -791,9 +824,7 @@ def merge_sorted_iterables(operator, *iterables): yield from first_iterable else: yield from __merge_two_sorted_iterables( - operator, - iter(first_iterable), - merge_sorted_iterables(operator, *iterables[1:]) + operator, iter(first_iterable), merge_sorted_iterables(operator, *iterables[1:]) ) @@ -848,7 +879,7 @@ class Params: """ # is NEVER_SANITIZE required now that sanitizing for tool parameters can be controlled on a per parameter basis and occurs via InputValueWrappers? - NEVER_SANITIZE = ['file_data', 'url_paste', 'URL', 'filesystem_paths'] + NEVER_SANITIZE = ["file_data", "url_paste", "URL", "filesystem_paths"] def __init__(self, params, sanitize=True): if sanitize: @@ -857,9 +888,12 @@ class Params: # name. Anything relying on NEVER_SANITIZE should be # changed to not require this and NEVER_SANITIZE should be # removed. - if (value is not None and key not in self.NEVER_SANITIZE - and True not in [key.endswith(f"|{nonsanitize_parameter}") for - nonsanitize_parameter in self.NEVER_SANITIZE]): + if ( + value is not None + and key not in self.NEVER_SANITIZE + and True + not in [key.endswith(f"|{nonsanitize_parameter}") for nonsanitize_parameter in self.NEVER_SANITIZE] + ): self.__dict__[key] = sanitize_param(value) else: self.__dict__[key] = value @@ -887,7 +921,7 @@ class Params: return self.__dict__.get(key, default) def __str__(self): - return f'{self.__dict__}' + return f"{self.__dict__}" def __len__(self): return len(self.__dict__) @@ -918,12 +952,12 @@ def rst_to_html(s, error=False): "template": os.path.join(os.path.dirname(__file__), "docutils_template.txt"), "warning_stream": FakeStream(), "doctitle_xform": False, # without option, very different rendering depending on - # number of sections in help content. + # number of sections in help content. } - return unicodify(docutils_core.publish_string( - s, writer=docutils_html4css1.Writer(), - settings_overrides=settings_overrides)) + return unicodify( + docutils_core.publish_string(s, writer=docutils_html4css1.Writer(), settings_overrides=settings_overrides) + ) def xml_text(root, name=None): @@ -938,10 +972,10 @@ def xml_text(root, name=None): else: elem = root if elem is not None and elem.text: - text = ''.join(elem.text.splitlines()) + text = "".join(elem.text.splitlines()) return text.strip() # No luck, return empty string - return '' + return "" def parse_resource_parameters(resource_param_file): @@ -961,8 +995,8 @@ def parse_resource_parameters(resource_param_file): # asbool implementation pulled from PasteDeploy -truthy = frozenset({'true', 'yes', 'on', 'y', 't', '1'}) -falsy = frozenset({'false', 'no', 'off', 'n', 'f', '0'}) +truthy = frozenset({"true", "yes", "on", "y", "t", "1"}) +falsy = frozenset({"false", "no", "off", "n", "f", "0"}) def asbool(obj): @@ -978,7 +1012,7 @@ def asbool(obj): def string_as_bool(string: str) -> bool: - if str(string).lower() in ('true', 'yes', 'on', '1'): + if str(string).lower() in ("true", "yes", "on", "1"): return True else: return False @@ -995,9 +1029,9 @@ def string_as_bool_or_none(string): function equivalently. """ string = str(string).lower() - if string in ('true', 'yes', 'on'): + if string in ("true", "yes", "on"): return True - elif string in ['none', 'null']: + elif string in ["none", "null"]: return None else: return False @@ -1026,18 +1060,18 @@ def listify(item, do_strip=False) -> typing.List[typing.Any]: return item elif isinstance(item, tuple): return list(item) - elif isinstance(item, str) and item.count(','): + elif isinstance(item, str) and item.count(","): if do_strip: - return [token.strip() for token in item.split(',')] + return [token.strip() for token in item.split(",")] else: - return item.split(',') + return item.split(",") else: return [item] def commaify(amount): orig = amount - new = re.sub(r"^(-?\d+)(\d{3})", r'\g<1>,\g<2>', amount) + new = re.sub(r"^(-?\d+)(\d{3})", r"\g<1>,\g<2>", amount) if orig == new: return new else: @@ -1051,10 +1085,10 @@ def roundify(amount, sfs=2): if len(amount) <= sfs: return amount else: - return amount[0:sfs] + '0' * (len(amount) - sfs) + return amount[0:sfs] + "0" * (len(amount) - sfs) -def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=False, log_exception=True): +def unicodify(value, encoding=DEFAULT_ENCODING, error="replace", strip_null=False, log_exception=True): """ Returns a Unicode string or None. @@ -1085,11 +1119,13 @@ def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=Fals log.exception(msg) raise if strip_null: - return value.replace('\0', '') + return value.replace("\0", "") return value -def filesystem_safe_string(s, max_len=255, truncation_chars='..', strip_leading_dot=True, invalid_chars=('/',), replacement_char='_'): +def filesystem_safe_string( + s, max_len=255, truncation_chars="..", strip_leading_dot=True, invalid_chars=("/",), replacement_char="_" +): """ Strip unicode null chars, truncate at 255 characters. Optionally replace additional ``invalid_chars`` with `replacement_char` . @@ -1099,16 +1135,16 @@ def filesystem_safe_string(s, max_len=255, truncation_chars='..', strip_leading_ """ sanitized_string = unicodify(s, strip_null=True) if strip_leading_dot: - sanitized_string = sanitized_string.lstrip('.') + sanitized_string = sanitized_string.lstrip(".") for invalid_char in invalid_chars: sanitized_string = sanitized_string.replace(invalid_char, replacement_char) if len(sanitized_string) > max_len: - sanitized_string = sanitized_string[:max_len - len(truncation_chars)] + sanitized_string = sanitized_string[: max_len - len(truncation_chars)] sanitized_string = f"{sanitized_string}{truncation_chars}" return sanitized_string -def smart_str(s, encoding=DEFAULT_ENCODING, strings_only=False, errors='strict'): +def smart_str(s, encoding=DEFAULT_ENCODING, strings_only=False, errors="strict"): """ Returns a bytestring version of 's', encoded as specified in 'encoding'. @@ -1153,7 +1189,7 @@ def string_to_object(s): return binascii.unhexlify(s) -def clean_multiline_string(multiline_string, sep='\n'): +def clean_multiline_string(multiline_string, sep="\n"): """ Dedent, split, remove first and last empty lines, rejoin. """ @@ -1163,12 +1199,11 @@ def clean_multiline_string(multiline_string, sep='\n'): string_list = string_list[1:] if not string_list[-1]: string_list = string_list[:-1] - return '\n'.join(string_list) + '\n' + return "\n".join(string_list) + "\n" class ParamsWithSpecs(collections.defaultdict): - """ - """ + """ """ def __init__(self, specs=None, params=None): self.specs = specs or dict() @@ -1176,19 +1211,19 @@ class ParamsWithSpecs(collections.defaultdict): for name, value in self.params.items(): if name not in self.specs: self._param_unknown_error(name) - if 'map' in self.specs[name]: + if "map" in self.specs[name]: try: - self.params[name] = self.specs[name]['map'](value) + self.params[name] = self.specs[name]["map"](value) except Exception: self._param_map_error(name, value) - if 'valid' in self.specs[name]: - if not self.specs[name]['valid'](value): + if "valid" in self.specs[name]: + if not self.specs[name]["valid"](value): self._param_vaildation_error(name, value) self.update(self.params) def __missing__(self, name): - return self.specs[name]['default'] + return self.specs[name]["default"] def __getattr__(self, name): return self[name] @@ -1216,7 +1251,7 @@ def compare_urls(url1, url2, compare_scheme=True, compare_hostname=True, compare def read_build_sites(filename, check_builds=True): - """ read db names to ucsc mappings from file, this file should probably be merged with the one above """ + """read db names to ucsc mappings from file, this file should probably be merged with the one above""" build_sites = [] try: for line in open(filename): @@ -1228,9 +1263,9 @@ def read_build_sites(filename, check_builds=True): site = fields[1] if check_builds: site_builds = fields[2].split(",") - site_dict = {'name': site_name, 'url': site, 'builds': site_builds} + site_dict = {"name": site_name, "url": site, "builds": site_builds} else: - site_dict = {'name': site_name, 'url': site} + site_dict = {"name": site_name, "url": site} build_sites.append(site_dict) except Exception: continue @@ -1266,7 +1301,7 @@ def stringify_dictionary_keys(in_dict): return out_dict -def mkstemp_ln(src, prefix='mkstemp_ln_'): +def mkstemp_ln(src, prefix="mkstemp_ln_"): """ From tempfile._mkstemp_inner, generate a hard link in the same dir with a random name. Created so we can persist the underlying file of a @@ -1279,7 +1314,7 @@ def mkstemp_ln(src, prefix='mkstemp_ln_'): file = os.path.join(dir, prefix + name) try: os.link(src, file) - return (os.path.abspath(file)) + return os.path.abspath(file) except OSError as e: if e.errno == errno.EEXIST: continue # try again @@ -1295,18 +1330,18 @@ def umask_fix_perms(path, umask, unmasked_perms, gid=None): try: st = os.stat(path) except OSError: - log.exception('Unable to set permissions or group on %s', path) + log.exception("Unable to set permissions or group on %s", path) return # fix modes if stat.S_IMODE(st.st_mode) != perms: try: os.chmod(path, perms) except Exception as e: - log.warning('Unable to honor umask ({}) for {}, tried to set: {} but mode remains {}, error was: {}'.format(oct(umask), - path, - oct(perms), - oct(stat.S_IMODE(st.st_mode)), - unicodify(e))) + log.warning( + "Unable to honor umask ({}) for {}, tried to set: {} but mode remains {}, error was: {}".format( + oct(umask), path, oct(perms), oct(stat.S_IMODE(st.st_mode)), unicodify(e) + ) + ) # fix group if gid is not None and st.st_gid != gid: try: @@ -1318,16 +1353,17 @@ def umask_fix_perms(path, umask, unmasked_perms, gid=None): except Exception: desired_group = gid current_group = st.st_gid - log.warning('Unable to honor primary group ({}) for {}, group remains {}, error was: {}'.format(desired_group, - path, - current_group, - unicodify(e))) + log.warning( + "Unable to honor primary group ({}) for {}, group remains {}, error was: {}".format( + desired_group, path, current_group, unicodify(e) + ) + ) def docstring_trim(docstring): """Trimming python doc strings. Taken from: http://www.python.org/dev/peps/pep-0257/""" if not docstring: - return '' + return "" # Convert tabs to spaces (following the normal Python rules) # and split into a list of lines: lines = docstring.expandtabs().splitlines() @@ -1348,7 +1384,7 @@ def docstring_trim(docstring): while trimmed and not trimmed[0]: trimmed.pop(0) # Return a single string: - return '\n'.join(trimmed) + return "\n".join(trimmed) def nice_size(size): @@ -1364,23 +1400,23 @@ def nice_size(size): >>> nice_size(100000000) '95.4 MB' """ - words = ['bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB'] - prefix = '' + words = ["bytes", "KB", "MB", "GB", "TB", "PB", "EB"] + prefix = "" try: size = float(size) if size < 0: size = abs(size) - prefix = '-' + prefix = "-" except Exception: - return '??? bytes' + return "??? bytes" for ind, word in enumerate(words): step = 1024 ** (ind + 1) if step > size: - size = size / float(1024 ** ind) - if word == 'bytes': # No decimals for bytes + size = size / float(1024**ind) + if word == "bytes": # No decimals for bytes return "%s%d bytes" % (prefix, size) return f"{prefix}{size:.1f} {word}" - return '??? bytes' + return "??? bytes" def size_to_bytes(size): @@ -1405,26 +1441,26 @@ def size_to_bytes(size): 122880 """ # The following number regexp is based on https://stackoverflow.com/questions/385558/extract-float-double-value/385597#385597 - size_re = re.compile(r'(?P(\d+(\.\d*)?|\.\d+)(e[+-]?\d+)?)\s*(?P[eptgmk]?(b|bytes?)?)?$') + size_re = re.compile(r"(?P(\d+(\.\d*)?|\.\d+)(e[+-]?\d+)?)\s*(?P[eptgmk]?(b|bytes?)?)?$") size_match = size_re.match(size.lower()) if size_match is None: raise ValueError(f"Could not parse string '{size}'") number = float(size_match.group("number")) multiple = size_match.group("multiple") - if multiple == "" or multiple.startswith('b'): + if multiple == "" or multiple.startswith("b"): return int(number) - elif multiple.startswith('k'): + elif multiple.startswith("k"): return int(number * 1024) - elif multiple.startswith('m'): - return int(number * 1024 ** 2) - elif multiple.startswith('g'): - return int(number * 1024 ** 3) - elif multiple.startswith('t'): - return int(number * 1024 ** 4) - elif multiple.startswith('p'): - return int(number * 1024 ** 5) - elif multiple.startswith('e'): - return int(number * 1024 ** 6) + elif multiple.startswith("m"): + return int(number * 1024**2) + elif multiple.startswith("g"): + return int(number * 1024**3) + elif multiple.startswith("t"): + return int(number * 1024**4) + elif multiple.startswith("p"): + return int(number * 1024**5) + elif multiple.startswith("e"): + return int(number * 1024**6) else: raise ValueError(f"Unknown multiplier '{multiple}' in '{size}'") @@ -1455,13 +1491,13 @@ def send_mail(frm, to, subject, body, config, html=None): to = listify(to) if html: - msg = MIMEMultipart('alternative') + msg = MIMEMultipart("alternative") else: - msg = MIMEText(body, 'plain', 'utf-8') + msg = MIMEText(body, "plain", "utf-8") - msg['To'] = ', '.join(to) - msg['From'] = frm - msg['Subject'] = subject + msg["To"] = ", ".join(to) + msg["From"] = frm + msg["Subject"] = subject if config.smtp_server is None: log.error("Mail is not configured for this Galaxy instance.") @@ -1469,12 +1505,12 @@ def send_mail(frm, to, subject, body, config, html=None): return if html: - mp_text = MIMEText(body, 'plain', 'utf-8') - mp_html = MIMEText(html, 'html', 'utf-8') + mp_text = MIMEText(body, "plain", "utf-8") + mp_html = MIMEText(html, "html", "utf-8") msg.attach(mp_text) msg.attach(mp_html) - smtp_ssl = asbool(getattr(config, 'smtp_ssl', False)) + smtp_ssl = asbool(getattr(config, "smtp_ssl", False)) if smtp_ssl: s = smtplib.SMTP_SSL(config.smtp_server) else: @@ -1482,15 +1518,15 @@ def send_mail(frm, to, subject, body, config, html=None): if not smtp_ssl: try: s.starttls() - log.debug('Initiated SSL/TLS connection to SMTP server: %s', config.smtp_server) + log.debug("Initiated SSL/TLS connection to SMTP server: %s", config.smtp_server) except RuntimeError as e: - log.warning('SSL/TLS support is not available to your Python interpreter: %s', unicodify(e)) + log.warning("SSL/TLS support is not available to your Python interpreter: %s", unicodify(e)) except smtplib.SMTPHeloError as e: log.error("The server didn't reply properly to the HELO greeting: %s", unicodify(e)) s.close() raise except smtplib.SMTPException as e: - log.warning('The server does not support the STARTTLS extension: %s', unicodify(e)) + log.warning("The server does not support the STARTTLS extension: %s", unicodify(e)) if config.smtp_username and config.smtp_password: try: s.login(config.smtp_username, config.smtp_password) @@ -1546,8 +1582,7 @@ def move_merge(source, target): def safe_str_cmp(a, b): - """safely compare two strings in a timing-attack-resistant manner - """ + """safely compare two strings in a timing-attack-resistant manner""" if len(a) != len(b): return False rv = 0 @@ -1558,7 +1593,7 @@ def safe_str_cmp(a, b): # Don't use these two directly, prefer method version that "works" with packaged Galaxy. galaxy_root_path = os.path.join(__path__[0], os.pardir, os.pardir, os.pardir) # type: ignore[name-defined] -galaxy_samples_path = os.path.join(__path__[0], os.pardir, 'config', 'sample') # type: ignore[name-defined] +galaxy_samples_path = os.path.join(__path__[0], os.pardir, "config", "sample") # type: ignore[name-defined] def galaxy_directory(): @@ -1569,7 +1604,7 @@ def galaxy_directory(): def galaxy_samples_directory(): - return os.path.join(galaxy_directory(), 'lib', 'galaxy', 'config', 'sample') + return os.path.join(galaxy_directory(), "lib", "galaxy", "config", "sample") def config_directories_from_setting(directories_setting, galaxy_root=galaxy_root_path): @@ -1592,10 +1627,10 @@ def config_directories_from_setting(directories_setting, galaxy_root=galaxy_root for directory in listify(directories_setting): directory = directory.strip() - if not directory.startswith('/'): + if not directory.startswith("/"): directory = os.path.join(galaxy_root, directory) if not os.path.exists(directory): - log.warning('directory not found: %s', directory) + log.warning("directory not found: %s", directory) continue directories.append(directory) return directories @@ -1640,32 +1675,32 @@ def parse_non_hex_float(s): """ f = float(s) # successfully parsed as float if here - check for format in original string - if 'e' in s and not ('+' in s or '-' in s): - raise ValueError('could not convert string to float: ' + s) + if "e" in s and not ("+" in s or "-" in s): + raise ValueError("could not convert string to float: " + s) return f -def build_url(base_url, port=80, scheme='http', pathspec=None, params=None, doseq=False): +def build_url(base_url, port=80, scheme="http", pathspec=None, params=None, doseq=False): if params is None: params = dict() if pathspec is None: pathspec = [] parsed_url = urlparse(base_url) - if scheme != 'http': + if scheme != "http": parsed_url.scheme = scheme - assert parsed_url.scheme in ('http', 'https', 'ftp'), f'Invalid URL scheme: {scheme}' + assert parsed_url.scheme in ("http", "https", "ftp"), f"Invalid URL scheme: {scheme}" if port != 80: - url = '%s://%s:%d/%s' % (parsed_url.scheme, parsed_url.netloc.rstrip('/'), int(port), parsed_url.path) + url = "%s://%s:%d/%s" % (parsed_url.scheme, parsed_url.netloc.rstrip("/"), int(port), parsed_url.path) else: url = f"{parsed_url.scheme}://{parsed_url.netloc.rstrip('/')}/{parsed_url.path.lstrip('/')}" if len(pathspec) > 0: url = f"{url.rstrip('/')}/{'/'.join(pathspec)}" if parsed_url.query: - for query_parameter in parsed_url.query.split('&'): - key, value = query_parameter.split('=') + for query_parameter in parsed_url.query.split("&"): + key, value = query_parameter.split("=") params[key] = value if params: - url += f'?{urlencode(params, doseq=doseq)}' + url += f"?{urlencode(params, doseq=doseq)}" return url @@ -1697,15 +1732,17 @@ def is_url(uri, allow_list=None): return any(uri.startswith(scheme) for scheme in allow_list) -def download_to_file(url, dest_file_path, timeout=30, chunk_size=2 ** 20): +def download_to_file(url, dest_file_path, timeout=30, chunk_size=2**20): """Download a URL to a file in chunks.""" - with requests.get(url, timeout=timeout, stream=True) as r, open(dest_file_path, 'wb') as f: + with requests.get(url, timeout=timeout, stream=True) as r, open(dest_file_path, "wb") as f: for chunk in r.iter_content(chunk_size): if chunk: f.write(chunk) -def stream_to_open_named_file(stream, fd, filename, source_encoding=None, source_error='strict', target_encoding=None, target_error='strict'): +def stream_to_open_named_file( + stream, fd, filename, source_encoding=None, source_error="strict", target_encoding=None, target_error="strict" +): """Writes a stream to the provided file descriptor, returns the file name. Closes file descriptor""" # signature and behavor is somewhat odd, due to backwards compatibility, but this can/should be done better CHUNK_SIZE = 1048576 @@ -1740,7 +1777,6 @@ def stream_to_open_named_file(stream, fd, filename, source_encoding=None, source class classproperty: - def __init__(self, f): self.f = f @@ -1749,7 +1785,6 @@ class classproperty: class ExecutionTimer: - def __init__(self): self.begin = time.time() @@ -1758,11 +1793,10 @@ class ExecutionTimer: @property def elapsed(self): - return (time.time() - self.begin) + return time.time() - self.begin class StructuredExecutionTimer: - def __init__(self, timer_id, template, **tags): self.begin = time.time() self.timer_id = timer_id @@ -1782,9 +1816,10 @@ class StructuredExecutionTimer: @property def elapsed(self): - return (time.time() - self.begin) + return time.time() - self.begin -if __name__ == '__main__': +if __name__ == "__main__": import doctest + doctest.testmod(sys.modules[__name__], verbose=False) diff --git a/lib/galaxy/util/path/__init__.py b/lib/galaxy/util/path/__init__.py index 5d8b4bffbaa..c53f028a472 100644 --- a/lib/galaxy/util/path/__init__.py +++ b/lib/galaxy/util/path/__init__.py @@ -6,17 +6,13 @@ import imp import logging import shlex from functools import partial -try: - from grp import getgrgid -except ImportError: - getgrgid = None # type: ignore[assignment] from itertools import starmap from operator import getitem from os import ( extsep, makedirs, stat, - walk + walk, ) from os.path import ( abspath, @@ -30,15 +26,20 @@ from os.path import ( pardir, realpath, relpath, - sep as separator, ) +from os.path import sep as separator from pathlib import Path + +try: + from grp import getgrgid +except ImportError: + getgrgid = None # type: ignore[assignment] + try: from pwd import getpwuid except ImportError: getpwuid = None # type: ignore[assignment] - import galaxy.util WALK_MAX_DIRS = 10000 @@ -86,7 +87,6 @@ def safe_contains(prefix, path, allowlist=None, real=None): class _SafeContainsDirectoryChecker: - def __init__(self, dirpath, prefix, allowlist=None): self.allowlist = allowlist self.dirpath = dirpath @@ -151,8 +151,9 @@ def safe_walk(path, allowlist=None): if allowlist and i % WALK_MAX_DIRS == 0: raise RuntimeError( - 'Breaking out of walk of %s after %s iterations (most likely infinite symlink recursion) at: %s' % - (path, WALK_MAX_DIRS, dirpath)) + "Breaking out of walk of %s after %s iterations (most likely infinite symlink recursion) at: %s" + % (path, WALK_MAX_DIRS, dirpath) + ) _prefix = partial(join, dirpath) prune = False @@ -216,9 +217,11 @@ def __path_permission_for_user(path, username): owner_permissions = int(oct_mode[-3]) group_permissions = int(oct_mode[-2]) other_permissions = int(oct_mode[-1]) - if other_permissions >= 4 or \ - (file_owner == username and owner_permissions >= 4) or \ - (username in group_members and group_permissions >= 4): + if ( + other_permissions >= 4 + or (file_owner == username and owner_permissions >= 4) + or (username in group_members and group_permissions >= 4) + ): return True return False @@ -323,6 +326,7 @@ class Extensions(dict): The first item in the sequence should match the key and is the "canonicalization". """ + def __missing__(self, key): for v in self.values(): if key in v: @@ -335,11 +339,13 @@ class Extensions(dict): return self[ext][0] -extensions = Extensions({ - 'ini': ['ini'], - 'json': ['json'], - 'yaml': ['yaml', 'yml'], -}) +extensions = Extensions( + { + "ini": ["ini"], + "json": ["json"], + "yaml": ["yaml", "yml"], + } +) def external_chown(path, pwent, external_chown_script, description="file"): @@ -352,7 +358,7 @@ def external_chown(path, pwent, external_chown_script, description="file"): """ try: if not external_chown_script: - raise ValueError('external_chown_script is not defined') + raise ValueError("external_chown_script is not defined") if Path(path).owner() == pwent[0]: return True @@ -362,13 +368,12 @@ def external_chown(path, pwent, external_chown_script, description="file"): galaxy.util.commands.execute(cmd) return True except galaxy.util.commands.CommandLineException as e: - log.warning(f'Changing ownership of {description} {path} failed: {galaxy.util.unicodify(e)}') + log.warning(f"Changing ownership of {description} {path} failed: {galaxy.util.unicodify(e)}") return False def __listify(item): - """A non-splitting version of :func:`galaxy.util.listify`. - """ + """A non-splitting version of :func:`galaxy.util.listify`.""" if not item: return [] elif isinstance(item, list) or isinstance(item, tuple): @@ -402,7 +407,7 @@ def __ext_strip_sep(ext): def __splitext_no_sep(path): path = galaxy.util.unicodify(path) - return (path.rsplit(extsep, 1) + [''])[0:2] + return (path.rsplit(extsep, 1) + [""])[0:2] def __splitext_ignore(path, ignore=None): @@ -410,7 +415,7 @@ def __splitext_ignore(path, ignore=None): ignore = map(__ext_strip_sep, __listify(ignore)) root, ext = __splitext_no_sep(path) if ext in ignore: - new_path = path[0:(-len(ext) - 1)] + new_path = path[0 : (-len(ext) - 1)] root, ext = __splitext_no_sep(new_path) return (root, ext) @@ -431,10 +436,9 @@ def _build_self(target, path_module): def __copy_self(names=__name__, parent=None): - """Returns a copy of this module that can be modified without modifying `galaxy.util.path`` in ``sys.modules``. - """ + """Returns a copy of this module that can be modified without modifying `galaxy.util.path`` in ``sys.modules``.""" if isinstance(names, str): - names = iter(names.split('.')) + names = iter(names.split(".")) try: name = next(names) except StopIteration: @@ -459,25 +463,25 @@ def __set_fxns_on(target, path_module): __pathfxns__ = ( - 'abspath', - 'basename', - 'exists', - 'isabs', - 'join', - 'normpath', - 'pardir', - 'realpath', - 'relpath', + "abspath", + "basename", + "exists", + "isabs", + "join", + "normpath", + "pardir", + "realpath", + "relpath", ) __all__ = ( - 'extensions', - 'get_ext', - 'has_ext', - 'joinext', - 'safe_contains', - 'safe_makedirs', - 'safe_relpath', - 'safe_walk', - 'unsafe_walk', + "extensions", + "get_ext", + "has_ext", + "joinext", + "safe_contains", + "safe_makedirs", + "safe_relpath", + "safe_walk", + "unsafe_walk", ) diff --git a/lib/galaxy/util/yaml_util.py b/lib/galaxy/util/yaml_util.py index 3427a70fd16..a9bbabdc30f 100644 --- a/lib/galaxy/util/yaml_util.py +++ b/lib/galaxy/util/yaml_util.py @@ -3,11 +3,12 @@ import os from collections import OrderedDict import yaml +from yaml.constructor import ConstructorError + try: from yaml import CSafeLoader as SafeLoader except ImportError: from yaml import SafeLoader # type: ignore[misc] -from yaml.constructor import ConstructorError log = logging.getLogger(__name__) @@ -36,6 +37,7 @@ def ordered_load(stream, merge_duplicate_keys=False): Otherwise, following YAML 1.2 specification which says that "each key is unique in the association", raise a ConstructionError exception. """ + def construct_mapping(loader, node, deep=False): loader.flatten_mapping(node) mapping = {} @@ -45,8 +47,12 @@ def ordered_load(stream, merge_duplicate_keys=False): value = loader.construct_object(value_node, deep=deep) if key in mapping: if not merge_duplicate_keys: - raise ConstructorError("while constructing a mapping", node.start_mark, - f"found duplicated key ({key})", key_node.start_mark) + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + f"found duplicated key ({key})", + key_node.start_mark, + ) log.debug("Merging values for duplicate key '%s' into a list", key) if merged_duplicate.get(key): mapping[key].append(value) @@ -57,10 +63,8 @@ def ordered_load(stream, merge_duplicate_keys=False): mapping[key] = value return mapping - OrderedLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - construct_mapping) - OrderedLoader.add_constructor('!include', OrderedLoader.include) + OrderedLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) + OrderedLoader.add_constructor("!include", OrderedLoader.include) return yaml.load(stream, OrderedLoader) @@ -70,8 +74,7 @@ def ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds): pass def _dict_representer(dumper, data): - return dumper.represent_mapping( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - list(data.items())) + return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, list(data.items())) + OrderedDumper.add_representer(OrderedDict, _dict_representer) return yaml.dump(data, stream, OrderedDumper, **kwds) diff --git a/lib/galaxy/version.py b/lib/galaxy/version.py index 2366316eecc..129af6925aa 100644 --- a/lib/galaxy/version.py +++ b/lib/galaxy/version.py @@ -1,3 +1,3 @@ -VERSION_MAJOR = "22.01" -VERSION_MINOR = "rc1" -VERSION = VERSION_MAJOR + (f".{VERSION_MINOR}" if VERSION_MINOR else '') +VERSION_MAJOR = "22.05" +VERSION_MINOR = "dev0" +VERSION = VERSION_MAJOR + (f".{VERSION_MINOR}" if VERSION_MINOR else "") diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index ce7a0a2efd1..648242a67bb 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -62,17 +62,11 @@ from . import ( log = logging.getLogger(__name__) -router = Router(tags=['datasets']) +router = Router(tags=["datasets"]) -DatasetIDPathParam: EncodedDatabaseIdField = Path( - ..., - description="The encoded database identifier of the dataset." -) +DatasetIDPathParam: EncodedDatabaseIdField = Path(..., description="The encoded database identifier of the dataset.") -HistoryIDPathParam: EncodedDatabaseIdField = Path( - ..., - description="The encoded database identifier of the History." -) +HistoryIDPathParam: EncodedDatabaseIdField = Path(..., description="The encoded database identifier of the History.") DatasetSourceQueryParam: DatasetSourceType = Query( default=DatasetSourceType.hda, @@ -85,15 +79,15 @@ class FastAPIDatasets: service: DatasetsService = depends(DatasetsService) @router.get( - '/api/datasets', - summary='Search datasets or collections using a query system.', + "/api/datasets", + summary="Search datasets or collections using a query system.", ) def index( self, trans=DependsOnTrans, history_id: Optional[EncodedDatabaseIdField] = Query( default=None, - description="Optional identifier of a History. Use it to restrict the search whithin a particular History." + description="Optional identifier of a History. Use it to restrict the search whithin a particular History.", ), serialization_params: SerializationParams = Depends(query_serialization_params), filter_query_params: FilterQueryParams = Depends(get_filter_query_params), @@ -101,8 +95,8 @@ class FastAPIDatasets: return self.service.index(trans, history_id, serialization_params, filter_query_params) @router.get( - '/api/datasets/{dataset_id}/storage', - summary='Display user-facing storage details related to the objectstore a dataset resides in.', + "/api/datasets/{dataset_id}/storage", + summary="Display user-facing storage details related to the objectstore a dataset resides in.", ) def show_storage( self, @@ -113,8 +107,8 @@ class FastAPIDatasets: return self.service.show_storage(trans, dataset_id, hda_ldda) @router.get( - '/api/datasets/{dataset_id}/inheritance_chain', - summary='For internal use, this endpoint may change without warning.', + "/api/datasets/{dataset_id}/inheritance_chain", + summary="For internal use, this endpoint may change without warning.", include_in_schema=True, # Can be changed to False if we don't really want to expose this ) def show_inheritance_chain( @@ -126,8 +120,8 @@ class FastAPIDatasets: return self.service.show_inheritance_chain(trans, dataset_id, hda_ldda) @router.get( - '/api/datasets/{dataset_id}/get_content_as_text', - summary='Returns dataset content as Text.', + "/api/datasets/{dataset_id}/get_content_as_text", + summary="Returns dataset content as Text.", ) def get_content_as_text( self, @@ -137,8 +131,8 @@ class FastAPIDatasets: return self.service.get_content_as_text(trans, dataset_id) @router.get( - '/api/datasets/{dataset_id}/converted/{ext}', - summary='Return information about datasets made by converting this dataset to a new format.', + "/api/datasets/{dataset_id}/converted/{ext}", + summary="Return information about datasets made by converting this dataset to a new format.", ) def converted_ext( self, @@ -160,10 +154,8 @@ class FastAPIDatasets: return self.service.converted_ext(trans, dataset_id, ext, serialization_params) @router.get( - '/api/datasets/{dataset_id}/converted', - summary=( - "Return a a map with all the existing converted datasets associated with this instance." - ), + "/api/datasets/{dataset_id}/converted", + summary=("Return a a map with all the existing converted datasets associated with this instance."), ) def converted( self, @@ -176,8 +168,8 @@ class FastAPIDatasets: return self.service.converted(trans, dataset_id) @router.put( - '/api/datasets/{dataset_id}/permissions', - summary='Set permissions of the given history dataset to the given role ids.', + "/api/datasets/{dataset_id}/permissions", + summary="Set permissions of the given history dataset to the given role ids.", ) def update_permissions( self, @@ -194,8 +186,8 @@ class FastAPIDatasets: return self.service.update_permissions(trans, dataset_id, update_payload) @router.get( - '/api/histories/{history_id}/contents/{history_content_id}/extra_files', - summary='Generate list of extra files.', + "/api/histories/{history_id}/contents/{history_content_id}/extra_files", + summary="Generate list of extra files.", tags=["histories"], ) def extra_files( @@ -207,9 +199,9 @@ class FastAPIDatasets: return self.service.extra_files(trans, history_content_id) @router.get( - '/api/histories/{history_id}/contents/{history_content_id}/display', + "/api/histories/{history_id}/contents/{history_content_id}/display", name="history_contents_display", - summary='Displays dataset (preview) content.', + summary="Displays dataset (preview) content.", tags=["histories"], response_class=StreamingResponse, ) @@ -235,7 +227,7 @@ class FastAPIDatasets: description=( "The file extension when downloading the display data. Use the value `data` to " "let the server infer it from the data type." - ) + ), ), raw: bool = Query( default=False, @@ -248,12 +240,14 @@ class FastAPIDatasets: ): """Streams the preview contents of a dataset to be displayed in a browser.""" extra_params = get_query_parameters_from_request_excluding(request, {"preview", "filename", "to_ext", "raw"}) - display_data, headers = self.service.display(trans, history_content_id, history_id, preview, filename, to_ext, raw, **extra_params) + display_data, headers = self.service.display( + trans, history_content_id, history_id, preview, filename, to_ext, raw, **extra_params + ) return StreamingResponse(display_data, headers=headers) @router.get( - '/api/histories/{history_id}/contents/{history_content_id}/metadata_file', - summary='Returns the metadata file associated with this history item.', + "/api/histories/{history_id}/contents/{history_content_id}/metadata_file", + summary="Returns the metadata file associated with this history item.", tags=["histories"], response_class=FileResponse, ) @@ -271,7 +265,7 @@ class FastAPIDatasets: return FileResponse(path=cast(str, metadata_file_path), headers=headers) @router.get( - '/api/datasets/{dataset_id}', + "/api/datasets/{dataset_id}", summary="Displays information about and/or content of a dataset.", ) def show( @@ -281,9 +275,7 @@ class FastAPIDatasets: dataset_id: EncodedDatabaseIdField = DatasetIDPathParam, hda_ldda: DatasetSourceType = Query( default=DatasetSourceType.hda, - description=( - "The type of information about the dataset to be requested." - ), + description=("The type of information about the dataset to be requested."), ), data_type: Optional[RequestDataType] = Query( default=None, @@ -368,25 +360,25 @@ class DatasetsController(BaseGalaxyAPIController): filter_parameters = FilterQueryParams(**kwd) filter_parameters.limit = filter_parameters.limit or limit filter_parameters.offset = filter_parameters.offset or offset - return self.service.index( - trans, history_id, serialization_params, filter_parameters - ) + return self.service.index(trans, history_id, serialization_params, filter_parameters) @web.expose_api_anonymous_and_sessionless - def show(self, trans, id, hda_ldda='hda', data_type=None, provider=None, **kwd): + def show(self, trans, id, hda_ldda="hda", data_type=None, provider=None, **kwd): """ GET /api/datasets/{encoded_dataset_id} Displays information about and/or content of a dataset. """ serialization_params = parse_serialization_params(**kwd) - kwd.update({ - "provider": provider, - }) + kwd.update( + { + "provider": provider, + } + ) rval = self.service.show(trans, id, hda_ldda, serialization_params, data_type, **kwd) return rval @web.expose_api_anonymous - def show_storage(self, trans, dataset_id, hda_ldda='hda', **kwd): + def show_storage(self, trans, dataset_id, hda_ldda="hda", **kwd): """ GET /api/datasets/{encoded_dataset_id}/storage @@ -396,7 +388,7 @@ class DatasetsController(BaseGalaxyAPIController): return self.service.show_storage(trans, dataset_id, hda_ldda) @web.expose_api_anonymous - def show_inheritance_chain(self, trans, dataset_id, hda_ldda='hda', **kwd): + def show_inheritance_chain(self, trans, dataset_id, hda_ldda="hda", **kwd): """ GET /api/datasets/{dataset_id}/inheritance_chain @@ -415,7 +407,7 @@ class DatasetsController(BaseGalaxyAPIController): :rtype: dict :returns: dictionary containing new permissions """ - hda_ldda = kwd.pop('hda_ldda', DatasetSourceType.hda) + hda_ldda = kwd.pop("hda_ldda", DatasetSourceType.hda) if payload: kwd.update(payload) update_payload = get_update_permission_payload(kwd) @@ -430,8 +422,9 @@ class DatasetsController(BaseGalaxyAPIController): return self.service.extra_files(trans, history_content_id) @web.expose_api_raw_anonymous_and_sessionless - def display(self, trans, history_content_id, history_id, - preview=False, filename=None, to_ext=None, raw=False, **kwd): + def display( + self, trans, history_content_id, history_id, preview=False, filename=None, to_ext=None, raw=False, **kwd + ): """ GET /api/histories/{encoded_history_id}/contents/{encoded_content_id}/display Displays history content (dataset). @@ -449,7 +442,7 @@ class DatasetsController(BaseGalaxyAPIController): @web.expose_api def get_content_as_text(self, trans, dataset_id): - """ Returns item content as Text. """ + """Returns item content as Text.""" return self.service.get_content_as_text(trans, dataset_id) @web.expose_api_raw_anonymous_and_sessionless diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index d17bee888a0..d1a206d5474 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -23,9 +23,7 @@ from pydantic.fields import Field from pydantic.main import BaseModel from starlette.responses import FileResponse -from galaxy import ( - util -) +from galaxy import util from galaxy.managers.context import ( ProvidesHistoryContext, ProvidesUserContext, @@ -52,9 +50,7 @@ from galaxy.schema.schema import ( SharingStatus, ) from galaxy.schema.types import LatestLiteral -from galaxy.util import ( - string_as_bool -) +from galaxy.util import string_as_bool from galaxy.web import ( expose_api, expose_api_anonymous, @@ -72,27 +68,25 @@ from . import ( depends, DependsOnTrans, Router, - try_get_request_body_as_json + try_get_request_body_as_json, ) log = logging.getLogger(__name__) -router = Router(tags=['histories']) +router = Router(tags=["histories"]) HistoryIDPathParam: EncodedDatabaseIdField = Path( - ..., - title="History ID", - description="The encoded database identifier of the History." + ..., title="History ID", description="The encoded database identifier of the History." ) JehaIDPathParam: Union[EncodedDatabaseIdField, LatestLiteral] = Path( default="latest", - title='Job Export History ID', + title="Job Export History ID", description=( - 'The ID of the specific Job Export History Association or ' - '`latest` (default) to download the last generated archive.' + "The ID of the specific Job Export History Association or " + "`latest` (default) to download the last generated archive." ), - example="latest" + example="latest", ) @@ -106,9 +100,7 @@ class HistoryIndexParams(HistoryFilterQueryParams): class DeleteHistoryPayload(BaseModel): purge: bool = Field( - default=False, - title="Purge", - description="Whether to definitely remove this history from disk." + default=False, title="Purge", description="Whether to definitely remove this history from disk." ) @@ -122,8 +114,8 @@ class FastAPIHistories: service: HistoriesService = depends(HistoriesService) @router.get( - '/api/histories', - summary='Returns histories for the current user.', + "/api/histories", + summary="Returns histories for the current user.", ) def index( self, @@ -135,13 +127,13 @@ class FastAPIHistories: title="Deleted Only", description="Whether to return only deleted items.", deprecated=True, # Marked as deprecated as it seems just like '/api/histories/deleted' - ) + ), ) -> List[AnyHistoryView]: return self.service.index(trans, serialization_params, params, deleted_only=deleted, all_histories=params.all) @router.get( - '/api/histories/deleted', - summary='Returns deleted histories for the current user.', + "/api/histories/deleted", + summary="Returns deleted histories for the current user.", ) def index_deleted( self, @@ -152,8 +144,8 @@ class FastAPIHistories: return self.service.index(trans, serialization_params, params, deleted_only=True, all_histories=params.all) @router.get( - '/api/histories/published', - summary='Return all histories that are published.', + "/api/histories/published", + summary="Return all histories that are published.", ) def published( self, @@ -164,8 +156,8 @@ class FastAPIHistories: return self.service.published(trans, serialization_params, filter_params) @router.get( - '/api/histories/shared_with_me', - summary='Return all histories that are shared with the current user.', + "/api/histories/shared_with_me", + summary="Return all histories that are shared with the current user.", ) def shared_with_me( self, @@ -176,8 +168,8 @@ class FastAPIHistories: return self.service.shared_with_me(trans, serialization_params, filter_params) @router.get( - '/api/histories/most_recently_used', - summary='Returns the most recently used history of the user.', + "/api/histories/most_recently_used", + summary="Returns the most recently used history of the user.", ) def show_recent( self, @@ -187,8 +179,8 @@ class FastAPIHistories: return self.service.show(trans, serialization_params) @router.get( - '/api/histories/{id}', - summary='Returns the history with the given ID.', + "/api/histories/{id}", + summary="Returns the history with the given ID.", ) def show( self, @@ -199,8 +191,8 @@ class FastAPIHistories: return self.service.show(trans, serialization_params, id) @router.get( - '/api/histories/{id}/citations', - summary='Return all the citations for the tools used to produce the datasets in the history.', + "/api/histories/{id}/citations", + summary="Return all the citations for the tools used to produce the datasets in the history.", ) def citations( self, @@ -210,8 +202,8 @@ class FastAPIHistories: return self.service.citations(trans, id) @router.post( - '/api/histories', - summary='Creates a new history.', + "/api/histories", + summary="Creates a new history.", ) def create( self, @@ -232,8 +224,8 @@ class FastAPIHistories: return self.service.create(trans, payload, serialization_params) @router.delete( - '/api/histories/{id}', - summary='Marks the history with the given ID as deleted.', + "/api/histories/{id}", + summary="Marks the history with the given ID as deleted.", ) def delete( self, @@ -241,14 +233,14 @@ class FastAPIHistories: id: EncodedDatabaseIdField = HistoryIDPathParam, serialization_params: SerializationParams = Depends(query_serialization_params), purge: bool = Query(default=False), - payload: Optional[DeleteHistoryPayload] = Body(default=None) + payload: Optional[DeleteHistoryPayload] = Body(default=None), ) -> AnyHistoryView: if payload: purge = payload.purge return self.service.delete(trans, id, serialization_params, purge) @router.post( - '/api/histories/deleted/{id}/undelete', + "/api/histories/deleted/{id}/undelete", summary="Restores a deleted history with the given ID (that hasn't been purged).", ) def undelete( @@ -260,7 +252,7 @@ class FastAPIHistories: return self.service.undelete(trans, id, serialization_params) @router.put( - '/api/histories/{id}', + "/api/histories/{id}", summary="Updates the values for the history with the given ID.", ) def update( @@ -276,10 +268,8 @@ class FastAPIHistories: return self.service.update(trans, id, payload, serialization_params) @router.get( - '/api/histories/{id}/exports', - summary=( - "Get previous history exports (to links). Effectively returns serialized JEHA objects." - ), + "/api/histories/{id}/exports", + summary=("Get previous history exports (to links). Effectively returns serialized JEHA objects."), ) def index_exports( self, @@ -290,17 +280,15 @@ class FastAPIHistories: return JobExportHistoryArchiveCollection.parse_obj(exports) @router.put( # PUT instead of POST because multiple requests should just result in one object being created. - '/api/histories/{id}/exports', - summary=( - "Start job (if needed) to create history export for corresponding history." - ), + "/api/histories/{id}/exports", + summary=("Start job (if needed) to create history export for corresponding history."), responses={ 200: { "description": "Object containing url to fetch export from.", }, 202: { "description": "The exported archive file is not ready yet.", - } + }, }, ) def archive_export( @@ -326,11 +314,9 @@ class FastAPIHistories: return export_result @router.get( - '/api/histories/{id}/exports/{jeha_id}', + "/api/histories/{id}/exports/{jeha_id}", name="history_archive_download", - summary=( - "If ready and available, return raw contents of exported history as a downloadable archive." - ), + summary=("If ready and available, return raw contents of exported history as a downloadable archive."), response_class=FileResponse, responses={ 200: { @@ -359,7 +345,7 @@ class FastAPIHistories: ) @router.get( - '/api/histories/{id}/custom_builds_metadata', + "/api/histories/{id}/custom_builds_metadata", summary="Returns meta data for custom builds.", ) def get_custom_builds_metadata( @@ -370,7 +356,7 @@ class FastAPIHistories: return self.service.get_custom_builds_metadata(trans, id) @router.get( - '/api/histories/{id}/sharing', + "/api/histories/{id}/sharing", summary="Get the current sharing status of the given item.", ) def sharing( @@ -382,7 +368,7 @@ class FastAPIHistories: return self.service.shareable_service.sharing(trans, id) @router.put( - '/api/histories/{id}/enable_link_access', + "/api/histories/{id}/enable_link_access", summary="Makes this item accessible by a URL link.", ) def enable_link_access( @@ -394,7 +380,7 @@ class FastAPIHistories: return self.service.shareable_service.enable_link_access(trans, id) @router.put( - '/api/histories/{id}/disable_link_access', + "/api/histories/{id}/disable_link_access", summary="Makes this item inaccessible by a URL link.", ) def disable_link_access( @@ -406,7 +392,7 @@ class FastAPIHistories: return self.service.shareable_service.disable_link_access(trans, id) @router.put( - '/api/histories/{id}/publish', + "/api/histories/{id}/publish", summary="Makes this item public and accessible by a URL link.", ) def publish( @@ -418,7 +404,7 @@ class FastAPIHistories: return self.service.shareable_service.publish(trans, id) @router.put( - '/api/histories/{id}/unpublish', + "/api/histories/{id}/unpublish", summary="Removes this item from the published list.", ) def unpublish( @@ -430,20 +416,20 @@ class FastAPIHistories: return self.service.shareable_service.unpublish(trans, id) @router.put( - '/api/histories/{id}/share_with_users', + "/api/histories/{id}/share_with_users", summary="Share this item with specific users.", ) def share_with_users( self, trans: ProvidesUserContext = DependsOnTrans, id: EncodedDatabaseIdField = HistoryIDPathParam, - payload: ShareWithPayload = Body(...) + payload: ShareWithPayload = Body(...), ) -> ShareWithStatus: """Shares this item with specific users and return the current sharing status.""" return self.service.shareable_service.share_with_users(trans, id, payload) @router.put( - '/api/histories/{id}/slug', + "/api/histories/{id}/slug", summary="Set a new slug for this shared item.", status_code=status.HTTP_204_NO_CONTENT, ) @@ -462,7 +448,7 @@ class HistoriesController(BaseGalaxyAPIController): service: HistoriesService = depends(HistoriesService) @expose_api_anonymous - def index(self, trans, deleted='False', **kwd): + def index(self, trans, deleted="False", **kwd): """ GET /api/histories @@ -554,13 +540,13 @@ class HistoriesController(BaseGalaxyAPIController): 'order' defaults to 'create_time-dsc' """ deleted_only = util.string_as_bool(deleted) - all_histories = util.string_as_bool(kwd.get('all', False)) + all_histories = util.string_as_bool(kwd.get("all", False)) serialization_params = parse_serialization_params(**kwd) filter_parameters = HistoryFilterQueryParams(**kwd) return self.service.index(trans, serialization_params, filter_parameters, deleted_only, all_histories) @expose_api_anonymous - def show(self, trans, id, deleted='False', **kwd): + def show(self, trans, id, deleted="False", **kwd): """ show( trans, id, deleted='False' ) * GET /api/histories/{id}: @@ -678,10 +664,10 @@ class HistoriesController(BaseGalaxyAPIController): """ history_id = id # a request body is optional here - purge = string_as_bool(kwd.get('purge', False)) + purge = string_as_bool(kwd.get("purge", False)) # for backwards compat, keep the payload sub-dictionary - if kwd.get('payload', None): - purge = string_as_bool(kwd['payload'].get('purge', purge)) + if kwd.get("payload", None): + purge = string_as_bool(kwd["payload"].get("purge", purge)) serialization_params = parse_serialization_params(**kwd) return self.service.delete(trans, history_id, serialization_params, purge) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 7a1f3989a00..505ee700be0 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -49,13 +49,10 @@ def app_pair(global_conf, load_app_kwds=None, wsgi_preflight=True, **kwargs): middleware to handle CORS options, etc.. """ load_app_kwds = load_app_kwds or {} - kwargs = load_app_properties( - kwds=kwargs, - **load_app_kwds - ) + kwargs = load_app_properties(kwds=kwargs, **load_app_kwds) # Create the Galaxy application unless passed in - if 'app' in kwargs: - app = kwargs.pop('app') + if "app" in kwargs: + app = kwargs.pop("app") galaxy.app.app = app else: try: @@ -65,52 +62,110 @@ def app_pair(global_conf, load_app_kwds=None, wsgi_preflight=True, **kwargs): traceback.print_exc() sys.exit(1) - if kwargs.get('register_shutdown_at_exit', True): + if kwargs.get("register_shutdown_at_exit", True): # Call app's shutdown method when the interpeter exits, this cleanly stops # the various Galaxy application daemon threads app.application_stack.register_postfork_function(atexit.register, app.shutdown) # Create the universe WSGI application - webapp = GalaxyWebApplication(app, session_cookie='galaxysession', name='galaxy') + webapp = GalaxyWebApplication(app, session_cookie="galaxysession", name="galaxy") # STANDARD CONTROLLER ROUTES - webapp.add_ui_controllers('galaxy.webapps.galaxy.controllers', app) + webapp.add_ui_controllers("galaxy.webapps.galaxy.controllers", app) # Force /history to go to view of current - webapp.add_route('/history', controller='history', action='view') - webapp.add_route('/history/view/{id}', controller='history', action='view') + webapp.add_route("/history", controller="history", action="view") + webapp.add_route("/history/view/{id}", controller="history", action="view") # Force /activate to go to the controller - webapp.add_route('/activate', controller='user', action='activate') + webapp.add_route("/activate", controller="user", action="activate") # Authentication endpoints. - webapp.add_route('/authnz/', controller='authnz', action='index', provider=None) - webapp.add_route('/authnz/{provider}/login', controller='authnz', action='login', provider=None) - webapp.add_route('/authnz/{provider}/callback', controller='authnz', action='callback', provider=None) - webapp.add_route('/authnz/{provider}/disconnect/{email}', controller='authnz', action='disconnect', provider=None, email=None) - webapp.add_route('/authnz/{provider}/logout', controller='authnz', action='logout', provider=None) - webapp.add_route('/authnz/{provider}/create_user', controller='authnz', action='create_user') + webapp.add_route("/authnz/", controller="authnz", action="index", provider=None) + webapp.add_route("/authnz/{provider}/login", controller="authnz", action="login", provider=None) + webapp.add_route("/authnz/{provider}/callback", controller="authnz", action="callback", provider=None) + webapp.add_route( + "/authnz/{provider}/disconnect/{email}", controller="authnz", action="disconnect", provider=None, email=None + ) + webapp.add_route("/authnz/{provider}/logout", controller="authnz", action="logout", provider=None) + webapp.add_route("/authnz/{provider}/create_user", controller="authnz", action="create_user") # Returns the provider specific logout url for currently logged in provider - webapp.add_route('/authnz/logout', controller='authnz', action='get_logout_url') - webapp.add_route('/authnz/get_cilogon_idps', controller='authnz', action='get_cilogon_idps') + webapp.add_route("/authnz/logout", controller="authnz", action="get_logout_url") + webapp.add_route("/authnz/get_cilogon_idps", controller="authnz", action="get_cilogon_idps") # These two routes handle our simple needs at the moment - webapp.add_route('/async/{tool_id}/{data_id}/{data_secret}', controller='async', action='index', tool_id=None, data_id=None, data_secret=None) - webapp.add_route('/{controller}/{action}', action='index') - webapp.add_route('/{action}', controller='root', action='index') + webapp.add_route( + "/async/{tool_id}/{data_id}/{data_secret}", + controller="async", + action="index", + tool_id=None, + data_id=None, + data_secret=None, + ) + webapp.add_route("/{controller}/{action}", action="index") + webapp.add_route("/{action}", controller="root", action="index") # allow for subdirectories in extra_files_path - webapp.add_route('/datasets/{dataset_id}/display/{filename:.+?}', controller='dataset', action='display', dataset_id=None, filename=None) - webapp.add_route('/datasets/{dataset_id}/{action}/{filename}', controller='dataset', action='index', dataset_id=None, filename=None) - webapp.add_route('/display_application/{dataset_id}/{app_name}/{link_name}/{user_id}/{app_action}/{action_param}/{action_param_extra:.+?}', - controller='dataset', action='display_application', dataset_id=None, user_id=None, - app_name=None, link_name=None, app_action=None, action_param=None, action_param_extra=None) + webapp.add_route( + "/datasets/{dataset_id}/display/{filename:.+?}", + controller="dataset", + action="display", + dataset_id=None, + filename=None, + ) + webapp.add_route( + "/datasets/{dataset_id}/{action}/{filename}", + controller="dataset", + action="index", + dataset_id=None, + filename=None, + ) + webapp.add_route( + "/display_application/{dataset_id}/{app_name}/{link_name}/{user_id}/{app_action}/{action_param}/{action_param_extra:.+?}", + controller="dataset", + action="display_application", + dataset_id=None, + user_id=None, + app_name=None, + link_name=None, + app_action=None, + action_param=None, + action_param_extra=None, + ) - USERNAME_REQS = {'username': VALID_PUBLICNAME_RE.pattern.strip("^$")} - webapp.add_route('/u/{username}/d/{slug}/{filename}', controller='dataset', action='display_by_username_and_slug', filename=None, requirements=USERNAME_REQS) - webapp.add_route('/u/{username}/p/{slug}', controller='page', action='display_by_username_and_slug', requirements=USERNAME_REQS) - webapp.add_route('/u/{username}/h/{slug}', controller='history', action='display_by_username_and_slug', requirements=USERNAME_REQS) - webapp.add_route('/u/{username}/w/{slug}', controller='workflow', action='display_by_username_and_slug', requirements=USERNAME_REQS) - webapp.add_route('/u/{username}/w/{slug}/{format}', controller='workflow', action='display_by_username_and_slug', requirements=USERNAME_REQS) - webapp.add_route('/u/{username}/v/{slug}', controller='visualization', action='display_by_username_and_slug', requirements=USERNAME_REQS) + USERNAME_REQS = {"username": VALID_PUBLICNAME_RE.pattern.strip("^$")} + webapp.add_route( + "/u/{username}/d/{slug}/{filename}", + controller="dataset", + action="display_by_username_and_slug", + filename=None, + requirements=USERNAME_REQS, + ) + webapp.add_route( + "/u/{username}/p/{slug}", controller="page", action="display_by_username_and_slug", requirements=USERNAME_REQS + ) + webapp.add_route( + "/u/{username}/h/{slug}", + controller="history", + action="display_by_username_and_slug", + requirements=USERNAME_REQS, + ) + webapp.add_route( + "/u/{username}/w/{slug}", + controller="workflow", + action="display_by_username_and_slug", + requirements=USERNAME_REQS, + ) + webapp.add_route( + "/u/{username}/w/{slug}/{format}", + controller="workflow", + action="display_by_username_and_slug", + requirements=USERNAME_REQS, + ) + webapp.add_route( + "/u/{username}/v/{slug}", + controller="visualization", + action="display_by_username_and_slug", + requirements=USERNAME_REQS, + ) # TODO: Refactor above routes into external method to allow testing in # isolation as well. @@ -118,11 +173,11 @@ def app_pair(global_conf, load_app_kwds=None, wsgi_preflight=True, **kwargs): if wsgi_preflight: # API OPTIONS RESPONSE webapp.mapper.connect( - 'options', - '/api/{path_info:.*?}', - controller='authenticate', - action='options', - conditions={'method': ['OPTIONS']}, + "options", + "/api/{path_info:.*?}", + controller="authenticate", + action="options", + conditions={"method": ["OPTIONS"]}, ) # CLIENTSIDE ROUTES @@ -130,88 +185,92 @@ def app_pair(global_conf, load_app_kwds=None, wsgi_preflight=True, **kwargs): # The following routes don't bootstrap any information, simply provide the # base analysis interface at which point the application takes over. - webapp.add_client_route('/admin/data_tables', 'admin') - webapp.add_client_route('/admin/data_types', 'admin') - webapp.add_client_route('/admin/jobs', 'admin') - webapp.add_client_route('/admin/invocations', 'admin') - webapp.add_client_route('/admin/toolbox_dependencies', 'admin') - webapp.add_client_route('/admin/data_manager{path_info:.*}', 'admin') - webapp.add_client_route('/admin/error_stack', 'admin') - webapp.add_client_route('/admin/users', 'admin') - webapp.add_client_route('/admin/users/create', 'admin') - webapp.add_client_route('/admin/display_applications', 'admin') - webapp.add_client_route('/admin/reset_metadata', 'admin') - webapp.add_client_route('/admin/roles', 'admin') - webapp.add_client_route('/admin/forms', 'admin') - webapp.add_client_route('/admin/groups', 'admin') - webapp.add_client_route('/admin/repositories', 'admin') - webapp.add_client_route('/admin/sanitize_allow', 'admin') - webapp.add_client_route('/admin/tool_versions', 'admin') - webapp.add_client_route('/admin/toolshed', 'admin') - webapp.add_client_route('/admin/quotas', 'admin') - webapp.add_client_route('/admin/form/{form_id}', 'admin') - webapp.add_client_route('/admin/api_keys', 'admin') - webapp.add_client_route('/login/confirm') - webapp.add_client_route('/tools/view') - webapp.add_client_route('/tools/json') - webapp.add_client_route('/tours') - webapp.add_client_route('/tours/{tour_id}') - webapp.add_client_route('/user') - webapp.add_client_route('/user/{form_id}') - webapp.add_client_route('/welcome/new') - webapp.add_client_route('/visualizations') - webapp.add_client_route('/visualizations/edit') - webapp.add_client_route('/visualizations/sharing') - webapp.add_client_route('/visualizations/list_published') - webapp.add_client_route('/visualizations/list') - webapp.add_client_route('/pages/list') - webapp.add_client_route('/pages/list_published') - webapp.add_client_route('/pages/create') - webapp.add_client_route('/pages/edit') - webapp.add_client_route('/pages/sharing') - webapp.add_client_route('/histories/citations') - webapp.add_client_route('/histories/list') - webapp.add_client_route('/histories/import') - webapp.add_client_route('/histories/{history_id}/export') - webapp.add_client_route('/histories/list_published') - webapp.add_client_route('/histories/list_shared') - webapp.add_client_route('/histories/rename') - webapp.add_client_route('/histories/sharing') - webapp.add_client_route('/histories/permissions') - webapp.add_client_route('/histories/view') - webapp.add_client_route('/histories/show_structure') - webapp.add_client_route('/datasets/list') - webapp.add_client_route('/datasets/edit') - webapp.add_client_route('/collection/edit/{collection_id}') - webapp.add_client_route('/datasets/error') - webapp.add_client_route('/jobs/{job_id}/view') - webapp.add_client_route('/datasets/{dataset_id}/details') - webapp.add_client_route('/datasets/{dataset_id}/show_params') - webapp.add_client_route('/workflows/list') - webapp.add_client_route('/workflows/list_published') - webapp.add_client_route('/workflows/create') - webapp.add_client_route('/workflows/run') - webapp.add_client_route('/workflows/import') - webapp.add_client_route('/workflows/trs_import') - webapp.add_client_route('/workflows/trs_search') - webapp.add_client_route('/workflows/invocations') - webapp.add_client_route('/workflows/invocations/report') + webapp.add_client_route("/admin/data_tables", "admin") + webapp.add_client_route("/admin/data_types", "admin") + webapp.add_client_route("/admin/jobs", "admin") + webapp.add_client_route("/admin/invocations", "admin") + webapp.add_client_route("/admin/toolbox_dependencies", "admin") + webapp.add_client_route("/admin/data_manager{path_info:.*}", "admin") + webapp.add_client_route("/admin/error_stack", "admin") + webapp.add_client_route("/admin/users", "admin") + webapp.add_client_route("/admin/users/create", "admin") + webapp.add_client_route("/admin/display_applications", "admin") + webapp.add_client_route("/admin/reset_metadata", "admin") + webapp.add_client_route("/admin/roles", "admin") + webapp.add_client_route("/admin/forms", "admin") + webapp.add_client_route("/admin/groups", "admin") + webapp.add_client_route("/admin/repositories", "admin") + webapp.add_client_route("/admin/sanitize_allow", "admin") + webapp.add_client_route("/admin/tool_versions", "admin") + webapp.add_client_route("/admin/toolshed", "admin") + webapp.add_client_route("/admin/quotas", "admin") + webapp.add_client_route("/admin/form/{form_id}", "admin") + webapp.add_client_route("/admin/api_keys", "admin") + webapp.add_client_route("/login/confirm") + webapp.add_client_route("/tools/view") + webapp.add_client_route("/tools/json") + webapp.add_client_route("/tours") + webapp.add_client_route("/tours/{tour_id}") + webapp.add_client_route("/user") + webapp.add_client_route("/user/{form_id}") + webapp.add_client_route("/welcome/new") + webapp.add_client_route("/visualizations") + webapp.add_client_route("/visualizations/edit") + webapp.add_client_route("/visualizations/sharing") + webapp.add_client_route("/visualizations/list_published") + webapp.add_client_route("/visualizations/list") + webapp.add_client_route("/pages/list") + webapp.add_client_route("/pages/list_published") + webapp.add_client_route("/pages/create") + webapp.add_client_route("/pages/edit") + webapp.add_client_route("/pages/sharing") + webapp.add_client_route("/histories/citations") + webapp.add_client_route("/histories/list") + webapp.add_client_route("/histories/import") + webapp.add_client_route("/histories/{history_id}/export") + webapp.add_client_route("/histories/list_published") + webapp.add_client_route("/histories/list_shared") + webapp.add_client_route("/histories/rename") + webapp.add_client_route("/histories/sharing") + webapp.add_client_route("/histories/permissions") + webapp.add_client_route("/histories/view") + webapp.add_client_route("/histories/show_structure") + webapp.add_client_route("/datasets/list") + webapp.add_client_route("/datasets/edit") + webapp.add_client_route("/collection/edit/{collection_id}") + webapp.add_client_route("/datasets/error") + webapp.add_client_route("/jobs/{job_id}/view") + webapp.add_client_route("/datasets/{dataset_id}/details") + webapp.add_client_route("/datasets/{dataset_id}/show_params") + webapp.add_client_route("/workflows/list") + webapp.add_client_route("/workflows/list_published") + webapp.add_client_route("/workflows/create") + webapp.add_client_route("/workflows/run") + webapp.add_client_route("/workflows/import") + webapp.add_client_route("/workflows/trs_import") + webapp.add_client_route("/workflows/trs_search") + webapp.add_client_route("/workflows/invocations") + webapp.add_client_route("/workflows/invocations/report") # webapp.add_client_route('/workflows/invocations/view_bco') - webapp.add_client_route('/custom_builds') - webapp.add_client_route('/interactivetool_entry_points/list') - webapp.add_client_route('/libraries{path:.*?}') + webapp.add_client_route("/custom_builds") + webapp.add_client_route("/interactivetool_entry_points/list") + webapp.add_client_route("/libraries{path:.*?}") # ==== Done # Indicate that all configuration settings have been provided webapp.finalize_config() # Wrap the webapp in some useful middleware - if kwargs.get('middleware', True): + if kwargs.get("middleware", True): webapp = wrap_in_middleware(webapp, global_conf, app.application_stack, **kwargs) - if asbool(kwargs.get('static_enabled', True)): - webapp = wrap_if_allowed(webapp, app.application_stack, wrap_in_static, - args=(global_conf,), - kwargs=dict(plugin_frameworks=[app.visualizations_registry], **kwargs)) + if asbool(kwargs.get("static_enabled", True)): + webapp = wrap_if_allowed( + webapp, + app.application_stack, + wrap_in_static, + args=(global_conf,), + kwargs=dict(plugin_frameworks=[app.visualizations_registry], **kwargs), + ) app.application_stack.register_postfork_function(postfork_setup) for th in threading.enumerate(): @@ -231,470 +290,821 @@ uwsgi_app_factory = uwsgi_app def postfork_setup(): from galaxy.app import app + app.application_stack.log_startup() def populate_api_routes(webapp, app): - webapp.add_api_controllers('galaxy.webapps.galaxy.api', app) + webapp.add_api_controllers("galaxy.webapps.galaxy.api", app) valid_history_contents_types = [ - 'dataset', - 'dataset_collection', + "dataset", + "dataset_collection", ] # Accesss HDA details via histories/{history_id}/contents/datasets/{hda_id} # and HDCA details via histories/{history_id}/contents/dataset_collections/{hdca_id} - webapp.mapper.resource("content_typed", - "{type:%s}s" % "|".join(valid_history_contents_types), - name_prefix="history_", - controller='history_contents', - path_prefix='/api/histories/{history_id}/contents', - parent_resources=dict(member_name='history', collection_name='histories')) + webapp.mapper.resource( + "content_typed", + "{type:%s}s" % "|".join(valid_history_contents_types), + name_prefix="history_", + controller="history_contents", + path_prefix="/api/histories/{history_id}/contents", + parent_resources=dict(member_name="history", collection_name="histories"), + ) # Legacy access to HDA details via histories/{history_id}/contents/{hda_id} - webapp.mapper.resource('content', - 'contents', - controller='history_contents', - name_prefix='history_', - path_prefix='/api/histories/{history_id}', - parent_resources=dict(member_name='history', collection_name='histories')) - webapp.mapper.connect("history_contents_batch_update", - "/api/histories/{history_id}/contents", - controller="history_contents", - action="update_batch", - conditions=dict(method=["PUT"])) - webapp.mapper.connect("history_contents_display", - "/api/histories/{history_id}/contents/{history_content_id}/display", - controller="datasets", - action="display", - conditions=dict(method=["GET"])) - webapp.mapper.connect("history_contents_update_permissions", - "/api/histories/{history_id}/contents/{history_content_id}/permissions", - controller="history_contents", - action="update_permissions", - conditions=dict(method=["PUT"])) - webapp.mapper.connect("history_contents_validate", - "/api/histories/{history_id}/contents/{history_content_id}/validate", - controller="history_contents", - action="validate", - conditions=dict(method=["PUT"])) - webapp.mapper.connect("history_contents_extra_files", - "/api/histories/{history_id}/contents/{history_content_id}/extra_files", - controller="datasets", - action="extra_files", - conditions=dict(method=["GET"])) - webapp.mapper.connect("history_content_as_text", - "/api/datasets/{dataset_id}/get_content_as_text", - controller="datasets", - action="get_content_as_text", - conditions=dict(method=["GET"])) - webapp.mapper.connect('dataset_storage', - '/api/datasets/{dataset_id}/storage', - controller='datasets', - action='show_storage', - conditions=dict(method=["GET"])) - webapp.mapper.connect('dataset_inheritance_chain', - '/api/datasets/{dataset_id}/inheritance_chain', - controller='datasets', - action='show_inheritance_chain', - conditions=dict(method=["GET"])) - webapp.mapper.connect("history_contents_metadata_file", - "/api/histories/{history_id}/contents/{history_content_id}/metadata_file", - controller="datasets", - action="get_metadata_file", - conditions=dict(method=["GET"])) - webapp.mapper.connect("/api/histories/{history_id}/contents/{direction:near|before|after}/{hid}/{limit}", - action="contents_near", - controller='history_contents', - conditions=dict(method=["GET"])) - webapp.mapper.resource('user', - 'users', - controller='group_users', - name_prefix='group_', - path_prefix='/api/groups/{group_id}', - parent_resources=dict(member_name='group', collection_name='groups')) - webapp.mapper.resource('role', - 'roles', - controller='group_roles', - name_prefix='group_', - path_prefix='/api/groups/{group_id}', - parent_resources=dict(member_name='group', collection_name='groups')) - _add_item_tags_controller(webapp, - name_prefix="history_content_", - path_prefix='/api/histories/{history_id}/contents/{history_content_id}') - webapp.mapper.connect('/api/histories/published', action='published', controller="histories", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/histories/shared_with_me', action='shared_with_me', controller="histories") + webapp.mapper.resource( + "content", + "contents", + controller="history_contents", + name_prefix="history_", + path_prefix="/api/histories/{history_id}", + parent_resources=dict(member_name="history", collection_name="histories"), + ) + webapp.mapper.connect( + "history_contents_batch_update", + "/api/histories/{history_id}/contents", + controller="history_contents", + action="update_batch", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "history_contents_display", + "/api/histories/{history_id}/contents/{history_content_id}/display", + controller="datasets", + action="display", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "history_contents_update_permissions", + "/api/histories/{history_id}/contents/{history_content_id}/permissions", + controller="history_contents", + action="update_permissions", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "history_contents_validate", + "/api/histories/{history_id}/contents/{history_content_id}/validate", + controller="history_contents", + action="validate", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "history_contents_extra_files", + "/api/histories/{history_id}/contents/{history_content_id}/extra_files", + controller="datasets", + action="extra_files", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "history_content_as_text", + "/api/datasets/{dataset_id}/get_content_as_text", + controller="datasets", + action="get_content_as_text", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "dataset_storage", + "/api/datasets/{dataset_id}/storage", + controller="datasets", + action="show_storage", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "dataset_inheritance_chain", + "/api/datasets/{dataset_id}/inheritance_chain", + controller="datasets", + action="show_inheritance_chain", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "history_contents_metadata_file", + "/api/histories/{history_id}/contents/{history_content_id}/metadata_file", + controller="datasets", + action="get_metadata_file", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "/api/histories/{history_id}/contents/{direction:near|before|after}/{hid}/{limit}", + action="contents_near", + controller="history_contents", + conditions=dict(method=["GET"]), + ) + webapp.mapper.resource( + "user", + "users", + controller="group_users", + name_prefix="group_", + path_prefix="/api/groups/{group_id}", + parent_resources=dict(member_name="group", collection_name="groups"), + ) + webapp.mapper.resource( + "role", + "roles", + controller="group_roles", + name_prefix="group_", + path_prefix="/api/groups/{group_id}", + parent_resources=dict(member_name="group", collection_name="groups"), + ) + _add_item_tags_controller( + webapp, name_prefix="history_content_", path_prefix="/api/histories/{history_id}/contents/{history_content_id}" + ) + webapp.mapper.connect( + "/api/histories/published", action="published", controller="histories", conditions=dict(method=["GET"]) + ) + webapp.mapper.connect("/api/histories/shared_with_me", action="shared_with_me", controller="histories") - webapp.mapper.connect('cloud_storage', - '/api/cloud/storage/', - controller='cloud', - action='index', - conditions=dict(method=["GET"])) - webapp.mapper.connect('cloud_storage_get', - '/api/cloud/storage/get', - controller='cloud', - action='get', - conditions=dict(method=["POST"])) - webapp.mapper.connect('cloud_storage_send', - '/api/cloud/storage/send', - controller='cloud', - action='send', - conditions=dict(method=["POST"])) + webapp.mapper.connect( + "cloud_storage", "/api/cloud/storage/", controller="cloud", action="index", conditions=dict(method=["GET"]) + ) + webapp.mapper.connect( + "cloud_storage_get", + "/api/cloud/storage/get", + controller="cloud", + action="get", + conditions=dict(method=["POST"]), + ) + webapp.mapper.connect( + "cloud_storage_send", + "/api/cloud/storage/send", + controller="cloud", + action="send", + conditions=dict(method=["POST"]), + ) - _add_item_tags_controller(webapp, - name_prefix="history_", - path_prefix='/api/histories/{history_id}') - _add_item_tags_controller(webapp, - name_prefix="workflow_", - path_prefix='/api/workflows/{workflow_id}') - _add_item_annotation_controller(webapp, - name_prefix="history_content_", - path_prefix='/api/histories/{history_id}/contents/{history_content_id}') - _add_item_annotation_controller(webapp, - name_prefix="history_", - path_prefix='/api/histories/{history_id}') - _add_item_annotation_controller(webapp, - name_prefix="workflow_", - path_prefix='/api/workflows/{workflow_id}') - _add_item_provenance_controller(webapp, - name_prefix="history_content_", - path_prefix='/api/histories/{history_id}/contents/{history_content_id}') + _add_item_tags_controller(webapp, name_prefix="history_", path_prefix="/api/histories/{history_id}") + _add_item_tags_controller(webapp, name_prefix="workflow_", path_prefix="/api/workflows/{workflow_id}") + _add_item_annotation_controller( + webapp, name_prefix="history_content_", path_prefix="/api/histories/{history_id}/contents/{history_content_id}" + ) + _add_item_annotation_controller(webapp, name_prefix="history_", path_prefix="/api/histories/{history_id}") + _add_item_annotation_controller(webapp, name_prefix="workflow_", path_prefix="/api/workflows/{workflow_id}") + _add_item_provenance_controller( + webapp, name_prefix="history_content_", path_prefix="/api/histories/{history_id}/contents/{history_content_id}" + ) - webapp.mapper.resource('dataset', 'datasets', path_prefix='/api') - webapp.mapper.resource('tool_data', 'tool_data', path_prefix='/api') - webapp.mapper.connect('/api/tool_data/{id:.+?}/fields/{value:.+?}/files/{path:.+?}', action='download_field_file', controller="tool_data") - webapp.mapper.connect('/api/tool_data/{id:.+?}/fields/{value:.+?}', action='show_field', controller="tool_data") - webapp.mapper.connect('/api/tool_data/{id:.+?}/reload', action='reload', controller="tool_data") + webapp.mapper.resource("dataset", "datasets", path_prefix="/api") + webapp.mapper.resource("tool_data", "tool_data", path_prefix="/api") + webapp.mapper.connect( + "/api/tool_data/{id:.+?}/fields/{value:.+?}/files/{path:.+?}", + action="download_field_file", + controller="tool_data", + ) + webapp.mapper.connect("/api/tool_data/{id:.+?}/fields/{value:.+?}", action="show_field", controller="tool_data") + webapp.mapper.connect("/api/tool_data/{id:.+?}/reload", action="reload", controller="tool_data") - webapp.mapper.resource('dataset_collection', 'dataset_collections', path_prefix='/api') - webapp.mapper.connect('contents_dataset_collection', - '/api/dataset_collections/{hdca_id}/contents/{parent_id}', + webapp.mapper.resource("dataset_collection", "dataset_collections", path_prefix="/api") + webapp.mapper.connect( + "contents_dataset_collection", + "/api/dataset_collections/{hdca_id}/contents/{parent_id}", controller="dataset_collections", - action="contents") + action="contents", + ) - webapp.mapper.resource('form', 'forms', path_prefix='/api') - webapp.mapper.resource('role', 'roles', path_prefix='/api') - webapp.mapper.resource('upload', 'uploads', path_prefix='/api') - webapp.mapper.connect('/api/upload/resumable_upload/{session_id}', controller="uploads", action="hooks", conditions=dict(method=['PATCH'])) - webapp.mapper.connect('/api/upload/resumable_upload', controller="uploads", action="hooks") - webapp.mapper.connect('/api/upload/hooks', controller="uploads", action="hooks", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/ftp_files', controller='remote_files') - webapp.mapper.connect('/api/remote_files', action='index', controller='remote_files', conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/remote_files/plugins', action='plugins', controller='remote_files', conditions=dict(method=["GET"])) - webapp.mapper.resource('group', 'groups', path_prefix='/api') - webapp.mapper.resource_with_deleted('quota', 'quotas', path_prefix='/api') + webapp.mapper.resource("form", "forms", path_prefix="/api") + webapp.mapper.resource("role", "roles", path_prefix="/api") + webapp.mapper.resource("upload", "uploads", path_prefix="/api") + webapp.mapper.connect( + "/api/upload/resumable_upload/{session_id}", + controller="uploads", + action="hooks", + conditions=dict(method=["PATCH"]), + ) + webapp.mapper.connect("/api/upload/resumable_upload", controller="uploads", action="hooks") + webapp.mapper.connect("/api/upload/hooks", controller="uploads", action="hooks", conditions=dict(method=["POST"])) + webapp.mapper.connect("/api/ftp_files", controller="remote_files") + webapp.mapper.connect( + "/api/remote_files", action="index", controller="remote_files", conditions=dict(method=["GET"]) + ) + webapp.mapper.connect( + "/api/remote_files/plugins", action="plugins", controller="remote_files", conditions=dict(method=["GET"]) + ) + webapp.mapper.resource("group", "groups", path_prefix="/api") + webapp.mapper.resource_with_deleted("quota", "quotas", path_prefix="/api") - webapp.mapper.connect('/api/cloud/authz/', action='index', controller='cloudauthz', conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/cloud/authz/', - action='create', - controller='cloudauthz', - conditions=dict(method=["POST"])) + webapp.mapper.connect("/api/cloud/authz/", action="index", controller="cloudauthz", conditions=dict(method=["GET"])) + webapp.mapper.connect( + "/api/cloud/authz/", action="create", controller="cloudauthz", conditions=dict(method=["POST"]) + ) - webapp.mapper.connect('delete_cloudauthz_item', - '/api/cloud/authz/{encoded_authz_id}', - action='delete', - controller='cloudauthz', - conditions=dict(method=["DELETE"])) + webapp.mapper.connect( + "delete_cloudauthz_item", + "/api/cloud/authz/{encoded_authz_id}", + action="delete", + controller="cloudauthz", + conditions=dict(method=["DELETE"]), + ) - webapp.mapper.connect('upload_cloudauthz_item', - '/api/cloud/authz/{encoded_authz_id}', - action='update', - controller="cloudauthz", - conditions=dict(method=["PUT"])) + webapp.mapper.connect( + "upload_cloudauthz_item", + "/api/cloud/authz/{encoded_authz_id}", + action="update", + controller="cloudauthz", + conditions=dict(method=["PUT"]), + ) - webapp.mapper.connect('get_custom_builds_metadata', - '/api/histories/{id}/custom_builds_metadata', - controller='histories', - action='get_custom_builds_metadata', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "get_custom_builds_metadata", + "/api/histories/{id}/custom_builds_metadata", + controller="histories", + action="get_custom_builds_metadata", + conditions=dict(method=["GET"]), + ) # ======================= # ====== TOOLS API ====== # ======================= - webapp.mapper.connect('/api/tools/fetch', action='fetch', controller='tools', conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/tools/all_requirements', action='all_requirements', controller="tools") - webapp.mapper.connect('/api/tools/error_stack', action='error_stack', controller="tools") - webapp.mapper.connect('/api/tools/{id:.+?}/build', action='build', controller="tools") - webapp.mapper.connect('/api/tools/{id:.+?}/reload', action='reload', controller="tools") - webapp.mapper.connect('/api/tools/tests_summary', action='tests_summary', controller="tools") - webapp.mapper.connect('/api/tools/{id:.+?}/test_data_path', action='test_data_path', controller="tools") - webapp.mapper.connect('/api/tools/{id:.+?}/test_data_download', action='test_data_download', controller="tools") - webapp.mapper.connect('/api/tools/{id:.+?}/test_data', action='test_data', controller="tools") - webapp.mapper.connect('/api/tools/{id:.+?}/diagnostics', action='diagnostics', controller="tools") - webapp.mapper.connect('/api/tools/{id:.+?}/citations', action='citations', controller="tools") - webapp.mapper.connect('/api/tools/{tool_id:.+?}/convert', action='conversion', controller="tools", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/tools/{id:.+?}/xrefs', action='xrefs', controller="tools") - webapp.mapper.connect('/api/tools/{id:.+?}/download', action='download', controller="tools") - webapp.mapper.connect('/api/tools/{id:.+?}/raw_tool_source', action='raw_tool_source', controller="tools") - webapp.mapper.connect('/api/tools/{id:.+?}/requirements', action='requirements', controller="tools") - webapp.mapper.connect('/api/tools/{id:.+?}/install_dependencies', action='install_dependencies', controller="tools", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/tools/{id:.+?}/dependencies', action='install_dependencies', controller="tools", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/tools/{id:.+?}/dependencies', action='uninstall_dependencies', controller="tools", conditions=dict(method=["DELETE"])) - webapp.mapper.connect('/api/tools/{id:.+?}/build_dependency_cache', action='build_dependency_cache', controller="tools", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/tools/{id:.+?}', action='show', controller="tools") - webapp.mapper.resource('tool', 'tools', path_prefix='/api') - webapp.mapper.resource('dynamic_tools', 'dynamic_tools', path_prefix='/api') - - webapp.mapper.connect('/api/sanitize_allow', action='index', controller='sanitize_allow', conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/sanitize_allow', action='create', controller='sanitize_allow', conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/sanitize_allow', action='delete', controller='sanitize_allow', conditions=dict(method=["DELETE"])) - - webapp.mapper.connect('/api/entry_points', action='index', controller="tool_entry_points") - webapp.mapper.connect('/api/entry_points/{id:.+?}/access', action='access_entry_point', controller="tool_entry_points") - webapp.mapper.connect('/api/entry_points/{id:.+?}', action='stop_entry_point', controller="tool_entry_points", conditions={'method': ['DELETE']}) - - webapp.mapper.connect('/api/dependency_resolvers/clean', action="clean", controller="tool_dependencies", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/dependency_resolvers/dependency', action="manager_dependency", controller="tool_dependencies", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/dependency_resolvers/dependency', action="install_dependency", controller="tool_dependencies", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/dependency_resolvers/requirements', action="manager_requirements", controller="tool_dependencies") - webapp.mapper.connect('/api/dependency_resolvers/unused_paths', action="unused_dependency_paths", controller="tool_dependencies", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/dependency_resolvers/unused_paths', action="delete_unused_dependency_paths", controller="tool_dependencies", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/dependency_resolvers/toolbox', controller="tool_dependencies", action="summarize_toolbox", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/dependency_resolvers/toolbox/install', controller="tool_dependencies", action="toolbox_install", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/dependency_resolvers/toolbox/uninstall', controller="tool_dependencies", action="toolbox_uninstall", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/dependency_resolvers/{index}/clean', action="clean", controller="tool_dependencies", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/dependency_resolvers/{index}/dependency', action="resolver_dependency", controller="tool_dependencies", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/dependency_resolvers/{index}/dependency', action="install_dependency", controller="tool_dependencies", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/dependency_resolvers/{index}/requirements', action="resolver_requirements", controller="tool_dependencies") - webapp.mapper.resource('dependency_resolver', 'dependency_resolvers', controller="tool_dependencies", path_prefix='api') - webapp.mapper.connect('/api/container_resolvers', action="index", controller="container_resolution", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/container_resolvers/resolve', action="resolve", controller="container_resolution", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/container_resolvers/toolbox', action="resolve_toolbox", controller="container_resolution", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/container_resolvers/resolve/install', action="resolve_with_install", controller="container_resolution", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/container_resolvers/toolbox/install', action="resolve_toolbox_with_install", controller="container_resolution", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/container_resolvers/{index}', action="show", controller="container_resolution", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/container_resolvers/{index}/resolve', action="resolve", controller="container_resolution", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/container_resolvers/{index}/toolbox', action="resolve_toolbox", controller="container_resolution", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/container_resolvers/{index}/resolve/install', action="resolve_with_install", controller="container_resolution", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/container_resolvers/{index}/toolbox/install', action="resolve_toolbox_with_install", controller="container_resolution", conditions=dict(method=["POST"])) - webapp.mapper.connect('/api/workflows/get_tool_predictions', action='get_tool_predictions', controller="workflows", conditions=dict(method=["POST"])) - - webapp.mapper.resource_with_deleted('user', 'users', path_prefix='/api') - webapp.mapper.resource('genome', 'genomes', path_prefix='/api') - webapp.mapper.connect('/api/genomes/{id}/indexes', controller='genomes', action='indexes') - webapp.mapper.connect('/api/genomes/{id}/sequences', controller='genomes', action='sequences') - webapp.mapper.resource('visualization', 'visualizations', path_prefix='/api') - webapp.mapper.connect('/api/visualizations/{id}/sharing', action='sharing', controller="visualizations", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/visualizations/{id}/enable_link_access', action='enable_link_access', controller="visualizations", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/visualizations/{id}/disable_link_access', action='disable_link_access', controller="visualizations", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/visualizations/{id}/publish', action='publish', controller="visualizations", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/visualizations/{id}/unpublish', action='unpublish', controller="visualizations", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/visualizations/{id}/share_with_users', action='share_with_users', controller="visualizations", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/visualizations/{id}/slug', action='set_slug', controller="visualizations", conditions=dict(method=["PUT"])) - webapp.mapper.resource('plugins', 'plugins', path_prefix='/api') - webapp.mapper.connect('/api/workflows/build_module', action='build_module', controller="workflows") - webapp.mapper.connect('/api/workflows/menu', action='get_workflow_menu', controller="workflows", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/workflows/menu', action='set_workflow_menu', controller="workflows", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/workflows/{id}/refactor', action='refactor', controller="workflows", conditions=dict(method=["PUT"])) - webapp.mapper.resource('workflow', 'workflows', path_prefix='/api') - webapp.mapper.connect('/api/licenses', controller='licenses', action='index', conditions=dict(method="GET")) - webapp.mapper.connect('/api/licenses/{id}', controller='licenses', action='get', conditions=dict(method="GET")) - webapp.mapper.resource_with_deleted('history', 'histories', path_prefix='/api') - webapp.mapper.connect('/api/histories/{history_id}/citations', action='citations', controller="histories") - webapp.mapper.connect('/api/histories/{id}/sharing', action='sharing', controller="histories", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/histories/{id}/enable_link_access', action='enable_link_access', controller="histories", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/histories/{id}/disable_link_access', action='disable_link_access', controller="histories", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/histories/{id}/publish', action='publish', controller="histories", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/histories/{id}/unpublish', action='unpublish', controller="histories", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/histories/{id}/share_with_users', action='share_with_users', controller="histories", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/histories/{id}/slug', action='set_slug', controller="histories", conditions=dict(method=["PUT"])) + webapp.mapper.connect("/api/tools/fetch", action="fetch", controller="tools", conditions=dict(method=["POST"])) + webapp.mapper.connect("/api/tools/all_requirements", action="all_requirements", controller="tools") + webapp.mapper.connect("/api/tools/error_stack", action="error_stack", controller="tools") + webapp.mapper.connect("/api/tools/{id:.+?}/build", action="build", controller="tools") + webapp.mapper.connect("/api/tools/{id:.+?}/reload", action="reload", controller="tools") + webapp.mapper.connect("/api/tools/tests_summary", action="tests_summary", controller="tools") + webapp.mapper.connect("/api/tools/{id:.+?}/test_data_path", action="test_data_path", controller="tools") + webapp.mapper.connect("/api/tools/{id:.+?}/test_data_download", action="test_data_download", controller="tools") + webapp.mapper.connect("/api/tools/{id:.+?}/test_data", action="test_data", controller="tools") + webapp.mapper.connect("/api/tools/{id:.+?}/diagnostics", action="diagnostics", controller="tools") + webapp.mapper.connect("/api/tools/{id:.+?}/citations", action="citations", controller="tools") webapp.mapper.connect( - 'dynamic_tool_confs', - '/api/configuration/dynamic_tool_confs', - controller="configuration", - action="dynamic_tool_confs" + "/api/tools/{tool_id:.+?}/convert", action="conversion", controller="tools", conditions=dict(method=["POST"]) + ) + webapp.mapper.connect("/api/tools/{id:.+?}/xrefs", action="xrefs", controller="tools") + webapp.mapper.connect("/api/tools/{id:.+?}/download", action="download", controller="tools") + webapp.mapper.connect("/api/tools/{id:.+?}/raw_tool_source", action="raw_tool_source", controller="tools") + webapp.mapper.connect("/api/tools/{id:.+?}/requirements", action="requirements", controller="tools") + webapp.mapper.connect( + "/api/tools/{id:.+?}/install_dependencies", + action="install_dependencies", + controller="tools", + conditions=dict(method=["POST"]), ) webapp.mapper.connect( - 'tool_lineages', - '/api/configuration/tool_lineages', - controller="configuration", - action="tool_lineages" + "/api/tools/{id:.+?}/dependencies", + action="install_dependencies", + controller="tools", + conditions=dict(method=["POST"]), ) webapp.mapper.connect( - '/api/configuration/toolbox', + "/api/tools/{id:.+?}/dependencies", + action="uninstall_dependencies", + controller="tools", + conditions=dict(method=["DELETE"]), + ) + webapp.mapper.connect( + "/api/tools/{id:.+?}/build_dependency_cache", + action="build_dependency_cache", + controller="tools", + conditions=dict(method=["POST"]), + ) + webapp.mapper.connect("/api/tools/{id:.+?}", action="show", controller="tools") + webapp.mapper.resource("tool", "tools", path_prefix="/api") + webapp.mapper.resource("dynamic_tools", "dynamic_tools", path_prefix="/api") + + webapp.mapper.connect( + "/api/sanitize_allow", action="index", controller="sanitize_allow", conditions=dict(method=["GET"]) + ) + webapp.mapper.connect( + "/api/sanitize_allow", action="create", controller="sanitize_allow", conditions=dict(method=["PUT"]) + ) + webapp.mapper.connect( + "/api/sanitize_allow", action="delete", controller="sanitize_allow", conditions=dict(method=["DELETE"]) + ) + + webapp.mapper.connect("/api/entry_points", action="index", controller="tool_entry_points") + webapp.mapper.connect( + "/api/entry_points/{id:.+?}/access", action="access_entry_point", controller="tool_entry_points" + ) + webapp.mapper.connect( + "/api/entry_points/{id:.+?}", + action="stop_entry_point", + controller="tool_entry_points", + conditions={"method": ["DELETE"]}, + ) + + webapp.mapper.connect( + "/api/dependency_resolvers/clean", + action="clean", + controller="tool_dependencies", + conditions=dict(method=["POST"]), + ) + webapp.mapper.connect( + "/api/dependency_resolvers/dependency", + action="manager_dependency", + controller="tool_dependencies", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "/api/dependency_resolvers/dependency", + action="install_dependency", + controller="tool_dependencies", + conditions=dict(method=["POST"]), + ) + webapp.mapper.connect( + "/api/dependency_resolvers/requirements", action="manager_requirements", controller="tool_dependencies" + ) + webapp.mapper.connect( + "/api/dependency_resolvers/unused_paths", + action="unused_dependency_paths", + controller="tool_dependencies", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "/api/dependency_resolvers/unused_paths", + action="delete_unused_dependency_paths", + controller="tool_dependencies", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "/api/dependency_resolvers/toolbox", + controller="tool_dependencies", + action="summarize_toolbox", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "/api/dependency_resolvers/toolbox/install", + controller="tool_dependencies", + action="toolbox_install", + conditions=dict(method=["POST"]), + ) + webapp.mapper.connect( + "/api/dependency_resolvers/toolbox/uninstall", + controller="tool_dependencies", + action="toolbox_uninstall", + conditions=dict(method=["POST"]), + ) + webapp.mapper.connect( + "/api/dependency_resolvers/{index}/clean", + action="clean", + controller="tool_dependencies", + conditions=dict(method=["POST"]), + ) + webapp.mapper.connect( + "/api/dependency_resolvers/{index}/dependency", + action="resolver_dependency", + controller="tool_dependencies", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "/api/dependency_resolvers/{index}/dependency", + action="install_dependency", + controller="tool_dependencies", + conditions=dict(method=["POST"]), + ) + webapp.mapper.connect( + "/api/dependency_resolvers/{index}/requirements", action="resolver_requirements", controller="tool_dependencies" + ) + webapp.mapper.resource( + "dependency_resolver", "dependency_resolvers", controller="tool_dependencies", path_prefix="api" + ) + webapp.mapper.connect( + "/api/container_resolvers", action="index", controller="container_resolution", conditions=dict(method=["GET"]) + ) + webapp.mapper.connect( + "/api/container_resolvers/resolve", + action="resolve", + controller="container_resolution", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "/api/container_resolvers/toolbox", + action="resolve_toolbox", + controller="container_resolution", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "/api/container_resolvers/resolve/install", + action="resolve_with_install", + controller="container_resolution", + conditions=dict(method=["POST"]), + ) + webapp.mapper.connect( + "/api/container_resolvers/toolbox/install", + action="resolve_toolbox_with_install", + controller="container_resolution", + conditions=dict(method=["POST"]), + ) + webapp.mapper.connect( + "/api/container_resolvers/{index}", + action="show", + controller="container_resolution", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "/api/container_resolvers/{index}/resolve", + action="resolve", + controller="container_resolution", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "/api/container_resolvers/{index}/toolbox", + action="resolve_toolbox", + controller="container_resolution", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "/api/container_resolvers/{index}/resolve/install", + action="resolve_with_install", + controller="container_resolution", + conditions=dict(method=["POST"]), + ) + webapp.mapper.connect( + "/api/container_resolvers/{index}/toolbox/install", + action="resolve_toolbox_with_install", + controller="container_resolution", + conditions=dict(method=["POST"]), + ) + webapp.mapper.connect( + "/api/workflows/get_tool_predictions", + action="get_tool_predictions", + controller="workflows", + conditions=dict(method=["POST"]), + ) + + webapp.mapper.resource_with_deleted("user", "users", path_prefix="/api") + webapp.mapper.resource("genome", "genomes", path_prefix="/api") + webapp.mapper.connect("/api/genomes/{id}/indexes", controller="genomes", action="indexes") + webapp.mapper.connect("/api/genomes/{id}/sequences", controller="genomes", action="sequences") + webapp.mapper.resource("visualization", "visualizations", path_prefix="/api") + webapp.mapper.connect( + "/api/visualizations/{id}/sharing", + action="sharing", + controller="visualizations", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "/api/visualizations/{id}/enable_link_access", + action="enable_link_access", + controller="visualizations", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "/api/visualizations/{id}/disable_link_access", + action="disable_link_access", + controller="visualizations", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "/api/visualizations/{id}/publish", + action="publish", + controller="visualizations", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "/api/visualizations/{id}/unpublish", + action="unpublish", + controller="visualizations", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "/api/visualizations/{id}/share_with_users", + action="share_with_users", + controller="visualizations", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "/api/visualizations/{id}/slug", action="set_slug", controller="visualizations", conditions=dict(method=["PUT"]) + ) + webapp.mapper.resource("plugins", "plugins", path_prefix="/api") + webapp.mapper.connect("/api/workflows/build_module", action="build_module", controller="workflows") + webapp.mapper.connect( + "/api/workflows/menu", action="get_workflow_menu", controller="workflows", conditions=dict(method=["GET"]) + ) + webapp.mapper.connect( + "/api/workflows/menu", action="set_workflow_menu", controller="workflows", conditions=dict(method=["PUT"]) + ) + webapp.mapper.connect( + "/api/workflows/{id}/refactor", action="refactor", controller="workflows", conditions=dict(method=["PUT"]) + ) + webapp.mapper.resource("workflow", "workflows", path_prefix="/api") + webapp.mapper.connect("/api/licenses", controller="licenses", action="index", conditions=dict(method="GET")) + webapp.mapper.connect("/api/licenses/{id}", controller="licenses", action="get", conditions=dict(method="GET")) + webapp.mapper.resource_with_deleted("history", "histories", path_prefix="/api") + webapp.mapper.connect("/api/histories/{history_id}/citations", action="citations", controller="histories") + webapp.mapper.connect( + "/api/histories/{id}/sharing", action="sharing", controller="histories", conditions=dict(method=["GET"]) + ) + webapp.mapper.connect( + "/api/histories/{id}/enable_link_access", + action="enable_link_access", + controller="histories", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "/api/histories/{id}/disable_link_access", + action="disable_link_access", + controller="histories", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "/api/histories/{id}/publish", action="publish", controller="histories", conditions=dict(method=["PUT"]) + ) + webapp.mapper.connect( + "/api/histories/{id}/unpublish", action="unpublish", controller="histories", conditions=dict(method=["PUT"]) + ) + webapp.mapper.connect( + "/api/histories/{id}/share_with_users", + action="share_with_users", + controller="histories", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "/api/histories/{id}/slug", action="set_slug", controller="histories", conditions=dict(method=["PUT"]) + ) + webapp.mapper.connect( + "dynamic_tool_confs", + "/api/configuration/dynamic_tool_confs", + controller="configuration", + action="dynamic_tool_confs", + ) + webapp.mapper.connect( + "tool_lineages", "/api/configuration/tool_lineages", controller="configuration", action="tool_lineages" + ) + webapp.mapper.connect( + "/api/configuration/toolbox", controller="configuration", action="reload_toolbox", - conditions=dict(method=["PUT"]) + conditions=dict(method=["PUT"]), + ) + webapp.mapper.resource("configuration", "configuration", path_prefix="/api") + webapp.mapper.connect( + "configuration_version", + "/api/version", + controller="configuration", + action="version", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "api_whoami", "/api/whoami", controller="configuration", action="whoami", conditions=dict(method=["GET"]) + ) + webapp.mapper.connect( + "api_decode", + "/api/configuration/decode/{encoded_id}", + controller="configuration", + action="decode_id", + conditions=dict(method=["GET"]), + ) + webapp.mapper.resource( + "datatype", + "datatypes", + path_prefix="/api", + collection={ + "sniffers": "GET", + "mapping": "GET", + "converters": "GET", + "edam_data": "GET", + "edam_formats": "GET", + "types_and_mapping": "GET", + }, + parent_resources=dict(member_name="datatype", collection_name="datatypes"), + ) + webapp.mapper.resource("search", "search", path_prefix="/api") + webapp.mapper.connect("/api/pages/{id}.pdf", action="show_pdf", controller="pages", conditions=dict(method=["GET"])) + webapp.mapper.resource("page", "pages", path_prefix="/api") + webapp.mapper.connect( + "/api/pages/{id}/sharing", action="sharing", controller="pages", conditions=dict(method=["GET"]) + ) + webapp.mapper.connect( + "/api/pages/{id}/enable_link_access", + action="enable_link_access", + controller="pages", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "/api/pages/{id}/disable_link_access", + action="disable_link_access", + controller="pages", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "/api/pages/{id}/publish", action="publish", controller="pages", conditions=dict(method=["PUT"]) + ) + webapp.mapper.connect( + "/api/pages/{id}/unpublish", action="unpublish", controller="pages", conditions=dict(method=["PUT"]) + ) + webapp.mapper.connect( + "/api/pages/{id}/share_with_users", + action="share_with_users", + controller="pages", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "/api/pages/{id}/slug", action="set_slug", controller="pages", conditions=dict(method=["PUT"]) + ) + webapp.mapper.resource( + "revision", + "revisions", + path_prefix="/api/pages/{page_id}", + controller="page_revisions", + parent_resources=dict(member_name="page", collection_name="pages"), ) - webapp.mapper.resource('configuration', 'configuration', path_prefix='/api') - webapp.mapper.connect("configuration_version", - "/api/version", controller="configuration", - action="version", conditions=dict(method=["GET"])) - webapp.mapper.connect("api_whoami", - "/api/whoami", controller='configuration', - action='whoami', - conditions=dict(method=["GET"])) - webapp.mapper.connect("api_decode", - "/api/configuration/decode/{encoded_id}", controller='configuration', - action='decode_id', - conditions=dict(method=["GET"])) - webapp.mapper.resource('datatype', - 'datatypes', - path_prefix='/api', - collection={'sniffers': 'GET', 'mapping': 'GET', 'converters': 'GET', 'edam_data': 'GET', 'edam_formats': 'GET', 'types_and_mapping': 'GET'}, - parent_resources=dict(member_name='datatype', collection_name='datatypes')) - webapp.mapper.resource('search', 'search', path_prefix='/api') - webapp.mapper.connect('/api/pages/{id}.pdf', action='show_pdf', controller="pages", conditions=dict(method=["GET"])) - webapp.mapper.resource('page', 'pages', path_prefix="/api") - webapp.mapper.connect('/api/pages/{id}/sharing', action='sharing', controller="pages", conditions=dict(method=["GET"])) - webapp.mapper.connect('/api/pages/{id}/enable_link_access', action='enable_link_access', controller="pages", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/pages/{id}/disable_link_access', action='disable_link_access', controller="pages", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/pages/{id}/publish', action='publish', controller="pages", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/pages/{id}/unpublish', action='unpublish', controller="pages", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/pages/{id}/share_with_users', action='share_with_users', controller="pages", conditions=dict(method=["PUT"])) - webapp.mapper.connect('/api/pages/{id}/slug', action='set_slug', controller="pages", conditions=dict(method=["PUT"])) - webapp.mapper.resource('revision', 'revisions', - path_prefix='/api/pages/{page_id}', - controller='page_revisions', - parent_resources=dict(member_name='page', collection_name='pages')) - webapp.mapper.connect("history_exports", - "/api/histories/{id}/exports", controller="histories", - action="index_exports", conditions=dict(method=["GET"])) - webapp.mapper.connect("history_archive_export", - "/api/histories/{id}/exports", controller="histories", - action="archive_export", conditions=dict(method=["PUT"])) - webapp.mapper.connect("history_archive_download", - "/api/histories/{id}/exports/{jeha_id}", controller="histories", - action="archive_download", conditions=dict(method=["GET"])) + webapp.mapper.connect( + "history_exports", + "/api/histories/{id}/exports", + controller="histories", + action="index_exports", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "history_archive_export", + "/api/histories/{id}/exports", + controller="histories", + action="archive_export", + conditions=dict(method=["PUT"]), + ) + webapp.mapper.connect( + "history_archive_download", + "/api/histories/{id}/exports/{jeha_id}", + controller="histories", + action="archive_download", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('/api/histories/{history_id}/contents/archive', - controller='history_contents', action='archive') - webapp.mapper.connect('/api/histories/{history_id}/contents/archive/{filename}{.format}', - controller='history_contents', action='archive') - webapp.mapper.connect("/api/histories/{history_id}/contents/dataset_collections/{id}/download", - controller='history_contents', - action='download_dataset_collection', - conditions=dict(method=["GET"])) - webapp.mapper.connect("/api/dataset_collections/{id}/download", - controller='history_contents', - action='download_dataset_collection', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "/api/histories/{history_id}/contents/archive", controller="history_contents", action="archive" + ) + webapp.mapper.connect( + "/api/histories/{history_id}/contents/archive/{filename}{.format}", + controller="history_contents", + action="archive", + ) + webapp.mapper.connect( + "/api/histories/{history_id}/contents/dataset_collections/{id}/download", + controller="history_contents", + action="download_dataset_collection", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "/api/dataset_collections/{id}/download", + controller="history_contents", + action="download_dataset_collection", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect("/api/dataset_collections/{id}/copy", - controller='dataset_collections', - action='update', - conditions=dict(method=["POST"])) + webapp.mapper.connect( + "/api/dataset_collections/{id}/copy", + controller="dataset_collections", + action="update", + conditions=dict(method=["POST"]), + ) - webapp.mapper.connect("/api/dataset_collections/{id}/attributes", - controller='dataset_collections', - action='attributes', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "/api/dataset_collections/{id}/attributes", + controller="dataset_collections", + action="attributes", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect("api_suitable_converters", - "/api/dataset_collections/{id}/suitable_converters", - controller='dataset_collections', - action='suitable_converters', - conditions=dict(method=['GET'])) + webapp.mapper.connect( + "api_suitable_converters", + "/api/dataset_collections/{id}/suitable_converters", + controller="dataset_collections", + action="suitable_converters", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect("/api/histories/{history_id}/jobs_summary", - action="index_jobs_summary", - controller='history_contents', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "/api/histories/{history_id}/jobs_summary", + action="index_jobs_summary", + controller="history_contents", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect("/api/histories/{history_id}/contents/{type:%s}s/{id}/jobs_summary" % "|".join(valid_history_contents_types), - action="show_jobs_summary", - controller='history_contents', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "/api/histories/{history_id}/contents/{type:%s}s/{id}/jobs_summary" % "|".join(valid_history_contents_types), + action="show_jobs_summary", + controller="history_contents", + conditions=dict(method=["GET"]), + ) # ---- visualizations registry ---- generic template renderer # @deprecated: this route should be considered deprecated - webapp.add_route('/visualization/show/{visualization_name}', controller='visualization', action='render', visualization_name=None) + webapp.add_route( + "/visualization/show/{visualization_name}", controller="visualization", action="render", visualization_name=None + ) # provide an alternate route to visualization plugins that's closer to their static assets # (/plugins/visualizations/{visualization_name}/static) and allow them to use relative urls to those - webapp.mapper.connect('visualization_plugin', '/plugins/visualizations/{visualization_name}/show', - controller='visualization', action='render') - webapp.mapper.connect('saved_visualization', '/plugins/visualizations/{visualization_name}/saved', - controller='visualization', action='saved') + webapp.mapper.connect( + "visualization_plugin", + "/plugins/visualizations/{visualization_name}/show", + controller="visualization", + action="render", + ) + webapp.mapper.connect( + "saved_visualization", + "/plugins/visualizations/{visualization_name}/saved", + controller="visualization", + action="saved", + ) # same with IE's - webapp.mapper.connect('interactive_environment_plugin', '/plugins/interactive_environments/{visualization_name}/show', - controller='visualization', action='render') - webapp.mapper.connect('saved_interactive_environment', '/plugins/interactive_environments/{visualization_name}/saved', - controller='visualization', action='saved') + webapp.mapper.connect( + "interactive_environment_plugin", + "/plugins/interactive_environments/{visualization_name}/show", + controller="visualization", + action="render", + ) + webapp.mapper.connect( + "saved_interactive_environment", + "/plugins/interactive_environments/{visualization_name}/saved", + controller="visualization", + action="saved", + ) # Deprecated in favor of POST /api/workflows with 'workflow' in payload. - webapp.mapper.connect('import_workflow_deprecated', - '/api/workflows/upload', - controller='workflows', - action='import_new_workflow_deprecated', - conditions=dict(method=['POST'])) - webapp.mapper.connect('workflow_dict', - '/api/workflows/{workflow_id}/download', - controller='workflows', - action='workflow_dict', - conditions=dict(method=['GET'])) - webapp.mapper.connect('show_versions', - '/api/workflows/{workflow_id}/versions', - controller='workflows', - action='show_versions', - conditions=dict(method=['GET'])) + webapp.mapper.connect( + "import_workflow_deprecated", + "/api/workflows/upload", + controller="workflows", + action="import_new_workflow_deprecated", + conditions=dict(method=["POST"]), + ) + webapp.mapper.connect( + "workflow_dict", + "/api/workflows/{workflow_id}/download", + controller="workflows", + action="workflow_dict", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "show_versions", + "/api/workflows/{workflow_id}/versions", + controller="workflows", + action="show_versions", + conditions=dict(method=["GET"]), + ) # Preserve the following download route for now for dependent applications -- deprecate at some point - webapp.mapper.connect('workflow_dict', - '/api/workflows/download/{workflow_id}', - controller='workflows', - action='workflow_dict', - conditions=dict(method=['GET'])) + webapp.mapper.connect( + "workflow_dict", + "/api/workflows/download/{workflow_id}", + controller="workflows", + action="workflow_dict", + conditions=dict(method=["GET"]), + ) # Deprecated in favor of POST /api/workflows with shared_workflow_id in payload. - webapp.mapper.connect('import_shared_workflow_deprecated', - '/api/workflows/import', - controller='workflows', - action='import_shared_workflow_deprecated', - conditions=dict(method=['POST'])) + webapp.mapper.connect( + "import_shared_workflow_deprecated", + "/api/workflows/import", + controller="workflows", + action="import_shared_workflow_deprecated", + conditions=dict(method=["POST"]), + ) webapp.mapper.connect( - 'trs_search', - '/api/trs_search', - controller='trs_search', - action="index", - conditions=dict(method=['GET']) + "trs_search", "/api/trs_search", controller="trs_search", action="index", conditions=dict(method=["GET"]) ) webapp.mapper.connect( - 'trs_consume_get_servers', - '/api/trs_consume/servers', - controller='trs_consumer', + "trs_consume_get_servers", + "/api/trs_consume/servers", + controller="trs_consumer", action="get_servers", - conditions=dict(method=['GET']) + conditions=dict(method=["GET"]), ) webapp.mapper.connect( - 'trs_consume_get_tool', - '/api/trs_consume/{trs_server}/tools/{tool_id}', - controller='trs_consumer', + "trs_consume_get_tool", + "/api/trs_consume/{trs_server}/tools/{tool_id}", + controller="trs_consumer", action="get_tool", - conditions=dict(method=['GET']) + conditions=dict(method=["GET"]), ) webapp.mapper.connect( - 'trs_consume_get_tool_versions', - '/api/trs_consume/{trs_server}/tools/{tool_id}/versions', - controller='trs_consumer', + "trs_consume_get_tool_versions", + "/api/trs_consume/{trs_server}/tools/{tool_id}/versions", + controller="trs_consumer", action="get_tool_versions", - conditions=dict(method=['GET']) + conditions=dict(method=["GET"]), ) webapp.mapper.connect( - 'trs_consume_get_tool_version', - '/api/trs_consume/{trs_server}/tools/{tool_id}/versions/{version_id}', - controller='trs_consumer', + "trs_consume_get_tool_version", + "/api/trs_consume/{trs_server}/tools/{tool_id}/versions/{version_id}", + controller="trs_consumer", action="get_tool_version", - conditions=dict(method=['GET']) + conditions=dict(method=["GET"]), ) webapp.mapper.connect( - 'trs_consume_import_tool_version', - '/api/trs_consume/{trs_server}/tools/{tool_id}/versions/{version_id}/import', - controller='trs_consumer', + "trs_consume_import_tool_version", + "/api/trs_consume/{trs_server}/tools/{tool_id}/versions/{version_id}/import", + controller="trs_consumer", action="import_tool_version", - conditions=dict(method=['POST']) + conditions=dict(method=["POST"]), ) # route for creating/getting converted datasets - webapp.mapper.connect('/api/datasets/{dataset_id}/converted', controller='datasets', action='converted', ext=None) - webapp.mapper.connect('/api/datasets/{dataset_id}/converted/{ext}', controller='datasets', action='converted') - webapp.mapper.connect('/api/datasets/{dataset_id}/permissions', controller='datasets', action='update_permissions', conditions=dict(method=["PUT"])) + webapp.mapper.connect("/api/datasets/{dataset_id}/converted", controller="datasets", action="converted", ext=None) + webapp.mapper.connect("/api/datasets/{dataset_id}/converted/{ext}", controller="datasets", action="converted") + webapp.mapper.connect( + "/api/datasets/{dataset_id}/permissions", + controller="datasets", + action="update_permissions", + conditions=dict(method=["PUT"]), + ) webapp.mapper.connect( - 'list_invocations', - '/api/invocations', - controller='workflows', - action='index_invocations', - conditions=dict(method=['GET']) + "list_invocations", + "/api/invocations", + controller="workflows", + action="index_invocations", + conditions=dict(method=["GET"]), ) # API refers to usages and invocations - these mean the same thing but the @@ -706,551 +1116,712 @@ def populate_api_routes(webapp, app): for noun, suffix in invoke_names.items(): name = f"{noun}{suffix}" webapp.mapper.connect( - f'list_workflow_{name}', - '/api/workflows/{workflow_id}/%s' % noun, - controller='workflows', - action='index_invocations', - conditions=dict(method=['GET']) + f"list_workflow_{name}", + "/api/workflows/{workflow_id}/%s" % noun, + controller="workflows", + action="index_invocations", + conditions=dict(method=["GET"]), ) webapp.mapper.connect( - f'workflow_{name}', - '/api/workflows/{workflow_id}/%s' % noun, - controller='workflows', - action='invoke', - conditions=dict(method=['POST']) + f"workflow_{name}", + "/api/workflows/{workflow_id}/%s" % noun, + controller="workflows", + action="invoke", + conditions=dict(method=["POST"]), ) def connect_invocation_endpoint(endpoint_name, endpoint_suffix, action, conditions=None): # /api/invocations/ # /api/workflows//invocations/ # /api/workflows//usage/ (deprecated) - conditions = conditions or dict(method=['GET']) + conditions = conditions or dict(method=["GET"]) webapp.mapper.connect( - f'workflow_invocation_{endpoint_name}', + f"workflow_invocation_{endpoint_name}", f"/api/workflows/{{workflow_id}}/invocations/{{invocation_id}}{endpoint_suffix}", - controller='workflows', + controller="workflows", action=action, conditions=conditions, ) webapp.mapper.connect( - f'workflow_usage_{endpoint_name}', + f"workflow_usage_{endpoint_name}", f"/api/workflows/{{workflow_id}}/usage/{{invocation_id}}{endpoint_suffix}", - controller='workflows', + controller="workflows", action=action, conditions=conditions, ) webapp.mapper.connect( - f'invocation_{endpoint_name}', + f"invocation_{endpoint_name}", f"/api/invocations/{{invocation_id}}{endpoint_suffix}", - controller='workflows', + controller="workflows", action=action, conditions=conditions, ) - connect_invocation_endpoint('show', '', action='show_invocation') - connect_invocation_endpoint('show_report', '/report', action='show_invocation_report') - connect_invocation_endpoint('show_report_pdf', '/report.pdf', action='show_invocation_report_pdf') - connect_invocation_endpoint('biocompute/download', '/biocompute/download', action='download_invocation_bco') - connect_invocation_endpoint('biocompute', '/biocompute', action='export_invocation_bco') - connect_invocation_endpoint('jobs_summary', '/jobs_summary', action='invocation_jobs_summary') - connect_invocation_endpoint('step_jobs_summary', '/step_jobs_summary', action='invocation_step_jobs_summary') - connect_invocation_endpoint('cancel', '', action='cancel_invocation', conditions=dict(method=['DELETE'])) - connect_invocation_endpoint('show_step', '/steps/{step_id}', action='invocation_step') - connect_invocation_endpoint('update_step', '/steps/{step_id}', action='update_invocation_step', conditions=dict(method=['PUT'])) + connect_invocation_endpoint("show", "", action="show_invocation") + connect_invocation_endpoint("show_report", "/report", action="show_invocation_report") + connect_invocation_endpoint("show_report_pdf", "/report.pdf", action="show_invocation_report_pdf") + connect_invocation_endpoint("biocompute/download", "/biocompute/download", action="download_invocation_bco") + connect_invocation_endpoint("biocompute", "/biocompute", action="export_invocation_bco") + connect_invocation_endpoint("jobs_summary", "/jobs_summary", action="invocation_jobs_summary") + connect_invocation_endpoint("step_jobs_summary", "/step_jobs_summary", action="invocation_step_jobs_summary") + connect_invocation_endpoint("cancel", "", action="cancel_invocation", conditions=dict(method=["DELETE"])) + connect_invocation_endpoint("show_step", "/steps/{step_id}", action="invocation_step") + connect_invocation_endpoint( + "update_step", "/steps/{step_id}", action="update_invocation_step", conditions=dict(method=["PUT"]) + ) # ============================ # ===== AUTHENTICATE API ===== # ============================ - webapp.mapper.connect('api_key_retrieval', - '/api/authenticate/baseauth/', - controller='authenticate', - action='get_api_key', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "api_key_retrieval", + "/api/authenticate/baseauth/", + controller="authenticate", + action="get_api_key", + conditions=dict(method=["GET"]), + ) # ====================================== # ====== DISPLAY APPLICATIONS API ====== # ====================================== - webapp.mapper.connect('index', - '/api/display_applications', - controller='display_applications', - action='index', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "index", + "/api/display_applications", + controller="display_applications", + action="index", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('reload', - '/api/display_applications/reload', - controller='display_applications', - action='reload', - conditions=dict(method=["POST"])) + webapp.mapper.connect( + "reload", + "/api/display_applications/reload", + controller="display_applications", + action="reload", + conditions=dict(method=["POST"]), + ) # ===================== # ===== TOURS API ===== # ===================== - webapp.mapper.connect('index', - '/api/tours', - controller='tours', - action='index', - conditions=dict(method=["GET"])) + webapp.mapper.connect("index", "/api/tours", controller="tours", action="index", conditions=dict(method=["GET"])) - webapp.mapper.connect('show', - '/api/tours/{tour_id}', - controller='tours', - action='show', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "show", "/api/tours/{tour_id}", controller="tours", action="show", conditions=dict(method=["GET"]) + ) - webapp.mapper.connect('update_tour', - '/api/tours/{tour_id}', - controller='tours', - action='update_tour', - conditions=dict(method=["POST"])) + webapp.mapper.connect( + "update_tour", + "/api/tours/{tour_id}", + controller="tours", + action="update_tour", + conditions=dict(method=["POST"]), + ) # ================================ # ===== USERS API ===== # ================================ - webapp.mapper.connect('api_key', - '/api/users/{id}/api_key', - controller='users', - action='api_key', - conditions=dict(method=["POST"])) + webapp.mapper.connect( + "api_key", "/api/users/{id}/api_key", controller="users", action="api_key", conditions=dict(method=["POST"]) + ) - webapp.mapper.connect('api_key', - '/api/users/{id}/api_key', - controller='users', - action='get_or_create_api_key', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "api_key", + "/api/users/{id}/api_key", + controller="users", + action="get_or_create_api_key", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('get_api_key', - '/api/users/{id}/api_key/inputs', - controller='users', - action='get_api_key', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "get_api_key", + "/api/users/{id}/api_key/inputs", + controller="users", + action="get_api_key", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('set_api_key', - '/api/users/{id}/api_key/inputs', - controller='users', - action='set_api_key', - conditions=dict(method=["PUT"])) + webapp.mapper.connect( + "set_api_key", + "/api/users/{id}/api_key/inputs", + controller="users", + action="set_api_key", + conditions=dict(method=["PUT"]), + ) - webapp.mapper.connect('get_information', - '/api/users/{id}/information/inputs', - controller='users', - action='get_information', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "get_information", + "/api/users/{id}/information/inputs", + controller="users", + action="get_information", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('set_information', - '/api/users/{id}/information/inputs', - controller='users', - action='set_information', - conditions=dict(method=["PUT"])) + webapp.mapper.connect( + "set_information", + "/api/users/{id}/information/inputs", + controller="users", + action="set_information", + conditions=dict(method=["PUT"]), + ) - webapp.mapper.connect('get_password', - '/api/users/{id}/password/inputs', - controller='users', - action='get_password', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "get_password", + "/api/users/{id}/password/inputs", + controller="users", + action="get_password", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('set_password', - '/api/users/{id}/password/inputs', - controller='users', - action='set_password', - conditions=dict(method=["PUT"])) + webapp.mapper.connect( + "set_password", + "/api/users/{id}/password/inputs", + controller="users", + action="set_password", + conditions=dict(method=["PUT"]), + ) - webapp.mapper.connect('get_permissions', - '/api/users/{id}/permissions/inputs', - controller='users', - action='get_permissions', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "get_permissions", + "/api/users/{id}/permissions/inputs", + controller="users", + action="get_permissions", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('set_permissions', - '/api/users/{id}/permissions/inputs', - controller='users', - action='set_permissions', - conditions=dict(method=["PUT"])) + webapp.mapper.connect( + "set_permissions", + "/api/users/{id}/permissions/inputs", + controller="users", + action="set_permissions", + conditions=dict(method=["PUT"]), + ) - webapp.mapper.connect('get_toolbox_filters', - '/api/users/{id}/toolbox_filters/inputs', - controller='users', - action='get_toolbox_filters', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "get_toolbox_filters", + "/api/users/{id}/toolbox_filters/inputs", + controller="users", + action="get_toolbox_filters", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('set_toolbox_filters', - '/api/users/{id}/toolbox_filters/inputs', - controller='users', - action='set_toolbox_filters', - conditions=dict(method=["PUT"])) + webapp.mapper.connect( + "set_toolbox_filters", + "/api/users/{id}/toolbox_filters/inputs", + controller="users", + action="set_toolbox_filters", + conditions=dict(method=["PUT"]), + ) - webapp.mapper.connect('get_communication', - '/api/users/{id}/communication/inputs', - controller='users', - action='get_communication', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "get_communication", + "/api/users/{id}/communication/inputs", + controller="users", + action="get_communication", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('set_communication', - '/api/users/{id}/communication/inputs', - controller='users', - action='set_communication', - conditions=dict(method=["PUT"])) + webapp.mapper.connect( + "set_communication", + "/api/users/{id}/communication/inputs", + controller="users", + action="set_communication", + conditions=dict(method=["PUT"]), + ) - webapp.mapper.connect('get_custom_builds', - '/api/users/{id}/custom_builds', - controller='users', - action='get_custom_builds', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "get_custom_builds", + "/api/users/{id}/custom_builds", + controller="users", + action="get_custom_builds", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('add_custom_builds', - '/api/users/{id}/custom_builds/{key}', - controller='users', - action='add_custom_builds', - conditions=dict(method=["PUT"])) + webapp.mapper.connect( + "add_custom_builds", + "/api/users/{id}/custom_builds/{key}", + controller="users", + action="add_custom_builds", + conditions=dict(method=["PUT"]), + ) - webapp.mapper.connect('delete_custom_builds', - '/api/users/{id}/custom_builds/{key}', - controller='users', - action='delete_custom_builds', - conditions=dict(method=["DELETE"])) + webapp.mapper.connect( + "delete_custom_builds", + "/api/users/{id}/custom_builds/{key}", + controller="users", + action="delete_custom_builds", + conditions=dict(method=["DELETE"]), + ) - webapp.mapper.connect('set_favorite_tool', - '/api/users/{id}/favorites/{object_type}', - controller='users', - action='set_favorite', - conditions=dict(method=["PUT"])) + webapp.mapper.connect( + "set_favorite_tool", + "/api/users/{id}/favorites/{object_type}", + controller="users", + action="set_favorite", + conditions=dict(method=["PUT"]), + ) - webapp.mapper.connect('remove_favorite_tool', - '/api/users/{id}/favorites/{object_type}/{object_id:.*?}', - controller='users', - action='remove_favorite', - conditions=dict(method=["DELETE"])) + webapp.mapper.connect( + "remove_favorite_tool", + "/api/users/{id}/favorites/{object_type}/{object_id:.*?}", + controller="users", + action="remove_favorite", + conditions=dict(method=["DELETE"]), + ) # ======================== # ===== WEBHOOKS API ===== # ======================== - webapp.mapper.connect('get_all_webhooks', - '/api/webhooks', - controller='webhooks', - action='all_webhooks', - conditions=dict(method=['GET'])) + webapp.mapper.connect( + "get_all_webhooks", + "/api/webhooks", + controller="webhooks", + action="all_webhooks", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('get_webhook_data', - '/api/webhooks/{webhook_id}/data', - controller='webhooks', - action='webhook_data', - conditions=dict(method=['GET'])) + webapp.mapper.connect( + "get_webhook_data", + "/api/webhooks/{webhook_id}/data", + controller="webhooks", + action="webhook_data", + conditions=dict(method=["GET"]), + ) # ==================== # ===== TAGS API ===== # ==================== - webapp.mapper.connect('update_tags', - '/api/tags', - controller='tags', - action='update', - conditions=dict(method=['PUT'])) + webapp.mapper.connect( + "update_tags", "/api/tags", controller="tags", action="update", conditions=dict(method=["PUT"]) + ) # ======================= # ===== LIBRARY API ===== # ======================= - webapp.mapper.connect('update_library', - '/api/libraries/{id}', - controller='libraries', - action='update', - conditions=dict(method=["PATCH", "PUT"])) + webapp.mapper.connect( + "update_library", + "/api/libraries/{id}", + controller="libraries", + action="update", + conditions=dict(method=["PATCH", "PUT"]), + ) - webapp.mapper.connect('show_library_permissions', - '/api/libraries/{encoded_library_id}/permissions', - controller='libraries', - action='get_permissions', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "show_library_permissions", + "/api/libraries/{encoded_library_id}/permissions", + controller="libraries", + action="get_permissions", + conditions=dict(method=["GET"]), + ) # POST for legacy reasons, but this should be a PUT. - webapp.mapper.connect('set_library_permissions', - '/api/libraries/{encoded_library_id}/permissions', - controller='libraries', - action='set_permissions', - conditions=dict(method=["POST", "PUT"])) + webapp.mapper.connect( + "set_library_permissions", + "/api/libraries/{encoded_library_id}/permissions", + controller="libraries", + action="set_permissions", + conditions=dict(method=["POST", "PUT"]), + ) - webapp.mapper.connect('show_ld_item', - '/api/libraries/datasets/{id}', - controller='library_datasets', - action='show', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "show_ld_item", + "/api/libraries/datasets/{id}", + controller="library_datasets", + action="show", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('load_ld', - '/api/libraries/datasets/', - controller='library_datasets', - action='load', - conditions=dict(method=["POST"])) + webapp.mapper.connect( + "load_ld", + "/api/libraries/datasets/", + controller="library_datasets", + action="load", + conditions=dict(method=["POST"]), + ) - webapp.mapper.connect('show_version_of_ld_item', - '/api/libraries/datasets/{encoded_dataset_id}/versions/{encoded_ldda_id}', - controller='library_datasets', - action='show_version', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "show_version_of_ld_item", + "/api/libraries/datasets/{encoded_dataset_id}/versions/{encoded_ldda_id}", + controller="library_datasets", + action="show_version", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('update_ld', - '/api/libraries/datasets/{encoded_dataset_id}', - controller='library_datasets', - action='update', - conditions=dict(method=["PATCH"])) + webapp.mapper.connect( + "update_ld", + "/api/libraries/datasets/{encoded_dataset_id}", + controller="library_datasets", + action="update", + conditions=dict(method=["PATCH"]), + ) - webapp.mapper.connect('show_legitimate_ld_roles', - '/api/libraries/datasets/{encoded_dataset_id}/permissions', - controller='library_datasets', - action='show_roles', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "show_legitimate_ld_roles", + "/api/libraries/datasets/{encoded_dataset_id}/permissions", + controller="library_datasets", + action="show_roles", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('update_ld_permissions', - '/api/libraries/datasets/{encoded_dataset_id}/permissions', - controller='library_datasets', - action='update_permissions', - conditions=dict(method=["POST"])) + webapp.mapper.connect( + "update_ld_permissions", + "/api/libraries/datasets/{encoded_dataset_id}/permissions", + controller="library_datasets", + action="update_permissions", + conditions=dict(method=["POST"]), + ) - webapp.mapper.connect('delete_ld_item', - '/api/libraries/datasets/{encoded_dataset_id}', - controller='library_datasets', - action='delete', - conditions=dict(method=["DELETE"])) + webapp.mapper.connect( + "delete_ld_item", + "/api/libraries/datasets/{encoded_dataset_id}", + controller="library_datasets", + action="delete", + conditions=dict(method=["DELETE"]), + ) - webapp.mapper.connect('download_ld_items', - '/api/libraries/datasets/download/{archive_format}', - controller='library_datasets', - action='download', - conditions=dict(method=["POST", "GET"])) + webapp.mapper.connect( + "download_ld_items", + "/api/libraries/datasets/download/{archive_format}", + controller="library_datasets", + action="download", + conditions=dict(method=["POST", "GET"]), + ) - webapp.mapper.resource_with_deleted('library', - 'libraries', - path_prefix='/api') + webapp.mapper.resource_with_deleted("library", "libraries", path_prefix="/api") - webapp.mapper.resource('content', - 'contents', - controller='library_contents', - name_prefix='library_', - path_prefix='/api/libraries/{library_id}', - parent_resources=dict(member_name='library', collection_name='libraries')) + webapp.mapper.resource( + "content", + "contents", + controller="library_contents", + name_prefix="library_", + path_prefix="/api/libraries/{library_id}", + parent_resources=dict(member_name="library", collection_name="libraries"), + ) - _add_item_extended_metadata_controller(webapp, - name_prefix="library_dataset_", - path_prefix='/api/libraries/{library_id}/contents/{library_content_id}') + _add_item_extended_metadata_controller( + webapp, name_prefix="library_dataset_", path_prefix="/api/libraries/{library_id}/contents/{library_content_id}" + ) # ======================= # ===== FOLDERS API ===== # ======================= - webapp.mapper.connect('add_history_datasets_to_library', - '/api/folders/{encoded_folder_id}/contents', - controller='folder_contents', - action='create', - conditions=dict(method=["POST"])) + webapp.mapper.connect( + "add_history_datasets_to_library", + "/api/folders/{encoded_folder_id}/contents", + controller="folder_contents", + action="create", + conditions=dict(method=["POST"]), + ) - webapp.mapper.connect('create_folder', - '/api/folders/{encoded_parent_folder_id}', - controller='folders', - action='create', - conditions=dict(method=["POST"])) + webapp.mapper.connect( + "create_folder", + "/api/folders/{encoded_parent_folder_id}", + controller="folders", + action="create", + conditions=dict(method=["POST"]), + ) - webapp.mapper.connect('delete_folder', - '/api/folders/{encoded_folder_id}', - controller='folders', - action='delete', - conditions=dict(method=["DELETE"])) + webapp.mapper.connect( + "delete_folder", + "/api/folders/{encoded_folder_id}", + controller="folders", + action="delete", + conditions=dict(method=["DELETE"]), + ) - webapp.mapper.connect('update_folder', - '/api/folders/{encoded_folder_id}', - controller='folders', - action='update', - conditions=dict(method=["PATCH", "PUT"])) + webapp.mapper.connect( + "update_folder", + "/api/folders/{encoded_folder_id}", + controller="folders", + action="update", + conditions=dict(method=["PATCH", "PUT"]), + ) - webapp.mapper.resource('folder', - 'folders', - path_prefix='/api') + webapp.mapper.resource("folder", "folders", path_prefix="/api") - webapp.mapper.connect('show_folder_permissions', - '/api/folders/{encoded_folder_id}/permissions', - controller='folders', - action='get_permissions', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "show_folder_permissions", + "/api/folders/{encoded_folder_id}/permissions", + controller="folders", + action="get_permissions", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('set_folder_permissions', - '/api/folders/{encoded_folder_id}/permissions', - controller='folders', - action='set_permissions', - conditions=dict(method=["POST"])) + webapp.mapper.connect( + "set_folder_permissions", + "/api/folders/{encoded_folder_id}/permissions", + controller="folders", + action="set_permissions", + conditions=dict(method=["POST"]), + ) - webapp.mapper.resource('content', - 'contents', - controller='folder_contents', - name_prefix='folder_', - path_prefix='/api/folders/{folder_id}', - parent_resources=dict(member_name='folder', collection_name='folders'), - conditions=dict(method=["GET"])) + webapp.mapper.resource( + "content", + "contents", + controller="folder_contents", + name_prefix="folder_", + path_prefix="/api/folders/{folder_id}", + parent_resources=dict(member_name="folder", collection_name="folders"), + conditions=dict(method=["GET"]), + ) - webapp.mapper.resource('job', - 'jobs', - path_prefix='/api') - webapp.mapper.connect('job_search', '/api/jobs/search', controller='jobs', action='search', conditions=dict(method=['POST'])) - webapp.mapper.connect('job_inputs', '/api/jobs/{id}/inputs', controller='jobs', action='inputs', conditions=dict(method=['GET'])) - webapp.mapper.connect('job_outputs', '/api/jobs/{id}/outputs', controller='jobs', action='outputs', conditions=dict(method=['GET'])) - webapp.mapper.connect('build_for_rerun', '/api/jobs/{id}/build_for_rerun', controller='jobs', action='build_for_rerun', conditions=dict(method=['GET'])) - webapp.mapper.connect('resume', '/api/jobs/{id}/resume', controller='jobs', action='resume', conditions=dict(method=['PUT'])) - webapp.mapper.connect('job_error', '/api/jobs/{id}/error', controller='jobs', action='error', conditions=dict(method=['POST'])) - webapp.mapper.connect('common_problems', '/api/jobs/{id}/common_problems', controller='jobs', action='common_problems', conditions=dict(method=['GET'])) + webapp.mapper.resource("job", "jobs", path_prefix="/api") + webapp.mapper.connect( + "job_search", "/api/jobs/search", controller="jobs", action="search", conditions=dict(method=["POST"]) + ) + webapp.mapper.connect( + "job_inputs", "/api/jobs/{id}/inputs", controller="jobs", action="inputs", conditions=dict(method=["GET"]) + ) + webapp.mapper.connect( + "job_outputs", "/api/jobs/{id}/outputs", controller="jobs", action="outputs", conditions=dict(method=["GET"]) + ) + webapp.mapper.connect( + "build_for_rerun", + "/api/jobs/{id}/build_for_rerun", + controller="jobs", + action="build_for_rerun", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "resume", "/api/jobs/{id}/resume", controller="jobs", action="resume", conditions=dict(method=["PUT"]) + ) + webapp.mapper.connect( + "job_error", "/api/jobs/{id}/error", controller="jobs", action="error", conditions=dict(method=["POST"]) + ) + webapp.mapper.connect( + "common_problems", + "/api/jobs/{id}/common_problems", + controller="jobs", + action="common_problems", + conditions=dict(method=["GET"]), + ) # Job metrics and parameters by job id or dataset id (for slightly different accessibility checking) - webapp.mapper.connect('metrics', '/api/jobs/{job_id}/metrics', controller='jobs', action='metrics', conditions=dict(method=['GET'])) - webapp.mapper.connect('destination_params', '/api/jobs/{job_id}/destination_params', controller='jobs', action='destination_params', conditions=dict(method=['GET'])) - webapp.mapper.connect('show_job_lock', '/api/job_lock', controller='jobs', action='show_job_lock', conditions=dict(method=['GET'])) - webapp.mapper.connect('update_job_lock', '/api/job_lock', controller='jobs', action='update_job_lock', conditions=dict(method=['PUT'])) - webapp.mapper.connect('dataset_metrics', '/api/datasets/{dataset_id}/metrics', controller='jobs', action='metrics', conditions=dict(method=['GET'])) - webapp.mapper.connect('parameters_display', '/api/jobs/{job_id}/parameters_display', controller='jobs', action='parameters_display', conditions=dict(method=['GET'])) - webapp.mapper.connect('dataset_parameters_display', '/api/datasets/{dataset_id}/parameters_display', controller='jobs', action='parameters_display', conditions=dict(method=['GET'])) + webapp.mapper.connect( + "metrics", "/api/jobs/{job_id}/metrics", controller="jobs", action="metrics", conditions=dict(method=["GET"]) + ) + webapp.mapper.connect( + "destination_params", + "/api/jobs/{job_id}/destination_params", + controller="jobs", + action="destination_params", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "show_job_lock", "/api/job_lock", controller="jobs", action="show_job_lock", conditions=dict(method=["GET"]) + ) + webapp.mapper.connect( + "update_job_lock", "/api/job_lock", controller="jobs", action="update_job_lock", conditions=dict(method=["PUT"]) + ) + webapp.mapper.connect( + "dataset_metrics", + "/api/datasets/{dataset_id}/metrics", + controller="jobs", + action="metrics", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "parameters_display", + "/api/jobs/{job_id}/parameters_display", + controller="jobs", + action="parameters_display", + conditions=dict(method=["GET"]), + ) + webapp.mapper.connect( + "dataset_parameters_display", + "/api/datasets/{dataset_id}/parameters_display", + controller="jobs", + action="parameters_display", + conditions=dict(method=["GET"]), + ) # Job files controllers. Only for consumption by remote job runners. - webapp.mapper.resource('file', - 'files', - controller="job_files", - name_prefix="job_", - path_prefix='/api/jobs/{job_id}', - parent_resources=dict(member_name="job", collection_name="jobs")) - webapp.mapper.resource('port', - 'ports', - controller="job_ports", - name_prefix="job_", - path_prefix='/api/jobs/{job_id}', - parent_resources=dict(member_name="job", collection_name="jobs")) + webapp.mapper.resource( + "file", + "files", + controller="job_files", + name_prefix="job_", + path_prefix="/api/jobs/{job_id}", + parent_resources=dict(member_name="job", collection_name="jobs"), + ) + webapp.mapper.resource( + "port", + "ports", + controller="job_ports", + name_prefix="job_", + path_prefix="/api/jobs/{job_id}", + parent_resources=dict(member_name="job", collection_name="jobs"), + ) - _add_item_extended_metadata_controller(webapp, - name_prefix="history_dataset_", - path_prefix='/api/histories/{history_id}/contents/{history_content_id}') + _add_item_extended_metadata_controller( + webapp, name_prefix="history_dataset_", path_prefix="/api/histories/{history_id}/contents/{history_content_id}" + ) # ==================== # ===== TOOLSHED ===== # ==================== # Handle displaying tool help images and README file images contained in repositories installed from the tool shed. - webapp.add_route('/admin_toolshed/static/images/{repository_id}/{image_file:.+?}', - controller='admin_toolshed', - action='display_image_in_repository', - repository_id=None, - image_file=None) + webapp.add_route( + "/admin_toolshed/static/images/{repository_id}/{image_file:.+?}", + controller="admin_toolshed", + action="display_image_in_repository", + repository_id=None, + image_file=None, + ) # Do the same but without a repository id - webapp.add_route('/shed_tool_static/{shed}/{owner}/{repo}/{tool}/{version}/{image_file:.+?}', - controller='shed_tool_static', - action='index', - shed=None, - owner=None, - repo=None, - tool=None, - version=None, - image_file=None) + webapp.add_route( + "/shed_tool_static/{shed}/{owner}/{repo}/{tool}/{version}/{image_file:.+?}", + controller="shed_tool_static", + action="index", + shed=None, + owner=None, + repo=None, + tool=None, + version=None, + image_file=None, + ) - webapp.mapper.connect('tool_shed_contents', - '/api/tool_shed/contents', - controller='toolshed', - action='show', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "tool_shed_contents", + "/api/tool_shed/contents", + controller="toolshed", + action="show", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('tool_shed_category_contents', - '/api/tool_shed/category', - controller='toolshed', - action='category', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "tool_shed_category_contents", + "/api/tool_shed/category", + controller="toolshed", + action="category", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('tool_shed_repository_details', - '/api/tool_shed/repository', - controller='toolshed', - action='repository', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "tool_shed_repository_details", + "/api/tool_shed/repository", + controller="toolshed", + action="repository", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('tool_sheds', - '/api/tool_shed', - controller='toolshed', - action='index', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "tool_sheds", "/api/tool_shed", controller="toolshed", action="index", conditions=dict(method=["GET"]) + ) - webapp.mapper.connect('tool_shed_search', - '/api/tool_shed/search', - controller='toolshed', - action='search', - conditions=dict(method=["GET", "POST"])) + webapp.mapper.connect( + "tool_shed_search", + "/api/tool_shed/search", + controller="toolshed", + action="search", + conditions=dict(method=["GET", "POST"]), + ) - webapp.mapper.connect('tool_shed_request', - '/api/tool_shed/request', - controller='toolshed', - action='request', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "tool_shed_request", + "/api/tool_shed/request", + controller="toolshed", + action="request", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('tool_shed_status', - '/api/tool_shed/status', - controller='toolshed', - action='status', - conditions=dict(method=["GET", "POST"])) + webapp.mapper.connect( + "tool_shed_status", + "/api/tool_shed/status", + controller="toolshed", + action="status", + conditions=dict(method=["GET", "POST"]), + ) - webapp.mapper.connect('shed_tool_json', - '/api/tool_shed/tool_json', - controller='toolshed', - action='tool_json', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "shed_tool_json", + "/api/tool_shed/tool_json", + controller="toolshed", + action="tool_json", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('tool_shed_repository', - '/api/tool_shed_repositories/:id/status', - controller='tool_shed_repositories', - action='status', - conditions=dict(method=["GET"])) + webapp.mapper.connect( + "tool_shed_repository", + "/api/tool_shed_repositories/:id/status", + controller="tool_shed_repositories", + action="status", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('install_repository', - '/api/tool_shed_repositories', - controller='tool_shed_repositories', - action='install_repository_revision', - conditions=dict(method=['POST'])) + webapp.mapper.connect( + "install_repository", + "/api/tool_shed_repositories", + controller="tool_shed_repositories", + action="install_repository_revision", + conditions=dict(method=["POST"]), + ) - webapp.mapper.connect('install_repository', - '/api/tool_shed_repositories/install', - controller='tool_shed_repositories', - action='install', - conditions=dict(method=['POST'])) + webapp.mapper.connect( + "install_repository", + "/api/tool_shed_repositories/install", + controller="tool_shed_repositories", + action="install", + conditions=dict(method=["POST"]), + ) - webapp.mapper.connect('check_for_updates', - '/api/tool_shed_repositories/check_for_updates', - controller='tool_shed_repositories', - action='check_for_updates', - conditions=dict(method=['GET'])) + webapp.mapper.connect( + "check_for_updates", + "/api/tool_shed_repositories/check_for_updates", + controller="tool_shed_repositories", + action="check_for_updates", + conditions=dict(method=["GET"]), + ) - webapp.mapper.connect('tool_shed_repository', - '/api/tool_shed_repositories', - controller='tool_shed_repositories', - action='uninstall_repository', - conditions=dict(method=["DELETE"])) + webapp.mapper.connect( + "tool_shed_repository", + "/api/tool_shed_repositories", + controller="tool_shed_repositories", + action="uninstall_repository", + conditions=dict(method=["DELETE"]), + ) - webapp.mapper.connect('tool_shed_repository', - '/api/tool_shed_repositories/{id}', - controller='tool_shed_repositories', - action='uninstall_repository', - conditions=dict(method=["DELETE"])) + webapp.mapper.connect( + "tool_shed_repository", + "/api/tool_shed_repositories/{id}", + controller="tool_shed_repositories", + action="uninstall_repository", + conditions=dict(method=["DELETE"]), + ) - webapp.mapper.connect('reset_metadata_on_selected_installed_repositories', - '/api/tool_shed_repositories/reset_metadata_on_selected_installed_repositories', - controller='tool_shed_repositories', - action='reset_metadata_on_selected_installed_repositories', - conditions=dict(method=['POST'])) + webapp.mapper.connect( + "reset_metadata_on_selected_installed_repositories", + "/api/tool_shed_repositories/reset_metadata_on_selected_installed_repositories", + controller="tool_shed_repositories", + action="reset_metadata_on_selected_installed_repositories", + conditions=dict(method=["POST"]), + ) # Galaxy API for tool shed features. - webapp.mapper.resource('tool_shed_repository', - 'tool_shed_repositories', - member={'repair_repository_revision': 'POST', - 'exported_workflows': 'GET', - 'import_workflow': 'POST', - 'import_workflows': 'POST'}, - collection={'get_latest_installable_revision': 'POST', - 'reset_metadata_on_installed_repositories': 'POST'}, - controller='tool_shed_repositories', - name_prefix='tool_shed_repository_', - path_prefix='/api', - new={'install_repository_revision': 'POST'}, - parent_resources=dict(member_name='tool_shed_repository', collection_name='tool_shed_repositories')) + webapp.mapper.resource( + "tool_shed_repository", + "tool_shed_repositories", + member={ + "repair_repository_revision": "POST", + "exported_workflows": "GET", + "import_workflow": "POST", + "import_workflows": "POST", + }, + collection={"get_latest_installable_revision": "POST", "reset_metadata_on_installed_repositories": "POST"}, + controller="tool_shed_repositories", + name_prefix="tool_shed_repository_", + path_prefix="/api", + new={"install_repository_revision": "POST"}, + parent_resources=dict(member_name="tool_shed_repository", collection_name="tool_shed_repositories"), + ) # ==== Trace/Metrics Logger # Connect logger from app @@ -1262,8 +1833,9 @@ def populate_api_routes(webapp, app): # controller="metrics", action="index", conditions=dict( method=["GET"] ) ) # webapp.mapper.connect( "show", "/api/metrics/{id}", # controller="metrics", action="show", conditions=dict( method=["GET"] ) ) - webapp.mapper.connect("create", "/api/metrics", controller="metrics", - action="create", conditions=dict(method=["POST"])) + webapp.mapper.connect( + "create", "/api/metrics", controller="metrics", action="create", conditions=dict(method=["POST"]) + ) def _add_item_tags_controller(webapp, name_prefix, path_prefix, **kwd): @@ -1273,25 +1845,39 @@ def _add_item_tags_controller(webapp, name_prefix, path_prefix, **kwd): path = f"{path_prefix}/tags" map = webapp.mapper # Allow view items' tags. - map.connect(name, path, - controller=controller, action="index", - conditions=dict(method=["GET"])) + map.connect(name, path, controller=controller, action="index", conditions=dict(method=["GET"])) # Allow remove tag from item - map.connect(f"{name}_delete", "%s/tags/{tag_name}" % path_prefix, - controller=controller, action="delete", - conditions=dict(method=["DELETE"])) + map.connect( + f"{name}_delete", + "%s/tags/{tag_name}" % path_prefix, + controller=controller, + action="delete", + conditions=dict(method=["DELETE"]), + ) # Allow create a new tag with from name - map.connect(f"{name}_create", "%s/tags/{tag_name}" % path_prefix, - controller=controller, action="create", - conditions=dict(method=["POST"])) + map.connect( + f"{name}_create", + "%s/tags/{tag_name}" % path_prefix, + controller=controller, + action="create", + conditions=dict(method=["POST"]), + ) # Allow update tag value - map.connect(f"{name}_update", "%s/tags/{tag_name}" % path_prefix, - controller=controller, action="update", - conditions=dict(method=["PUT"])) + map.connect( + f"{name}_update", + "%s/tags/{tag_name}" % path_prefix, + controller=controller, + action="update", + conditions=dict(method=["PUT"]), + ) # Allow show tag by name - map.connect(f"{name}_show", "%s/tags/{tag_name}" % path_prefix, - controller=controller, action="show", - conditions=dict(method=["GET"])) + map.connect( + f"{name}_show", + "%s/tags/{tag_name}" % path_prefix, + controller=controller, + action="show", + conditions=dict(method=["GET"]), + ) def _add_item_extended_metadata_controller(webapp, name_prefix, path_prefix, **kwd): @@ -1323,81 +1909,105 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf): # Merge the global and local configurations conf = global_conf.copy() conf.update(local_conf) - debug = asbool(conf.get('debug', False)) + debug = asbool(conf.get("debug", False)) # First put into place httpexceptions, which must be most closely # wrapped around the application (it can interact poorly with # other middleware): - app = wrap_if_allowed(app, stack, httpexceptions.make_middleware, name='paste.httpexceptions', args=(conf,)) + app = wrap_if_allowed(app, stack, httpexceptions.make_middleware, name="paste.httpexceptions", args=(conf,)) # Statsd request timing and profiling - statsd_host = conf.get('statsd_host', None) + statsd_host = conf.get("statsd_host", None) if statsd_host: from galaxy.web.framework.middleware.statsd import StatsdMiddleware - app = wrap_if_allowed(app, stack, StatsdMiddleware, - args=(statsd_host, - conf.get('statsd_port', 8125), - conf.get('statsd_prefix', 'galaxy'), - conf.get('statsd_influxdb', False), - conf.get('statsd_mock_calls', False))) + + app = wrap_if_allowed( + app, + stack, + StatsdMiddleware, + args=( + statsd_host, + conf.get("statsd_port", 8125), + conf.get("statsd_prefix", "galaxy"), + conf.get("statsd_influxdb", False), + conf.get("statsd_mock_calls", False), + ), + ) log.info("Enabling 'statsd' middleware") # If we're using remote_user authentication, add middleware that # protects Galaxy from improperly configured authentication in the # upstream server - single_user = conf.get('single_user', None) - use_remote_user = asbool(conf.get('use_remote_user', False)) or single_user + single_user = conf.get("single_user", None) + use_remote_user = asbool(conf.get("use_remote_user", False)) or single_user if use_remote_user: from galaxy.web.framework.middleware.remoteuser import RemoteUser - app = wrap_if_allowed(app, stack, RemoteUser, - kwargs=dict( - maildomain=conf.get('remote_user_maildomain', None), - display_servers=util.listify(conf.get('display_servers', '')), - single_user=single_user, - admin_users=conf.get('admin_users', '').split(','), - remote_user_header=conf.get('remote_user_header', 'HTTP_REMOTE_USER'), - remote_user_secret_header=conf.get('remote_user_secret', None), - normalize_remote_user_email=conf.get('normalize_remote_user_email', False))) + + app = wrap_if_allowed( + app, + stack, + RemoteUser, + kwargs=dict( + maildomain=conf.get("remote_user_maildomain", None), + display_servers=util.listify(conf.get("display_servers", "")), + single_user=single_user, + admin_users=conf.get("admin_users", "").split(","), + remote_user_header=conf.get("remote_user_header", "HTTP_REMOTE_USER"), + remote_user_secret_header=conf.get("remote_user_secret", None), + normalize_remote_user_email=conf.get("normalize_remote_user_email", False), + ), + ) # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. - if asbool(conf.get('use_recursive', True)): + if asbool(conf.get("use_recursive", True)): from paste import recursive + app = wrap_if_allowed(app, stack, recursive.RecursiveMiddleware, args=(conf,)) # If sentry logging is enabled, log here before propogating up to # the error middleware - sentry_dsn = conf.get('sentry_dsn', None) + sentry_dsn = conf.get("sentry_dsn", None) if sentry_dsn: from sentry_sdk.integrations.wsgi import SentryWsgiMiddleware + app = wrap_if_allowed(app, stack, SentryWsgiMiddleware) # Various debug middleware that can only be turned on if the debug # flag is set, either because they are insecure or greatly hurt # performance if debug: # Middleware to check for WSGI compliance - if asbool(conf.get('use_lint', False)): + if asbool(conf.get("use_lint", False)): from paste import lint - app = wrap_if_allowed(app, stack, lint.make_middleware, name='paste.lint', args=(conf,)) + + app = wrap_if_allowed(app, stack, lint.make_middleware, name="paste.lint", args=(conf,)) # Middleware to run the python profiler on each request - if asbool(conf.get('use_profile', False)): + if asbool(conf.get("use_profile", False)): from paste.debug import profile + app = wrap_if_allowed(app, stack, profile.ProfileMiddleware, args=(conf,)) # Error middleware app = wrap_if_allowed(app, stack, ErrorMiddleware, args=(conf,)) # Transaction logging (apache access.log style) - if asbool(conf.get('use_translogger', True)): + if asbool(conf.get("use_translogger", True)): from galaxy.web.framework.middleware.translogger import TransLogger + app = wrap_if_allowed(app, stack, TransLogger) # X-Forwarded-Host handling app = wrap_if_allowed(app, stack, XForwardedHostMiddleware) # Request ID middleware app = wrap_if_allowed(app, stack, RequestIDMiddleware) # TUS upload middleware - app = wrap_if_allowed(app, stack, TusMiddleware, kwargs={ - 'upload_path': urljoin(f"{application_stack.config.galaxy_url_prefix}/", 'api/upload/resumable_upload'), - 'tmp_dir': application_stack.config.tus_upload_store or application_stack.config.new_file_path, - 'max_size': application_stack.config.maximum_upload_file_size - }) + app = wrap_if_allowed( + app, + stack, + TusMiddleware, + kwargs={ + "upload_path": urljoin(f"{application_stack.config.galaxy_url_prefix}/", "api/upload/resumable_upload"), + "tmp_dir": application_stack.config.tus_upload_store or application_stack.config.new_file_path, + "max_size": application_stack.config.maximum_upload_file_size, + }, + ) # api batch call processing middleware app = wrap_if_allowed(app, stack, BatchMiddleware, args=(webapp, {})) - if asbool(conf.get('enable_per_request_sql_debugging', False)): + if asbool(conf.get("enable_per_request_sql_debugging", False)): from galaxy.web.framework.middleware.sqldebug import SQLDebugMiddleware + app = wrap_if_allowed(app, stack, SQLDebugMiddleware, args=(webapp, {})) return app diff --git a/lib/galaxy/webapps/galaxy/services/datasets.py b/lib/galaxy/webapps/galaxy/services/datasets.py index 39c155791e4..53e3613f483 100644 --- a/lib/galaxy/webapps/galaxy/services/datasets.py +++ b/lib/galaxy/webapps/galaxy/services/datasets.py @@ -70,6 +70,7 @@ DEFAULT_LIMIT = 500 class RequestDataType(str, Enum): """Particular pieces of information that can be requested for a dataset.""" + state = "state" converted_datasets_state = "converted_datasets_state" data = "data" @@ -140,6 +141,7 @@ class DatasetTextContentDetails(Model): class ConvertedDatasetsMap(BaseModel): """Map of `file extension` -> `converted dataset encoded id`""" + __root__: Dict[str, EncodedDatabaseIdField] # extension -> dataset ID class Config: @@ -168,7 +170,6 @@ class BamDataResult(DataResult): class DatasetsService(ServiceBase, UsesVisualizationMixin): - def __init__( self, security: IdEncodingHelper, @@ -179,7 +180,7 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): ldda_manager: LDDAManager, history_contents_manager: HistoryContentsManager, history_contents_filters: HistoryContentsFilters, - data_provider_registry: DataProviderRegistry + data_provider_registry: DataProviderRegistry, ): super().__init__(security) self.history_manager = history_manager @@ -193,11 +194,11 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): @property def serializer_by_type(self) -> Dict[str, ModelSerializer]: - return {'dataset': self.hda_serializer, 'dataset_collection': self.hdca_serializer} + return {"dataset": self.hda_serializer, "dataset_collection": self.hdca_serializer} @property def dataset_manager_by_type(self) -> Dict[str, DatasetAssociationManager]: - return {'hda': self.hda_manager, 'ldda': self.ldda_manager} + return {"hda": self.hda_manager, "ldda": self.ldda_manager} def index( self, @@ -212,7 +213,7 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): """ user = self.get_authenticated_user(trans) filters = self.history_contents_filters.parse_query_filters(filter_query_params) - view = serialization_params.view or 'summary' + view = serialization_params.view or "summary" order_by = self.build_order_by(self.history_contents_manager, filter_query_params.order or "create_time-dsc") container = None if history_id: @@ -226,7 +227,9 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): user_id=user.id, ) return [ - self.serializer_by_type[content.history_content_type].serialize_to_view(content, user=user, trans=trans, view=view) + self.serializer_by_type[content.history_content_type].serialize_to_view( + content, user=user, trans=trans, view=view + ) for content in contents ] @@ -251,7 +254,8 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): rval = self._dataset_state(dataset) elif data_type == RequestDataType.converted_datasets_state: rval = self._converted_datasets_state( - trans, dataset, + trans, + dataset, chrom=extra_params.get("chrom", None), retry=extra_params.get("retry", False), ) @@ -271,10 +275,7 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): # Default: return dataset as dict. if hda_ldda == DatasetSourceType.hda: return self.hda_serializer.serialize_to_view( - dataset, - view=serialization_params.view or 'detailed', - user=trans.user, - trans=trans + dataset, view=serialization_params.view or "detailed", user=trans.user, trans=trans ) else: dataset_dict = dataset.to_dict() @@ -362,7 +363,9 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): rval = [] for root, directories, files in safe_walk(extra_files_path): for directory in directories: - rval.append({"class": "Directory", "path": os.path.relpath(os.path.join(root, directory), extra_files_path)}) + rval.append( + {"class": "Directory", "path": os.path.relpath(os.path.join(root, directory), extra_files_path)} + ) for file in files: rval.append({"class": "File", "path": os.path.relpath(os.path.join(root, file), extra_files_path)}) @@ -388,26 +391,25 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): """ decoded_content_id = self.decode_id(history_content_id) headers = {} - rval: Any = '' + rval: Any = "" try: hda = self.hda_manager.get_accessible(decoded_content_id, trans.user) if raw: - if filename and filename != 'index': + if filename and filename != "index": object_store = trans.app.object_store dir_name = hda.dataset.extra_files_path_name - file_path = object_store.get_filename(hda.dataset, - extra_dir=dir_name, - alt_name=filename) + file_path = object_store.get_filename(hda.dataset, extra_dir=dir_name, alt_name=filename) else: file_path = hda.file_name - rval = open(file_path, 'rb') + rval = open(file_path, "rb") else: rval, headers = hda.datatype.display_data(trans, hda, preview, filename, to_ext, **kwd) except galaxy_exceptions.MessageException: raise except Exception as e: - log.exception("Server error getting display data for dataset (%s) from history (%s)", - history_content_id, history_id) + log.exception( + "Server error getting display data for dataset (%s) from history (%s)", history_content_id, history_id + ) raise galaxy_exceptions.InternalServerError(f"Could not get display data for dataset: {util.unicodify(e)}") return rval, headers @@ -416,14 +418,18 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): trans: ProvidesHistoryContext, dataset_id: EncodedDatabaseIdField, ) -> DatasetTextContentDetails: - """ Returns dataset content as Text. """ + """Returns dataset content as Text.""" user = self.get_authenticated_user(trans) decoded_id = self.decode_id(dataset_id) hda = self.hda_manager.get_accessible(decoded_id, user) hda = self.hda_manager.error_if_uploading(hda) truncated, dataset_data = self.hda_manager.text_data(hda, preview=True) item_url = web.url_for( - controller='dataset', action='display_by_username_and_slug', username=hda.history.user.username, slug=self.encode_id(hda.id), preview=False + controller="dataset", + action="display_by_username_and_slug", + username=hda.history.user.username, + slug=self.encode_id(hda.id), + preview=False, ) return DatasetTextContentDetails( item_data=dataset_data, @@ -447,13 +453,13 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): decoded_content_id = self.decode_id(history_content_id) hda = self.hda_manager.get_accessible(decoded_content_id, trans.user) file_ext = hda.metadata.spec.get(metadata_file).get("file_ext", metadata_file) - fname = ''.join(c in util.FILENAME_VALID_CHARS and c or '_' for c in hda.name)[0:150] + fname = "".join(c in util.FILENAME_VALID_CHARS and c or "_" for c in hda.name)[0:150] headers = {} headers["Content-Type"] = "application/octet-stream" headers["Content-Disposition"] = f'attachment; filename="Galaxy{hda.hid}-[{fname}].{file_ext}"' file_path = hda.metadata.get(metadata_file).file_name if open_file: - return open(file_path, 'rb'), headers + return open(file_path, "rb"), headers return file_path, headers def converted_ext( @@ -471,10 +477,7 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): serialization_params.default_view = "detailed" converted = self._get_or_create_converted(trans, hda, ext) return self.hda_serializer.serialize_to_view( - converted, - user=trans.user, - trans=trans, - **serialization_params.dict() + converted, user=trans.user, trans=trans, **serialization_params.dict() ) def converted( @@ -488,7 +491,7 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): """ decoded_id = self.decode_id(dataset_id) hda = self.hda_manager.get_accessible(decoded_id, trans.user) - return self.hda_serializer.serialize_converted_datasets(hda, 'converted') + return self.hda_serializer.serialize_converted_datasets(hda, "converted") def _get_or_create_converted(self, trans, original: model.DatasetInstance, target_ext: str): try: @@ -498,11 +501,9 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): except model.NoConverterException: exc_data = dict( - source=original.ext, - target=target_ext, - available=list(original.get_converter_types().keys()) + source=original.ext, target=target_ext, available=list(original.get_converter_types().keys()) ) - raise galaxy_exceptions.RequestParameterInvalidException('Conversion not possible', **exc_data) + raise galaxy_exceptions.RequestParameterInvalidException("Conversion not possible", **exc_data) def _dataset_in_use_state(self, dataset: model.DatasetInstance) -> bool: """ @@ -537,7 +538,7 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): # Get datasources and check for messages (which indicate errors). Retry if flag is set. data_sources = dataset.get_datasources(trans) - messages_list = [data_source_dict['message'] for data_source_dict in data_sources.values()] + messages_list = [data_source_dict["message"] for data_source_dict in data_sources.values()] msg = self._get_highest_priority_msg(messages_list) if msg: if retry: @@ -549,8 +550,9 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): # If there is a chrom, check for data on the chrom. if chrom: - data_provider = self.data_provider_registry.get_data_provider(trans, - original_dataset=dataset, source='index') + data_provider = self.data_provider_registry.get_data_provider( + trans, original_dataset=dataset, source="index" + ) if not data_provider.has_data(chrom): return dataset.conversion_messages.NO_DATA @@ -558,7 +560,10 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): return {"status": dataset.conversion_messages.DATA, "valid_chroms": None} def _search_features( - self, trans, dataset: model.DatasetInstance, query: Optional[str], + self, + trans, + dataset: model.DatasetInstance, + query: Optional[str], ) -> List[List[str]]: """ Returns features, locations in dataset that match query. Format is a @@ -602,7 +607,7 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): # Get datasources and check for messages. data_sources = dataset.get_datasources(trans) - messages_list = [data_source_dict['message'] for data_source_dict in data_sources.values()] + messages_list = [data_source_dict["message"] for data_source_dict in data_sources.values()] return_message = self._get_highest_priority_msg(messages_list) if return_message: return return_message @@ -614,7 +619,7 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): # Coverage mode uses index data. if mode == "Coverage": # Get summary using minimal cutoffs. - indexer = self.data_provider_registry.get_data_provider(trans, original_dataset=dataset, source='index') + indexer = self.data_provider_registry.get_data_provider(trans, original_dataset=dataset, source="index") return indexer.get_data(chrom, low, high, **kwargs) # TODO: @@ -624,19 +629,19 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): # If mode is Auto, need to determine what type of data to return. if mode == "Auto": # Get stats from indexer. - indexer = self.data_provider_registry.get_data_provider(trans, original_dataset=dataset, source='index') + indexer = self.data_provider_registry.get_data_provider(trans, original_dataset=dataset, source="index") stats = indexer.get_data(chrom, low, high, stats=True) # If stats were requested, return them. - if 'stats' in kwargs: - if stats['data']['max'] == 0: + if "stats" in kwargs: + if stats["data"]["max"] == 0: return DataResult(dataset_type=indexer.dataset_type, data=None) else: return stats # Stats provides features/base and resolution is bases/pixel, so # multiplying them yields features/pixel. - features_per_pixel = stats['data']['max'] * float(kwargs['resolution']) + features_per_pixel = stats["data"]["max"] * float(kwargs["resolution"]) # Use heuristic based on features/pixel and region size to determine whether to # return coverage data. When zoomed out and region is large, features/pixel @@ -650,7 +655,7 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): # # Get data provider. - data_provider = self.data_provider_registry.get_data_provider(trans, original_dataset=dataset, source='data') + data_provider = self.data_provider_registry.get_data_provider(trans, original_dataset=dataset, source="data") # Allow max_vals top be data provider set if not passed if max_vals is None: @@ -665,24 +670,33 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): # FIXME: increase region 1M each way to provide sequence for # spliced/gapped reads. Probably should provide refseq object # directly to data provider. - region = trans.app.genomes.reference(trans, dbkey=dataset.dbkey, chrom=chrom, - low=(max(0, int(low) - 1000000)), - high=(int(high) + 1000000)) + region = trans.app.genomes.reference( + trans, + dbkey=dataset.dbkey, + chrom=chrom, + low=(max(0, int(low) - 1000000)), + high=(int(high) + 1000000), + ) # Get mean depth. if not indexer: - indexer = self.data_provider_registry.get_data_provider(trans, original_dataset=dataset, source='index') + indexer = self.data_provider_registry.get_data_provider(trans, original_dataset=dataset, source="index") stats = indexer.get_data(chrom, low, high, stats=True) - mean_depth = stats['data']['mean'] + mean_depth = stats["data"]["mean"] # Get and return data from data_provider. - result = data_provider.get_data(chrom, int(low), int(high), int(start_val), int(max_vals), - ref_seq=region, mean_depth=mean_depth, **kwargs) - result.update({'dataset_type': data_provider.dataset_type, 'extra_info': extra_info}) + result = data_provider.get_data( + chrom, int(low), int(high), int(start_val), int(max_vals), ref_seq=region, mean_depth=mean_depth, **kwargs + ) + result.update({"dataset_type": data_provider.dataset_type, "extra_info": extra_info}) return result def _raw_data( - self, trans, dataset, provider=None, **kwargs, + self, + trans, + dataset, + provider=None, + **kwargs, ) -> Union[model.Dataset.conversion_messages, BamDataResult, DataResult]: """ Uses original (raw) dataset to return data. This method is useful @@ -705,9 +719,7 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): elif dataset.datatype.has_dataprovider(provider): kwargs = dataset.datatype.dataproviders[provider].parse_query_string_settings(kwargs) # use dictionary to allow more than the data itself to be returned (data totals, other meta, etc.) - return DataResult( - data=list(dataset.datatype.dataprovider(dataset, provider, **kwargs)) - ) + return DataResult(data=list(dataset.datatype.dataprovider(dataset, provider, **kwargs))) else: raise dataproviders.exceptions.NoProviderAvailable(dataset.datatype, provider) diff --git a/lib/galaxy_test/api/test_histories.py b/lib/galaxy_test/api/test_histories.py index 4d1f08c842b..4b1c70fab5c 100644 --- a/lib/galaxy_test/api/test_histories.py +++ b/lib/galaxy_test/api/test_histories.py @@ -1,8 +1,6 @@ import time -from requests import ( - put -) +from requests import put from galaxy_test.api.sharable import SharingApiTests from galaxy_test.base.populators import ( @@ -14,7 +12,6 @@ from ._framework import ApiTestCase class BaseHistories: - def _show(self, history_id): return self._get(f"histories/{history_id}").json() @@ -32,7 +29,6 @@ class BaseHistories: class HistoriesApiTestCase(ApiTestCase, BaseHistories): - def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) @@ -60,16 +56,23 @@ class HistoriesApiTestCase(ApiTestCase, BaseHistories): history_id = self._create_history("TestHistoryForShow")["id"] show_response = self._show(history_id) self._assert_has_key( - show_response, - 'id', 'name', 'annotation', 'size', 'contents_url', - 'state', 'state_details', 'state_ids' + show_response, "id", "name", "annotation", "size", "contents_url", "state", "state_details", "state_ids" ) state_details = show_response["state_details"] state_ids = show_response["state_ids"] states = [ - 'discarded', 'empty', 'error', 'failed_metadata', 'new', - 'ok', 'paused', 'queued', 'running', 'setting_metadata', 'upload' + "discarded", + "empty", + "error", + "failed_metadata", + "new", + "ok", + "paused", + "queued", + "running", + "setting_metadata", + "upload", ] assert isinstance(state_details, dict) assert isinstance(state_ids, dict) @@ -114,7 +117,7 @@ class HistoriesApiTestCase(ApiTestCase, BaseHistories): def test_purge(self): history_id = self._create_history("TestHistoryForPurge")["id"] - data = {'purge': True} + data = {"purge": True} self._delete(f"histories/{history_id}", data=data, json=True) show_response = self._show(history_id) assert show_response["deleted"] @@ -134,7 +137,7 @@ class HistoriesApiTestCase(ApiTestCase, BaseHistories): show_response = self._show(history_id) assert show_response["name"] == "New Name" - unicode_name = '桜ゲノム' + unicode_name = "桜ゲノム" self._update(history_id, {"name": unicode_name}) show_response = self._show(history_id) assert show_response["name"] == unicode_name, show_response @@ -182,7 +185,7 @@ class HistoriesApiTestCase(ApiTestCase, BaseHistories): for str_key in ["name", "annotation"]: assert self._update(history_id, {str_key: False}).status_code == 400 - for bool_key in ['deleted', 'importable', 'published']: + for bool_key in ["deleted", "importable", "published"]: assert self._update(history_id, {bool_key: "a string"}).status_code == 400 assert self._update(history_id, {"tags": "a simple string"}).status_code == 400 @@ -217,29 +220,32 @@ class HistoriesApiTestCase(ApiTestCase, BaseHistories): def test_copy_history(self): history_id = self.dataset_populator.new_history() - fetch_response = self.dataset_collection_populator.create_list_in_history(history_id, contents=["Hello", "World"], direct_upload=True) + fetch_response = self.dataset_collection_populator.create_list_in_history( + history_id, contents=["Hello", "World"], direct_upload=True + ) dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response.json()) copied_history_response = self.dataset_populator.copy_history(history_id) copied_history_response.raise_for_status() copied_history = copied_history_response.json() - copied_collection = self.dataset_populator.get_history_collection_details(history_id=copied_history['id'], history_content_type="dataset_collection") - assert dataset_collection['name'] == copied_collection['name'] - assert dataset_collection['id'] != copied_collection['id'] - assert len(dataset_collection['elements']) == len(copied_collection['elements']) == 2 - source_element = dataset_collection['elements'][0] - copied_element = copied_collection['elements'][0] - assert source_element['element_identifier'] == copied_element['element_identifier'] == 'data0' - assert source_element['id'] != copied_element['id'] - source_hda = source_element['object'] - copied_hda = copied_element['object'] - assert source_hda['name'] == copied_hda['name'] == 'data0' - assert source_hda['id'] != copied_hda['id'] - assert source_hda['history_id'] != copied_hda['history_id'] - assert source_hda['hid'] == copied_hda['hid'] == 2 + copied_collection = self.dataset_populator.get_history_collection_details( + history_id=copied_history["id"], history_content_type="dataset_collection" + ) + assert dataset_collection["name"] == copied_collection["name"] + assert dataset_collection["id"] != copied_collection["id"] + assert len(dataset_collection["elements"]) == len(copied_collection["elements"]) == 2 + source_element = dataset_collection["elements"][0] + copied_element = copied_collection["elements"][0] + assert source_element["element_identifier"] == copied_element["element_identifier"] == "data0" + assert source_element["id"] != copied_element["id"] + source_hda = source_element["object"] + copied_hda = copied_element["object"] + assert source_hda["name"] == copied_hda["name"] == "data0" + assert source_hda["id"] != copied_hda["id"] + assert source_hda["history_id"] != copied_hda["history_id"] + assert source_hda["hid"] == copied_hda["hid"] == 2 class ImportExportTests(BaseHistories): - def _set_up_populators(self): self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor) @@ -258,7 +264,13 @@ class ImportExportTests(BaseHistories): assert hda["purged"] is True self._check_imported_dataset(history_id=imported_history_id, hid=1, job_checker=upload_job_check) - self._check_imported_dataset(history_id=imported_history_id, hid=2, has_job=False, hda_checker=check_discarded, job_checker=upload_job_check) + self._check_imported_dataset( + history_id=imported_history_id, + hid=2, + has_job=False, + hda_checker=check_discarded, + job_checker=upload_job_check, + ) imported_content = self.dataset_populator.get_history_dataset_content( history_id=imported_history_id, @@ -267,8 +279,8 @@ class ImportExportTests(BaseHistories): assert imported_content == "1 2 3\n" def test_import_1901_histories(self): - f = open(self.test_data_resolver.get_filename("exports/1901_two_datasets.tgz"), 'rb') - import_data = dict(archive_source='', archive_file=f) + f = open(self.test_data_resolver.get_filename("exports/1901_two_datasets.tgz"), "rb") + import_data = dict(archive_source="", archive_file=f) self._import_history_and_wait(import_data, "API Test History", wait_on_history_length=2) def test_import_export_include_deleted(self): @@ -278,7 +290,9 @@ class ImportExportTests(BaseHistories): deleted_hda = self.dataset_populator.new_dataset(history_id, content="1 2 3", wait=True) self.dataset_populator.delete_dataset(history_id, deleted_hda["id"]) - imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=2, export_kwds={"include_deleted": "True"}) + imported_history_id = self._reimport_history( + history_id, history_name, wait_on_history_length=2, export_kwds={"include_deleted": "True"} + ) self._assert_history_length(imported_history_id, 2) def upload_job_check(job): @@ -290,7 +304,9 @@ class ImportExportTests(BaseHistories): assert hda["purged"] is False, hda self._check_imported_dataset(history_id=imported_history_id, hid=1, job_checker=upload_job_check) - self._check_imported_dataset(history_id=imported_history_id, hid=2, hda_checker=check_deleted_not_purged, job_checker=upload_job_check) + self._check_imported_dataset( + history_id=imported_history_id, hid=2, hda_checker=check_deleted_not_purged, job_checker=upload_job_check + ) imported_content = self.dataset_populator.get_history_dataset_content( history_id=imported_history_id, @@ -302,10 +318,12 @@ class ImportExportTests(BaseHistories): def test_import_export_failed_job(self): history_name = "for_export_include_failed_job" history_id = self.dataset_populator.new_history(name=history_name) - self.dataset_populator.run_tool_raw('job_properties', inputs={'failbool': True}, history_id=history_id) + self.dataset_populator.run_tool_raw("job_properties", inputs={"failbool": True}, history_id=history_id) self.dataset_populator.wait_for_history(history_id, assert_ok=False) - imported_history_id = self._reimport_history(history_id, history_name, assert_ok=False, wait_on_history_length=4, export_kwds={"include_deleted": "True"}) + imported_history_id = self._reimport_history( + history_id, history_name, assert_ok=False, wait_on_history_length=4, export_kwds={"include_deleted": "True"} + ) self._assert_history_length(imported_history_id, 4) def check_failed(hda_or_job): @@ -314,12 +332,16 @@ class ImportExportTests(BaseHistories): self.dataset_populator._summarize_history(imported_history_id) - self._check_imported_dataset(history_id=imported_history_id, hid=1, assert_ok=False, hda_checker=check_failed, job_checker=check_failed) + self._check_imported_dataset( + history_id=imported_history_id, hid=1, assert_ok=False, hda_checker=check_failed, job_checker=check_failed + ) def test_import_metadata_regeneration(self): history_name = "for_import_metadata_regeneration" history_id = self.dataset_populator.new_history(name=history_name) - self.dataset_populator.new_dataset(history_id, content=open(self.test_data_resolver.get_filename("1.bam"), 'rb'), file_type='bam', wait=True) + self.dataset_populator.new_dataset( + history_id, content=open(self.test_data_resolver.get_filename("1.bam"), "rb"), file_type="bam", wait=True + ) imported_history_id = self._reimport_history(history_id, history_name) self._assert_history_length(imported_history_id, 1) self._check_imported_dataset(history_id=imported_history_id, hid=1) @@ -345,7 +367,9 @@ class ImportExportTests(BaseHistories): def test_import_export_collection(self): history_name = "for_export_with_collections" history_id = self.dataset_populator.new_history(name=history_name) - self.dataset_collection_populator.create_list_in_history(history_id, contents=["Hello", "World"], direct_upload=True) + self.dataset_collection_populator.create_list_in_history( + history_id, contents=["Hello", "World"], direct_upload=True + ) imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=3) self._assert_history_length(imported_history_id, 3) @@ -362,7 +386,9 @@ class ImportExportTests(BaseHistories): assert element0["hid"] == 2 assert element1["hid"] == 3 - self._check_imported_collection(imported_history_id, hid=1, collection_type="list", elements_checker=check_elements) + self._check_imported_collection( + imported_history_id, hid=1, collection_type="list", elements_checker=check_elements + ) def test_import_export_nested_collection(self): history_name = "for_export_with_nested_collections" @@ -382,15 +408,22 @@ class ImportExportTests(BaseHistories): assert len(child_elements) == 2 assert element0["collection_type"] == "paired" - self._check_imported_collection(imported_history_id, hid=1, collection_type="list:paired", elements_checker=check_elements) + self._check_imported_collection( + imported_history_id, hid=1, collection_type="list:paired", elements_checker=check_elements + ) - def _reimport_history(self, history_id, history_name, wait_on_history_length=None, assert_ok=True, export_kwds=None): + def _reimport_history( + self, history_id, history_name, wait_on_history_length=None, assert_ok=True, export_kwds=None + ): # Ensure the history is ready to go... export_kwds = export_kwds or {} self.dataset_populator.wait_for_history(history_id, assert_ok=assert_ok) return self.dataset_populator.reimport_history( - history_id, history_name, wait_on_history_length=wait_on_history_length, export_kwds=export_kwds, + history_id, + history_name, + wait_on_history_length=wait_on_history_length, + export_kwds=export_kwds, ) def _import_history_and_wait(self, import_data, history_name, wait_on_history_length=None): @@ -408,7 +441,9 @@ class ImportExportTests(BaseHistories): contents = contents_response.json() assert len(contents) == n, contents - def _check_imported_dataset(self, history_id, hid, assert_ok=True, has_job=True, hda_checker=None, job_checker=None): + def _check_imported_dataset( + self, history_id, hid, assert_ok=True, has_job=True, hda_checker=None, job_checker=None + ): imported_dataset_metadata = self.dataset_populator.get_history_dataset_details( history_id=history_id, hid=hid, @@ -428,8 +463,8 @@ class ImportExportTests(BaseHistories): job_details = self.dataset_populator.get_job_details(job_id, full=True) assert job_details.status_code == 200, job_details.content job = job_details.json() - assert 'history_id' in job, job - assert job['history_id'] == history_id, job + assert "history_id" in job, job + assert job["history_id"] == history_id, job if job_checker is not None: job_checker(job) @@ -451,7 +486,6 @@ class ImportExportTests(BaseHistories): class ImportExportHistoryTestCase(ApiTestCase, ImportExportTests): - def setUp(self): super().setUp() self._set_up_populators() @@ -495,10 +529,7 @@ class SharingHistoryTestCase(ApiTestCase, BaseHistories, SharingApiTests): assert not sharing_response["users_shared_with"] # Now we provide the share_option - payload = { - "user_ids": [target_user_id], - "share_option": "make_accessible_to_shared" - } + payload = {"user_ids": [target_user_id], "share_option": "make_accessible_to_shared"} sharing_response = self._share_history_with_payload(history_id, payload) assert sharing_response["users_shared_with"] assert sharing_response["users_shared_with"][0]["id"] == target_user_id @@ -531,10 +562,7 @@ class SharingHistoryTestCase(ApiTestCase, BaseHistories, SharingApiTests): # Trying to change the permissions when sharing should fail # because we don't have manage permissions - payload = { - "user_ids": [target_user_id], - "share_option": "make_public" - } + payload = {"user_ids": [target_user_id], "share_option": "make_public"} sharing_response = self._share_history_with_payload(history_id, payload) assert sharing_response["extra"] assert sharing_response["extra"]["can_share"] is False @@ -542,10 +570,7 @@ class SharingHistoryTestCase(ApiTestCase, BaseHistories, SharingApiTests): assert not sharing_response["users_shared_with"] # we can share if we don't try to make any permission changes - payload = { - "user_ids": [target_user_id], - "share_option": "no_changes" - } + payload = {"user_ids": [target_user_id], "share_option": "no_changes"} sharing_response = self._share_history_with_payload(history_id, payload) assert not sharing_response["errors"] assert sharing_response["users_shared_with"] diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py index 6cc73032a39..29606cfefe1 100644 --- a/lib/galaxy_test/api/test_history_contents.py +++ b/lib/galaxy_test/api/test_history_contents.py @@ -18,7 +18,6 @@ from ._framework import ApiTestCase # TODO: Test anonymous access. class HistoryContentsApiTestCase(ApiTestCase): - def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) @@ -115,7 +114,9 @@ class HistoryContentsApiTestCase(ApiTestCase): user_id = self.dataset_populator.user_id() with self._different_user(): different_user_id = self.dataset_populator.user_id() - combined_user_role = self.dataset_populator.create_role([user_id, different_user_id], description="role for testing permissions") + combined_user_role = self.dataset_populator.create_role( + [user_id, different_user_id], description="role for testing permissions" + ) payload = { "access": [combined_user_role["id"]], @@ -166,7 +167,7 @@ class HistoryContentsApiTestCase(ApiTestCase): def _create_copy(self): hda1 = self.dataset_populator.new_dataset(self.history_id) create_data = dict( - source='hda', + source="hda", content=hda1["id"], ) second_history_id = self.dataset_populator.new_history() @@ -177,7 +178,7 @@ class HistoryContentsApiTestCase(ApiTestCase): def test_hda_copy(self): response = self._create_copy() - assert self.__count_contents(response['history_id']) == 1 + assert self.__count_contents(response["history_id"]) == 1 def test_inheritance_chain(self): response = self._create_copy() @@ -189,7 +190,7 @@ class HistoryContentsApiTestCase(ApiTestCase): def test_library_copy(self): ld = self.library_populator.new_library_dataset("lda_test_library") create_data = dict( - source='library', + source="library", content=ld["id"], ) assert self.__count_contents(self.history_id) == 0 @@ -211,7 +212,7 @@ class HistoryContentsApiTestCase(ApiTestCase): update_response = self._update(hda1["id"], dict(name="Updated Name")) assert self.__show(hda1).json()["name"] == "Updated Name" - unicode_name = 'ржевский сапоги' + unicode_name = "ржевский сапоги" update_response = self._update(hda1["id"], dict(name=unicode_name)) updated_hda = self.__show(hda1).json() assert updated_hda["name"] == unicode_name, updated_hda @@ -227,7 +228,7 @@ class HistoryContentsApiTestCase(ApiTestCase): "dbkey": "?", "annotation": None, "info": "my info is", - "operation": "attributes" + "operation": "attributes", } update_response = self._set_edit_update(data) # No key or anything supplied, expect a permission problem. @@ -282,7 +283,9 @@ class HistoryContentsApiTestCase(ApiTestCase): assert objects[0]["visible"] is False # update both flags - payload = dict(items=[{"history_content_type": "dataset_collection", "id": hdca["id"]}], deleted=False, visible=True) + payload = dict( + items=[{"history_content_type": "dataset_collection", "id": hdca["id"]}], deleted=False, visible=True + ) update_response = self._update_batch(payload) objects = update_response.json() assert objects[0]["deleted"] is False @@ -290,7 +293,7 @@ class HistoryContentsApiTestCase(ApiTestCase): def test_update_type_failures(self): hda1 = self._wait_for_new_hda() - update_response = self._update(hda1["id"], dict(deleted='not valid')) + update_response = self._update(hda1["id"], dict(deleted="not valid")) self._assert_status_code_is(update_response, 400) def _wait_for_new_hda(self): @@ -325,7 +328,7 @@ class HistoryContentsApiTestCase(ApiTestCase): def test_delete_anon(self): with self._different_user(anon=True): - history_id = self._get(urllib.parse.urljoin(self.url, "history/current_history_json")).json()['id'] + history_id = self._get(urllib.parse.urljoin(self.url, "history/current_history_json")).json()["id"] hda1 = self.dataset_populator.new_dataset(history_id) self.dataset_populator.wait_for_history(history_id) assert str(self.__show(hda1).json()["deleted"]).lower() == "false" @@ -338,24 +341,21 @@ class HistoryContentsApiTestCase(ApiTestCase): with self._different_user(anon=True): delete_response = self._delete(f"histories/{self.history_id}/contents/{hda1['id']}") assert delete_response.status_code == 403 - assert delete_response.json()['err_msg'] == 'HistoryDatasetAssociation is not owned by user' + assert delete_response.json()["err_msg"] == "HistoryDatasetAssociation is not owned by user" def test_purge(self): hda1 = self.dataset_populator.new_dataset(self.history_id) self.dataset_populator.wait_for_history(self.history_id) assert str(self.__show(hda1).json()["deleted"]).lower() == "false" assert str(self.__show(hda1).json()["purged"]).lower() == "false" - data = {'purge': True} + data = {"purge": True} delete_response = self._delete(f"histories/{self.history_id}/contents/{hda1['id']}", data=data, json=True) assert delete_response.status_code < 300 # Something in the 200s :). assert str(self.__show(hda1).json()["deleted"]).lower() == "true" assert str(self.__show(hda1).json()["purged"]).lower() == "true" def test_dataset_collection_creation_on_contents(self): - payload = self.dataset_collection_populator.create_pair_payload( - self.history_id, - type="dataset_collection" - ) + payload = self.dataset_collection_populator.create_pair_payload(self.history_id, type="dataset_collection") endpoint = f"histories/{self.history_id}/contents" self._check_pair_creation(endpoint, payload) @@ -368,30 +368,27 @@ class HistoryContentsApiTestCase(ApiTestCase): def test_dataset_collection_create_from_exisiting_datasets_with_new_tags(self): with self.dataset_populator.test_history() as history_id: - hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id'] - hda2_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id'] - update_response = self._update(hda2_id, dict(tags=['existing:tag']), history_id=history_id).json() - assert update_response['tags'] == ['existing:tag'] - creation_payload = {'collection_type': 'list', - 'history_id': history_id, - 'element_identifiers': [{'id': hda_id, - 'src': 'hda', - 'name': 'element_id1', - 'tags': ['my_new_tag']}, - {'id': hda2_id, - 'src': 'hda', - 'name': 'element_id2', - 'tags': ['another_new_tag']} - ], - 'type': 'dataset_collection', - 'copy_elements': True} + hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")["id"] + hda2_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")["id"] + update_response = self._update(hda2_id, dict(tags=["existing:tag"]), history_id=history_id).json() + assert update_response["tags"] == ["existing:tag"] + creation_payload = { + "collection_type": "list", + "history_id": history_id, + "element_identifiers": [ + {"id": hda_id, "src": "hda", "name": "element_id1", "tags": ["my_new_tag"]}, + {"id": hda2_id, "src": "hda", "name": "element_id2", "tags": ["another_new_tag"]}, + ], + "type": "dataset_collection", + "copy_elements": True, + } r = self._post(f"histories/{self.history_id}/contents", creation_payload, json=True).json() - assert r['elements'][0]['object']['id'] != hda_id, "HDA has not been copied" - assert len(r['elements'][0]['object']['tags']) == 1 - assert r['elements'][0]['object']['tags'][0] == 'my_new_tag' - assert len(r['elements'][1]['object']['tags']) == 2, r['elements'][1]['object']['tags'] + assert r["elements"][0]["object"]["id"] != hda_id, "HDA has not been copied" + assert len(r["elements"][0]["object"]["tags"]) == 1 + assert r["elements"][0]["object"]["tags"][0] == "my_new_tag" + assert len(r["elements"][1]["object"]["tags"]) == 2, r["elements"][1]["object"]["tags"] original_hda = self.dataset_populator.get_history_dataset_details(history_id=history_id, dataset_id=hda_id) - assert len(original_hda['tags']) == 0, original_hda['tags'] + assert len(original_hda["tags"]) == 0, original_hda["tags"] def _check_pair_creation(self, endpoint, payload): pre_collection_count = self.__count_contents(type="dataset_collection") @@ -431,10 +428,12 @@ class HistoryContentsApiTestCase(ApiTestCase): @skip_without_tool("collection_creates_list") def test_jobs_summary_simple_hdca(self): - create_response = self.dataset_collection_populator.create_list_in_history(self.history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"]) + create_response = self.dataset_collection_populator.create_list_in_history( + self.history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"] + ) hdca_id = create_response.json()["id"] run = self.dataset_populator.run_collection_creates_list(self.history_id, hdca_id) - collections = run['output_collections'] + collections = run["output_collections"] collection = collections[0] jobs_summary_url = f"histories/{self.history_id}/contents/dataset_collections/{collection['id']}/jobs_summary" jobs_summary_response = self._get(jobs_summary_url) @@ -444,14 +443,16 @@ class HistoryContentsApiTestCase(ApiTestCase): @skip_without_tool("cat1") def test_jobs_summary_implicit_hdca(self): - create_response = self.dataset_collection_populator.create_pair_in_history(self.history_id, contents=["123", "456"]) + create_response = self.dataset_collection_populator.create_pair_in_history( + self.history_id, contents=["123", "456"] + ) hdca_id = create_response.json()["id"] inputs = { - "input1": {'batch': True, 'values': [{'src': 'hdca', 'id': hdca_id}]}, + "input1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}, } run = self.dataset_populator.run_tool("cat1", inputs=inputs, history_id=self.history_id) self.dataset_populator.wait_for_history_jobs(self.history_id) - collections = run['implicit_collections'] + collections = run["implicit_collections"] collection = collections[0] jobs_summary_url = f"histories/{self.history_id}/contents/dataset_collections/{collection['id']}/jobs_summary" jobs_summary_response = self._get(jobs_summary_url) @@ -462,17 +463,16 @@ class HistoryContentsApiTestCase(ApiTestCase): assert states.get("ok") == 2, states def test_dataset_collection_hide_originals(self): - payload = self.dataset_collection_populator.create_pair_payload( - self.history_id, - type="dataset_collection" - ) + payload = self.dataset_collection_populator.create_pair_payload(self.history_id, type="dataset_collection") payload["hide_source_items"] = True dataset_collection_response = self._post(f"histories/{self.history_id}/contents", payload, json=True) self.__check_create_collection_response(dataset_collection_response) contents_response = self._get(f"histories/{self.history_id}/contents") - datasets = [d for d in contents_response.json() if d["history_content_type"] == "dataset" and d["hid"] in [1, 2]] + datasets = [ + d for d in contents_response.json() if d["history_content_type"] == "dataset" and d["hid"] in [1, 2] + ] # Assert two datasets in source were hidden. assert len(datasets) == 2 assert not datasets[0]["visible"] @@ -490,23 +490,14 @@ class HistoryContentsApiTestCase(ApiTestCase): def test_update_batch_dataset_collection(self): hdca = self._create_pair_collection() - body = { - "items": [{ - "history_content_type": "dataset_collection", - "id": hdca["id"] - }], - "name": "newnameforpair" - } + body = {"items": [{"history_content_type": "dataset_collection", "id": hdca["id"]}], "name": "newnameforpair"} update_response = self._put(f"histories/{self.history_id}/contents", data=body, json=True) self._assert_status_code_is(update_response, 200) show_response = self.__show(hdca) assert str(show_response.json()["name"]) == "newnameforpair" def _create_pair_collection(self): - payload = self.dataset_collection_populator.create_pair_payload( - self.history_id, - type="dataset_collection" - ) + payload = self.dataset_collection_populator.create_pair_payload(self.history_id, type="dataset_collection") dataset_collection_response = self._post(f"histories/{self.history_id}/contents", payload, json=True) self._assert_status_code_is(dataset_collection_response, 200) hdca = dataset_collection_response.json() @@ -517,11 +508,13 @@ class HistoryContentsApiTestCase(ApiTestCase): hdca_id = hdca["id"] second_history_id = self.dataset_populator.new_history() create_data = dict( - source='hdca', + source="hdca", content=hdca_id, ) assert len(self._get(f"histories/{second_history_id}/contents/dataset_collections").json()) == 0 - create_response = self._post(f"histories/{second_history_id}/contents/dataset_collections", create_data, json=True) + create_response = self._post( + f"histories/{second_history_id}/contents/dataset_collections", create_data, json=True + ) self.__check_create_collection_response(create_response) contents = self._get(f"histories/{second_history_id}/contents/dataset_collections").json() assert len(contents) == 1 @@ -534,10 +527,12 @@ class HistoryContentsApiTestCase(ApiTestCase): hdca_id = hdca["id"] assert hdca["elements"][0]["object"]["metadata_dbkey"] == "?" assert hdca["elements"][0]["object"]["genome_build"] == "?" - create_data = {'source': 'hdca', 'content': hdca_id, 'dbkey': 'hg19'} - create_response = self._post(f"histories/{self.history_id}/contents/dataset_collections", create_data, json=True) + create_data = {"source": "hdca", "content": hdca_id, "dbkey": "hg19"} + create_response = self._post( + f"histories/{self.history_id}/contents/dataset_collections", create_data, json=True + ) collection = self.__check_create_collection_response(create_response) - new_forward = collection['elements'][0]['object'] + new_forward = collection["elements"][0]["object"] assert new_forward["metadata_dbkey"] == "hg19" assert new_forward["genome_build"] == "hg19" @@ -546,12 +541,14 @@ class HistoryContentsApiTestCase(ApiTestCase): hdca_id = hdca["id"] second_history_id = self.dataset_populator.new_history() create_data = dict( - source='hdca', + source="hdca", content=hdca_id, copy_elements=True, ) assert len(self._get(f"histories/{second_history_id}/contents/dataset_collections").json()) == 0 - create_response = self._post(f"histories/{second_history_id}/contents/dataset_collections", create_data, json=True) + create_response = self._post( + f"histories/{second_history_id}/contents/dataset_collections", create_data, json=True + ) self.__check_create_collection_response(create_response) contents = self._get(f"histories/{second_history_id}/contents/dataset_collections").json() @@ -592,10 +589,12 @@ class HistoryContentsApiTestCase(ApiTestCase): assert hda["hda_ldda"] == "hda" assert hda["history_content_type"] == "dataset" assert hda["copied_from_ldda_id"] == ldda_id - assert hda['history_id'] == history_id + assert hda["history_id"] == history_id def test_hdca_from_inaccessible_library_datasets(self): - library, library_dataset = self.library_populator.new_library_dataset_in_private_library("HDCACreateInaccesibleLibrary") + library, library_dataset = self.library_populator.new_library_dataset_in_private_library( + "HDCACreateInaccesibleLibrary" + ) ldda_id = library_dataset["id"] element_identifiers = [{"name": "el1", "src": "ldda", "id": ldda_id}] create_data = dict( @@ -607,7 +606,9 @@ class HistoryContentsApiTestCase(ApiTestCase): ) with self._different_user(): second_history_id = self.dataset_populator.new_history() - create_response = self._post(f"histories/{second_history_id}/contents/dataset_collections", create_data, json=True) + create_response = self._post( + f"histories/{second_history_id}/contents/dataset_collections", create_data, json=True + ) self._assert_status_code_is(create_response, 403) def __check_create_collection_response(self, response): @@ -617,7 +618,9 @@ class HistoryContentsApiTestCase(ApiTestCase): return dataset_collection def __show(self, contents): - show_response = self._get(f"histories/{self.history_id}/contents/{contents['history_content_type']}s/{contents['id']}") + show_response = self._get( + f"histories/{self.history_id}/contents/{contents['history_content_type']}s/{contents['id']}" + ) return show_response def __count_contents(self, history_id=None, **kwds): @@ -643,15 +646,17 @@ class HistoryContentsApiTestCase(ApiTestCase): assert input_hda["id"] == query_hda["id"] def test_job_state_summary_field(self): - create_response = self.dataset_collection_populator.create_pair_in_history(self.history_id, contents=["123", "456"]) + create_response = self.dataset_collection_populator.create_pair_in_history( + self.history_id, contents=["123", "456"] + ) self._assert_status_code_is(create_response, 200) contents_response = self._get(f"histories/{self.history_id}/contents?v=dev&keys=job_state_summary&view=summary") self._assert_status_code_is(contents_response, 200) contents = contents_response.json() - for c in filter(lambda c: c['history_content_type'] == 'dataset_collection', contents): + for c in filter(lambda c: c["history_content_type"] == "dataset_collection", contents): assert isinstance(c, dict) - assert 'job_state_summary' in c - assert isinstance(c['job_state_summary'], dict) + assert "job_state_summary" in c + assert isinstance(c["job_state_summary"], dict) def _get_content(self, history_id, update_time): return self._get(f"/api/histories/{history_id}/contents/near/100/100?update_time-gt={update_time}").json() @@ -669,7 +674,7 @@ class HistoryContentsApiTestCase(ApiTestCase): def test_history_contents_near_with_since(self): with self.dataset_populator.test_history() as history_id: original_history = self._get(f"/api/histories/{history_id}").json() - original_history_stamp = original_history['update_time'] + original_history_stamp = original_history["update_time"] # check empty contents, with no since flag, should return an empty 200 result history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100") @@ -677,7 +682,9 @@ class HistoryContentsApiTestCase(ApiTestCase): assert len(history_contents.json()) == 0 # adding a since parameter, should return a 204 if history has not changed at all - history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={original_history_stamp}") + history_contents = self._get( + f"/api/histories/{history_id}/contents/near/100/100?since={original_history_stamp}" + ) assert history_contents.status_code == 204 # add some stuff @@ -691,19 +698,21 @@ class HistoryContentsApiTestCase(ApiTestCase): # check to make sure the history date has actually changed due to changing the contents changed_history = self._get(f"/api/histories/{history_id}").json() - changed_history_stamp = changed_history['update_time'] + changed_history_stamp = changed_history["update_time"] assert original_history_stamp != changed_history_stamp # a repeated contents request with since=original_history_stamp should now return data # because we have added datasets and the update_time should have been changed - changed_content = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={original_history_stamp}") + changed_content = self._get( + f"/api/histories/{history_id}/contents/near/100/100?since={original_history_stamp}" + ) assert changed_content.status_code == 200 assert len(changed_content.json()) == 4 def test_history_contents_near_since_with_standard_iso8601_date(self): with self.dataset_populator.test_history() as history_id: original_history = self._get(f"/api/histories/{history_id}").json() - original_history_stamp = original_history['update_time'] + original_history_stamp = original_history["update_time"] # this is the standard date format that javascript will emit using .toISOString(), it # should be the expected date format for any modern api @@ -711,25 +720,27 @@ class HistoryContentsApiTestCase(ApiTestCase): # checking to make sure that the same exact history.update_time returns a "not changed" # result after date parsing - valid_iso8601_date = original_history_stamp + 'Z' + valid_iso8601_date = original_history_stamp + "Z" encoded_valid_date = urllib.parse.quote_plus(valid_iso8601_date) - history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={encoded_valid_date}") + history_contents = self._get( + f"/api/histories/{history_id}/contents/near/100/100?since={encoded_valid_date}" + ) assert history_contents.status_code == 204 # test parsing for other standard is08601 formats - sample_formats = ['2021-08-26T15:53:02+00:00', '2021-08-26T15:53:02Z', '2002-10-10T12:00:00-05:00'] + sample_formats = ["2021-08-26T15:53:02+00:00", "2021-08-26T15:53:02Z", "2002-10-10T12:00:00-05:00"] for date_str in sample_formats: encoded_date = urllib.parse.quote_plus(date_str) # handles pluses, minuses history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={encoded_date}") self._assert_status_code_is_ok(history_contents) - @skip_without_tool('cat_data_and_sleep') + @skip_without_tool("cat_data_and_sleep") def test_history_contents_near_with_update_time_implicit_collection(self): with self.dataset_populator.test_history() as history_id: - hdca_id = self.dataset_collection_populator.create_list_in_history(history_id=history_id).json()['id'] + hdca_id = self.dataset_collection_populator.create_list_in_history(history_id=history_id).json()["id"] self.dataset_populator.wait_for_history(history_id) inputs = { - "input1": {'batch': True, 'values': [{"src": "hdca", "id": hdca_id}]}, + "input1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}, "sleep_time": 2, } response = self.dataset_populator.run_tool( @@ -738,31 +749,43 @@ class HistoryContentsApiTestCase(ApiTestCase): history_id, ) update_time = datetime.utcnow().isoformat() - collection_id = response['implicit_collections'][0]['id'] + collection_id = response["implicit_collections"][0]["id"] for _ in range(20): time.sleep(1) update = self._get_content(history_id, update_time=update_time) - if any(c for c in update if c['history_content_type'] == 'dataset_collection' and c['job_state_summary']['ok'] == 3): + if any( + c + for c in update + if c["history_content_type"] == "dataset_collection" and c["job_state_summary"]["ok"] == 3 + ): return - raise Exception(f"History content update time query did not include final update for implicit collection {collection_id}") + raise Exception( + f"History content update time query did not include final update for implicit collection {collection_id}" + ) - @skip_without_tool('collection_creates_dynamic_nested') + @skip_without_tool("collection_creates_dynamic_nested") def test_history_contents_near_with_update_time_explicit_collection(self): with self.dataset_populator.test_history() as history_id: - inputs = {'foo': 'bar', 'sleep_time': 2} + inputs = {"foo": "bar", "sleep_time": 2} response = self.dataset_populator.run_tool( "collection_creates_dynamic_nested", inputs, history_id, ) update_time = datetime.utcnow().isoformat() - collection_id = response['output_collections'][0]['id'] + collection_id = response["output_collections"][0]["id"] for _ in range(20): time.sleep(1) update = self._get_content(history_id, update_time=update_time) - if any(c for c in update if c['history_content_type'] == 'dataset_collection' and c['populated_state'] == 'ok'): + if any( + c + for c in update + if c["history_content_type"] == "dataset_collection" and c["populated_state"] == "ok" + ): return - raise Exception(f"History content update time query did not include populated_state update for dynamic nested collection {collection_id}") + raise Exception( + f"History content update time query did not include populated_state update for dynamic nested collection {collection_id}" + ) def test_index_filter_by_type(self): history_id = self.dataset_populator.new_history() @@ -806,7 +829,9 @@ class HistoryContentsApiTestCase(ApiTestCase): self._assert_status_code_is_ok(create_homogeneous_response) def _assert_collection_has_expected_elements_datatypes(self, history_id, collection_name, expected_datatypes): - contents_response = self._get(f"histories/{history_id}/contents?v=dev&view=betawebclient&q=name-eq&qv={collection_name}") + contents_response = self._get( + f"histories/{history_id}/contents?v=dev&view=betawebclient&q=name-eq&qv={collection_name}" + ) self._assert_status_code_is(contents_response, 200) collection = contents_response.json()[0] self.assertCountEqual(collection["elements_datatypes"], expected_datatypes) @@ -816,6 +841,7 @@ class HistoryContentsApiNearTestCase(ApiTestCase): """ Test the /api/histories/{history_id}/contents/{direction}/{hid}/{limit} endpoint. """ + NEAR = DirectionOptions.near BEFORE = DirectionOptions.before AFTER = DirectionOptions.after @@ -838,87 +864,87 @@ class HistoryContentsApiNearTestCase(ApiTestCase): self._create_list_in_history(history_id) result = self._get_content(history_id, self.NEAR, hid=1) assert len(result) == 8 - assert result[0]['hid'] == 8 - assert result[1]['hid'] == 7 - assert result[2]['hid'] == 6 - assert result[3]['hid'] == 5 - assert result[4]['hid'] == 4 - assert result[5]['hid'] == 3 - assert result[6]['hid'] == 2 - assert result[7]['hid'] == 1 + assert result[0]["hid"] == 8 + assert result[1]["hid"] == 7 + assert result[2]["hid"] == 6 + assert result[3]["hid"] == 5 + assert result[4]["hid"] == 4 + assert result[5]["hid"] == 3 + assert result[6]["hid"] == 2 + assert result[7]["hid"] == 1 def test_near_even_limit(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.NEAR, hid=5, limit=3) assert len(result) == 3 - assert result[0]['hid'] == 6 # hid + 1 - assert result[1]['hid'] == 5 # hid - assert result[2]['hid'] == 4 # hid - 1 + assert result[0]["hid"] == 6 # hid + 1 + assert result[1]["hid"] == 5 # hid + assert result[2]["hid"] == 4 # hid - 1 def test_near_odd_limit(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.NEAR, hid=5, limit=4) assert len(result) == 4 - assert result[0]['hid'] == 7 # hid + 2 - assert result[1]['hid'] == 6 # hid + 1 - assert result[2]['hid'] == 5 # hid - assert result[3]['hid'] == 4 # hid - 1 + assert result[0]["hid"] == 7 # hid + 2 + assert result[1]["hid"] == 6 # hid + 1 + assert result[2]["hid"] == 5 # hid + assert result[3]["hid"] == 4 # hid - 1 def test_near_less_than_before_limit(self): # n before < limit // 2 with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.NEAR, hid=1, limit=3) assert len(result) == 2 - assert result[0]['hid'] == 2 # hid + 1 - assert result[1]['hid'] == 1 # hid (there's nothing before hid=1) + assert result[0]["hid"] == 2 # hid + 1 + assert result[1]["hid"] == 1 # hid (there's nothing before hid=1) def test_near_less_than_after_limit(self): # n after < limit // 2 + 1 with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.NEAR, hid=8, limit=3) assert len(result) == 2 - assert result[0]['hid'] == 8 # hid (there's nothing after hid=8) - assert result[1]['hid'] == 7 # hid - 1 + assert result[0]["hid"] == 8 # hid (there's nothing after hid=8) + assert result[1]["hid"] == 7 # hid - 1 def test_near_less_than_before_and_after_limit(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id, n=1) result = self._get_content(history_id, self.NEAR, hid=2, limit=10) assert len(result) == 4 - assert result[0]['hid'] == 4 # hid + 2 (can't go after hid=4) - assert result[1]['hid'] == 3 # hid + 1 - assert result[2]['hid'] == 2 # hid - assert result[3]['hid'] == 1 # hid - 1 (can't go before hid=1) + assert result[0]["hid"] == 4 # hid + 2 (can't go after hid=4) + assert result[1]["hid"] == 3 # hid + 1 + assert result[2]["hid"] == 2 # hid + assert result[3]["hid"] == 1 # hid - 1 (can't go before hid=1) def test_before(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.BEFORE, hid=5, limit=3) assert len(result) == 3 - assert result[0]['hid'] == 4 # hid - 1 - assert result[1]['hid'] == 3 # hid - 2 - assert result[2]['hid'] == 2 # hid - 3 + assert result[0]["hid"] == 4 # hid - 1 + assert result[1]["hid"] == 3 # hid - 2 + assert result[2]["hid"] == 2 # hid - 3 def test_before_less_than_limit(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.BEFORE, hid=2, limit=3) assert len(result) == 1 - assert result[0]['hid'] == 1 # hid - 1 + assert result[0]["hid"] == 1 # hid - 1 def test_after(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.AFTER, hid=5, limit=2) assert len(result) == 2 - assert result[0]['hid'] == 7 # hid + 2 (hid + 3 not included: tests reversed order) - assert result[1]['hid'] == 6 # hid + 1 + assert result[0]["hid"] == 7 # hid + 2 (hid + 3 not included: tests reversed order) + assert result[1]["hid"] == 6 # hid + 1 def test_after_less_than_limit(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.AFTER, hid=7, limit=3) assert len(result) == 1 - assert result[0]['hid'] == 8 # hid + 1 + assert result[0]["hid"] == 8 # hid + 1 diff --git a/lib/galaxy_test/api/test_workflow_extraction.py b/lib/galaxy_test/api/test_workflow_extraction.py index 7d509c0d9e5..4f78005d1b2 100644 --- a/lib/galaxy_test/api/test_workflow_extraction.py +++ b/lib/galaxy_test/api/test_workflow_extraction.py @@ -1,9 +1,15 @@ import functools import operator from collections import namedtuple -from json import dumps, loads +from json import ( + dumps, + loads, +) -from galaxy_test.base.populators import skip_without_tool, summarize_instance_history_on_error +from galaxy_test.base.populators import ( + skip_without_tool, + summarize_instance_history_on_error, +) from .test_workflows import BaseWorkflowsApiTestCase @@ -46,7 +52,7 @@ class WorkflowExtractionApiTestCase(BaseWorkflowsApiTestCase): for old_dataset in old_contents: self.__copy_content_to_history(self.history_id, old_dataset) new_contents = self._history_contents() - input_hids = [c["hid"] for c in new_contents[(offset + 0):(offset + 2)]] + input_hids = [c["hid"] for c in new_contents[(offset + 0) : (offset + 2)]] cat1_job_id = self.__job_id(self.history_id, new_contents[(offset + 2)]["id"]) def reimport_jobs_ids(new_history_id): @@ -69,7 +75,7 @@ class WorkflowExtractionApiTestCase(BaseWorkflowsApiTestCase): for old_dataset in old_contents: self.__copy_content_to_history(self.history_id, old_dataset) new_contents = self._history_contents() - input_hids = [c["hid"] for c in new_contents[(offset + 0):(offset + 2)]] + input_hids = [c["hid"] for c in new_contents[(offset + 0) : (offset + 2)]] def reimport_jobs_ids(new_history_id): return [j["id"] for j in self.dataset_populator.history_jobs(new_history_id) if j["tool_id"] == "cat1"] @@ -110,7 +116,10 @@ class WorkflowExtractionApiTestCase(BaseWorkflowsApiTestCase): def test_extract_copied_mapping_from_history_reimported(self): import unittest - raise unittest.SkipTest("Mapping connection for copied collections not yet implemented in history import/export") + + raise unittest.SkipTest( + "Mapping connection for copied collections not yet implemented in history import/export" + ) old_history_id = self.dataset_populator.new_history() hdca, job_id1, job_id2 = self.__run_random_lines_mapped_over_singleton(old_history_id) @@ -120,7 +129,9 @@ class WorkflowExtractionApiTestCase(BaseWorkflowsApiTestCase): self.__copy_content_to_history(self.history_id, old_content) def reimport_jobs_ids(new_history_id): - rval = [j["id"] for j in self.dataset_populator.history_jobs(new_history_id) if j["tool_id"] == "random_lines1"] + rval = [ + j["id"] for j in self.dataset_populator.history_jobs(new_history_id) if j["tool_id"] == "random_lines1" + ] assert len(rval) == 2 print(rval) return rval @@ -139,12 +150,11 @@ class WorkflowExtractionApiTestCase(BaseWorkflowsApiTestCase): @skip_without_tool("random_lines1") @skip_without_tool("multi_data_param") def test_extract_reduction_from_history(self): - hdca = self.dataset_collection_populator.create_pair_in_history(self.history_id, contents=["1 2 3\n4 5 6", "7 8 9\n10 11 10"]).json() + hdca = self.dataset_collection_populator.create_pair_in_history( + self.history_id, contents=["1 2 3\n4 5 6", "7 8 9\n10 11 10"] + ).json() hdca_id = hdca["id"] - inputs1 = { - "input": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}, - "num_lines": 2 - } + inputs1 = {"input": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}, "num_lines": 2} implicit_hdca1, job_id1 = self._run_tool_get_collection_and_job_id(self.history_id, "random_lines1", inputs1) inputs2 = { "f1": {"src": "hdca", "id": implicit_hdca1["id"]}, @@ -180,7 +190,8 @@ class WorkflowExtractionApiTestCase(BaseWorkflowsApiTestCase): @skip_without_tool("collection_paired_test") def test_extract_workflows_with_dataset_collections(self): - jobs_summary = self._run_workflow(""" + jobs_summary = self._run_workflow( + """ class: GalaxyWorkflow steps: - label: text_input1 @@ -192,7 +203,8 @@ steps: test_data: text_input1: collection_type: paired -""") +""" + ) job_id = self._job_id_for_tool(jobs_summary.jobs, "collection_paired_test") downloaded_workflow = self._extract_and_download_workflow( reimport_as="extract_from_history_with_basic_collections", @@ -205,7 +217,7 @@ test_data: verify_connected=True, data_input_count=0, data_collection_input_count=1, - tool_ids=["collection_paired_test"] + tool_ids=["collection_paired_test"], ) collection_step = self._get_steps_of_type(downloaded_workflow, "data_collection_input", expected_len=1)[0] @@ -214,7 +226,8 @@ test_data: @skip_without_tool("cat_collection") def test_subcollection_mapping(self): - jobs_summary = self._run_workflow(""" + jobs_summary = self._run_workflow( + """ class: GalaxyWorkflow steps: - label: text_input1 @@ -231,7 +244,8 @@ steps: test_data: text_input1: collection_type: "list:paired" - """) + """ + ) job1_id = self._job_id_for_tool(jobs_summary.jobs, "cat1") job2_id = self._job_id_for_tool(jobs_summary.jobs, "cat_collection") downloaded_workflow = self._extract_and_download_workflow( @@ -255,7 +269,8 @@ test_data: @skip_without_tool("cat_list") @skip_without_tool("collection_creates_dynamic_nested") def test_subcollection_reduction(self): - jobs_summary = self._run_workflow(""" + jobs_summary = self._run_workflow( + """ class: GalaxyWorkflow steps: creates_nested_list: @@ -264,7 +279,8 @@ steps: tool_id: cat_list in: input1: creates_nested_list/list_output -""") +""" + ) job1_id = self._job_id_for_tool(jobs_summary.jobs, "cat_list") job2_id = self._job_id_for_tool(jobs_summary.jobs, "collection_creates_dynamic_nested") self._extract_and_download_workflow( @@ -277,7 +293,8 @@ steps: @skip_without_tool("collection_split_on_column") def test_extract_workflow_with_output_collections(self): - jobs_summary = self._run_workflow(""" + jobs_summary = self._run_workflow( + """ class: GalaxyWorkflow steps: - label: text_input1 @@ -304,7 +321,8 @@ steps: test_data: text_input1: "samp1\t10.0\nsamp2\t20.0\n" text_input2: "samp1\t30.0\nsamp2\t40.0\n" -""") +""" + ) tool_ids = ["cat1", "collection_split_on_column", "cat_list"] job_ids = [functools.partial(self._job_id_for_tool, jobs_summary.jobs)(_) for _ in tool_ids] downloaded_workflow = self._extract_and_download_workflow( @@ -324,7 +342,8 @@ test_data: @skip_without_tool("collection_creates_pair") @summarize_instance_history_on_error def test_extract_with_mapped_output_collections(self): - jobs_summary = self._run_workflow(""" + jobs_summary = self._run_workflow( + """ class: GalaxyWorkflow steps: - label: text_input1 @@ -356,7 +375,8 @@ test_data: content: "samp1\t10.0\nsamp2\t20.0\n" - identifier: samp2 content: "samp1\t30.0\nsamp2\t40.0\n" -""") +""" + ) tool_ids = ["cat1", "collection_creates_pair", "cat_collection", "cat_list"] job_ids = [functools.partial(self._job_id_for_tool, jobs_summary.jobs)(_) for _ in tool_ids] downloaded_workflow = self._extract_and_download_workflow( @@ -385,32 +405,22 @@ test_data: return tool_jobs[-1] def __run_random_lines_mapped_over_pair(self, history_id): - hdca = self.dataset_collection_populator.create_pair_in_history(history_id, contents=["1 2 3\n4 5 6", "7 8 9\n10 11 10"]).json() + hdca = self.dataset_collection_populator.create_pair_in_history( + history_id, contents=["1 2 3\n4 5 6", "7 8 9\n10 11 10"] + ).json() hdca_id = hdca["id"] - inputs1 = { - "input": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}, - "num_lines": 2 - } + inputs1 = {"input": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}, "num_lines": 2} implicit_hdca1, job_id1 = self._run_tool_get_collection_and_job_id(history_id, "random_lines1", inputs1) - inputs2 = { - "input": {"batch": True, "values": [{"src": "hdca", "id": implicit_hdca1["id"]}]}, - "num_lines": 1 - } + inputs2 = {"input": {"batch": True, "values": [{"src": "hdca", "id": implicit_hdca1["id"]}]}, "num_lines": 1} _, job_id2 = self._run_tool_get_collection_and_job_id(history_id, "random_lines1", inputs2) return hdca, job_id1, job_id2 def __run_random_lines_mapped_over_singleton(self, history_id): hdca = self.dataset_collection_populator.create_list_in_history(history_id, contents=["1 2 3\n4 5 6"]).json() hdca_id = hdca["id"] - inputs1 = { - "input": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}, - "num_lines": 2 - } + inputs1 = {"input": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}, "num_lines": 2} implicit_hdca1, job_id1 = self._run_tool_get_collection_and_job_id(history_id, "random_lines1", inputs1) - inputs2 = { - "input": {"batch": True, "values": [{"src": "hdca", "id": implicit_hdca1["id"]}]}, - "num_lines": 1 - } + inputs2 = {"input": {"batch": True, "values": [{"src": "hdca", "id": implicit_hdca1["id"]}]}, "num_lines": 1} _, job_id2 = self._run_tool_get_collection_and_job_id(history_id, "random_lines1", inputs2) return hdca, job_id1, job_id2 @@ -450,17 +460,11 @@ test_data: def __copy_content_to_history(self, history_id, content): if content["history_content_type"] == "dataset": - payload = dict( - source="hda", - content=content["id"] - ) + payload = dict(source="hda", content=content["id"]) response = self._post(f"histories/{history_id}/contents/datasets", payload, json=True) else: - payload = dict( - source="hdca", - content=content["id"] - ) + payload = dict(source="hdca", content=content["id"]) response = self._post(f"histories/{history_id}/contents/dataset_collections", payload, json=True) self._assert_status_code_is(response, 200) return response.json() @@ -499,11 +503,15 @@ test_data: history_length = self.dataset_populator.history_length(history_id) new_history_id = self.dataset_populator.reimport_history( - history_id, history_name, wait_on_history_length=history_length, export_kwds={}, + history_id, + history_name, + wait_on_history_length=history_length, + export_kwds={}, ) # wait a little more for those jobs, todo fix to wait for history imported false or # for a specific number of jobs... import time + time.sleep(1) if "reimport_jobs_ids" in extract_payload: @@ -513,8 +521,14 @@ test_data: # Assume no copying or anything so just straight map job ids by index. # Jobs are created after datasets, need to also wait on those... - history_jobs = [j for j in self.dataset_populator.history_jobs(history_id) if j["tool_id"] != "__EXPORT_HISTORY__"] - new_history_jobs = [j for j in self.dataset_populator.history_jobs(new_history_id) if j["tool_id"] != "__EXPORT_HISTORY__"] + history_jobs = [ + j for j in self.dataset_populator.history_jobs(history_id) if j["tool_id"] != "__EXPORT_HISTORY__" + ] + new_history_jobs = [ + j + for j in self.dataset_populator.history_jobs(new_history_id) + if j["tool_id"] != "__EXPORT_HISTORY__" + ] history_job_ids = [j["id"] for j in history_jobs] new_history_job_ids = [j["id"] for j in new_history_jobs] @@ -592,7 +606,7 @@ test_data: data_collection_input_count=None, tool_ids=None, ): - steps = workflow['steps'] + steps = workflow["steps"] if step_count is not None: assert len(steps) == step_count @@ -612,7 +626,7 @@ test_data: disconnected_inputs = [] for value in steps.values(): - if value['type'] == "tool": + if value["type"] == "tool": input_connections = value["input_connections"] if not input_connections: disconnected_inputs.append(value) @@ -623,4 +637,4 @@ test_data: raise AssertionError(message) -RunJobsSummary = namedtuple('RunJobsSummary', ['history_id', 'workflow_id', 'inputs', 'jobs']) +RunJobsSummary = namedtuple("RunJobsSummary", ["history_id", "workflow_id", "inputs", "jobs"]) diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index 416f7dc31b3..c17f3d5d864 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -4,11 +4,22 @@ import shutil import time from json import dumps from tempfile import mkdtemp -from typing import Any, cast, Dict, Optional, Tuple, Union +from typing import ( + Any, + cast, + Dict, + Optional, + Tuple, + Union, +) from uuid import uuid4 import pytest -from requests import delete, get, put +from requests import ( + delete, + get, + put, +) from galaxy.exceptions import error_codes from galaxy_test.base import rules_test_data @@ -18,7 +29,7 @@ from galaxy_test.base.populators import ( RunJobsSummary, skip_without_tool, wait_on, - WorkflowPopulator + WorkflowPopulator, ) from galaxy_test.base.workflow_fixtures import ( WORKFLOW_NESTED_REPLACEMENT_PARAMETER, @@ -43,7 +54,6 @@ from galaxy_test.base.workflow_fixtures import ( ) from ._framework import ApiTestCase - WORKFLOW_SIMPLE = """ class: GalaxyWorkflow name: Simple Workflow @@ -124,10 +134,14 @@ class BaseWorkflowsApiTestCase(ApiTestCase): def _upload_yaml_workflow(self, has_yaml, **kwds) -> str: return self.workflow_populator.upload_yaml_workflow(has_yaml, **kwds) - def _setup_workflow_run(self, workflow: Optional[Dict[str, Any]] = None, inputs_by: str = 'step_id', history_id: Optional[str] = None, workflow_id: Optional[str] = None) -> Tuple[Dict[str, Any], str, str]: - return self.workflow_populator.setup_workflow_run( - workflow, inputs_by, history_id, workflow_id - ) + def _setup_workflow_run( + self, + workflow: Optional[Dict[str, Any]] = None, + inputs_by: str = "step_id", + history_id: Optional[str] = None, + workflow_id: Optional[str] = None, + ) -> Tuple[Dict[str, Any], str, str]: + return self.workflow_populator.setup_workflow_run(workflow, inputs_by, history_id, workflow_id) def _ds_entry(self, history_content): return self.dataset_populator.ds_entry(history_content) @@ -176,7 +190,8 @@ class ChangeDatatypeTestCase: def test_assign_column_pja(self): with self.dataset_populator.test_history() as history_id: - self.workflow_populator.run_workflow(""" + self.workflow_populator.run_workflow( + """ class: GalaxyWorkflow inputs: input1: data @@ -192,15 +207,20 @@ steps: chromCol: 1 endCol: 2 startCol: 3 -""", test_data=""" +""", + test_data=""" input1: value: 1.bed type: File -""", history_id=history_id) - details_dataset_new_col = self.dataset_populator.get_history_dataset_details(history_id, hid=2, wait=True, assert_ok=True) +""", + history_id=history_id, + ) + details_dataset_new_col = self.dataset_populator.get_history_dataset_details( + history_id, hid=2, wait=True, assert_ok=True + ) assert details_dataset_new_col["history_content_type"] == "dataset", details_dataset_new_col - assert details_dataset_new_col['metadata_endCol'] == 2 - assert details_dataset_new_col['metadata_startCol'] == 3 + assert details_dataset_new_col["metadata_endCol"] == 2 + assert details_dataset_new_col["metadata_startCol"] == 3 # Workflow API TODO: @@ -209,7 +229,6 @@ input1: # /workflows with id in payload. # - Much more testing obviously, always more testing. class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase): - def test_show_valid(self): workflow_id = self.workflow_populator.simple_workflow("dummy") workflow_id = self.workflow_populator.simple_workflow("test_regular") @@ -246,7 +265,7 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase): with self._different_user(): with pytest.raises(AssertionError) as excinfo: self._download_workflow(workflow_id) - assert '403' in str(excinfo.value) + assert "403" in str(excinfo.value) workflows_url = self._api_url(f"workflows/{workflow_id}/download") assert get(workflows_url).status_code == 403 @@ -255,7 +274,7 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase): workflows_url = self._api_url(f"workflows/{workflow_id}/download") response = get(workflows_url) response.raise_for_status() - assert response.json()['a_galaxy_workflow'] == 'true' + assert response.json()["a_galaxy_workflow"] == "true" def test_delete(self): workflow_id = self.workflow_populator.simple_workflow("test_delete") @@ -282,27 +301,27 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase): def test_index_deleted(self): workflow_id = self.workflow_populator.simple_workflow("test_delete") workflow_index = self._get("workflows").json() - assert [w for w in workflow_index if w['id'] == workflow_id] + assert [w for w in workflow_index if w["id"] == workflow_id] workflow_url = self._api_url(f"workflows/{workflow_id}", use_key=True) delete_response = delete(workflow_url) self._assert_status_code_is(delete_response, 200) workflow_index = self._get("workflows").json() - assert not [w for w in workflow_index if w['id'] == workflow_id] + assert not [w for w in workflow_index if w["id"] == workflow_id] workflow_index = self._get("workflows?show_deleted=true").json() - assert [w for w in workflow_index if w['id'] == workflow_id] + assert [w for w in workflow_index if w["id"] == workflow_id] def test_index_hidden(self): workflow_id = self.workflow_populator.simple_workflow("test_delete") workflow_index = self._get("workflows").json() - workflow = [w for w in workflow_index if w['id'] == workflow_id][0] - workflow['hidden'] = True + workflow = [w for w in workflow_index if w["id"] == workflow_id][0] + workflow["hidden"] = True update_response = self.workflow_populator.update_workflow(workflow_id, workflow) self._assert_status_code_is(update_response, 200) - assert update_response.json()['hidden'] + assert update_response.json()["hidden"] workflow_index = self._get("workflows").json() - assert not [w for w in workflow_index if w['id'] == workflow_id] + assert not [w for w in workflow_index if w["id"] == workflow_id] workflow_index = self._get("workflows?show_hidden=true").json() - assert [w for w in workflow_index if w['id'] == workflow_id] + assert [w for w in workflow_index if w["id"] == workflow_id] def test_upload(self): self.__test_upload(use_deprecated_route=False) @@ -314,7 +333,9 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase): response = self.__test_upload(import_tools=True, assert_ok=False) assert response.status_code == 403 - def __test_upload(self, use_deprecated_route=False, name="test_import", workflow=None, assert_ok=True, import_tools=False): + def __test_upload( + self, use_deprecated_route=False, name="test_import", workflow=None, assert_ok=True, import_tools=False + ): if workflow is None: workflow = self.workflow_populator.load_workflow(name=name) data = dict( @@ -333,8 +354,11 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase): return upload_response def test_get_tool_predictions(self): - request = {"tool_sequence": "Cut1", "remote_model_url": "https://github.com/galaxyproject/galaxy-test-data/raw/master/tool_recommendation_model.hdf5"} - actual_recommendations = ['Filter1', 'cat1', 'addValue', 'comp1', 'Grep1'] + request = { + "tool_sequence": "Cut1", + "remote_model_url": "https://github.com/galaxyproject/galaxy-test-data/raw/master/tool_recommendation_model.hdf5", + } + actual_recommendations = ["Filter1", "cat1", "addValue", "comp1", "Grep1"] route = "workflows/get_tool_predictions" response = self._post(route, data=request) recommendation_response = response.json() @@ -389,9 +413,9 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase): def tweak_step(step): order_index, step_dict = step check_label_and_uuid(order_index, step_dict) - assert step_dict['position']['top'] != 1 - assert step_dict['position']['left'] != 1 - step_dict['position'] = {'top': 1, 'left': 1} + assert step_dict["position"]["top"] != 1 + assert step_dict["position"]["left"] != 1 + step_dict["position"] = {"top": 1, "left": 1} map(tweak_step, steps.items()) @@ -400,11 +424,11 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase): def check_step(step): order_index, step_dict = step check_label_and_uuid(order_index, step_dict) - assert step_dict['position']['top'] == 1 - assert step_dict['position']['left'] == 1 + assert step_dict["position"]["top"] == 1 + assert step_dict["position"]["left"] == 1 updated_workflow_content = self._download_workflow(workflow_id) - map(check_step, updated_workflow_content['steps'].items()) + map(check_step, updated_workflow_content["steps"].items()) # Re-update against original workflow... update(original_workflow) @@ -412,21 +436,21 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase): updated_workflow_content = self._download_workflow(workflow_id) # Make sure the positions have been updated. - map(tweak_step, updated_workflow_content['steps'].items()) + map(tweak_step, updated_workflow_content["steps"].items()) def test_update_tags(self): workflow_object = self.workflow_populator.load_workflow(name="test_import") upload_response = self.__test_upload(workflow=workflow_object) workflow = upload_response.json() - workflow['tags'] = ['a_tag', 'b_tag'] - update_response = self._update_workflow(workflow['id'], workflow).json() - assert update_response['tags'] == ['a_tag', 'b_tag'] - del workflow['tags'] - update_response = self._update_workflow(workflow['id'], workflow).json() - assert update_response['tags'] == ['a_tag', 'b_tag'] - workflow['tags'] = [] - update_response = self._update_workflow(workflow['id'], workflow).json() - assert update_response['tags'] == [] + workflow["tags"] = ["a_tag", "b_tag"] + update_response = self._update_workflow(workflow["id"], workflow).json() + assert update_response["tags"] == ["a_tag", "b_tag"] + del workflow["tags"] + update_response = self._update_workflow(workflow["id"], workflow).json() + assert update_response["tags"] == ["a_tag", "b_tag"] + workflow["tags"] = [] + update_response = self._update_workflow(workflow["id"], workflow).json() + assert update_response["tags"] == [] def test_update_name(self): original_name = "test update name" @@ -434,19 +458,20 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase): workflow_object["license"] = "AAL" upload_response = self.__test_upload(workflow=workflow_object, name=original_name) workflow = upload_response.json() - workflow_id = workflow['id'] - assert workflow['name'] == original_name + workflow_id = workflow["id"] + assert workflow["name"] == original_name workflow_dict = self.workflow_populator.download_workflow(workflow_id) assert workflow_dict["license"] == "AAL" data = {"name": "my cool new name"} - update_response = self._update_workflow(workflow['id'], data).json() - assert update_response['name'] == "my cool new name" + update_response = self._update_workflow(workflow["id"], data).json() + assert update_response["name"] == "my cool new name" workflow_dict = self.workflow_populator.download_workflow(workflow_id) assert workflow_dict["license"] == "AAL" def test_refactor(self): - workflow_id = self.workflow_populator.upload_yaml_workflow(""" + workflow_id = self.workflow_populator.upload_yaml_workflow( + """ class: GalaxyWorkflow inputs: test_input: data @@ -455,7 +480,8 @@ steps: tool_id: cat in: input1: test_input -""") +""" + ) actions = [ {"action_type": "update_step_label", "step": {"order_index": 0}, "label": "new_label"}, ] @@ -465,7 +491,9 @@ steps: assert refactor_response.json()["workflow"]["steps"]["0"]["label"] == "new_label" # perform refactoring as dry run but specify editor style response - refactor_response = self.workflow_populator.refactor_workflow(workflow_id, actions, dry_run=True, style="editor") + refactor_response = self.workflow_populator.refactor_workflow( + workflow_id, actions, dry_run=True, style="editor" + ) refactor_response.raise_for_status() assert refactor_response.json()["workflow"]["steps"]["0"]["label"] == "new_label" @@ -521,7 +549,8 @@ steps: self._assert_user_has_workflow_with_name("imported: test_import_published_deprecated") def test_import_export_dynamic(self): - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow steps: - type: input @@ -547,7 +576,8 @@ steps: $link: embed1/output1 test_data: input1: "hello world" -""") +""" + ) downloaded_workflow = self._download_workflow(workflow_id) # The _upload_yaml_workflow entry point uses an admin key, but if we try to # do the raw re-import as a regular user we expect a 403 error. @@ -571,18 +601,18 @@ test_data: def test_import_subworkflows(self): def get_subworkflow_content_id(workflow_id): workflow_contents = self._download_workflow(workflow_id, style="editor") - steps = workflow_contents['steps'] + steps = workflow_contents["steps"] subworkflow_step = next(s for s in steps.values() if s["type"] == "subworkflow") - return subworkflow_step['content_id'] + return subworkflow_step["content_id"] workflow_id = self._upload_yaml_workflow(WORKFLOW_NESTED_SIMPLE, publish=True) subworkflow_content_id = get_subworkflow_content_id(workflow_id) instance_response = self._get(f"workflows/{subworkflow_content_id}?instance=true") self._assert_status_code_is(instance_response, 200) subworkflow = instance_response.json() - assert subworkflow['inputs']['0']['label'] == 'inner_input' - assert subworkflow['name'] == 'Workflow' - assert subworkflow['hidden'] + assert subworkflow["inputs"]["0"]["label"] == "inner_input" + assert subworkflow["name"] == "Workflow" + assert subworkflow["hidden"] with self._different_user(): other_import_response = self.__import_workflow(workflow_id) self._assert_status_code_is(other_import_response, 200) @@ -591,7 +621,8 @@ test_data: assert subworkflow_content_id != imported_subworkflow_content_id def test_subworkflow_inputs_optional_editor(self): - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow steps: nested_workflow: @@ -603,9 +634,10 @@ steps: outputs: - outputSource: inner_input/output steps: [] -""") +""" + ) workflow_contents = self._download_workflow(workflow_id, style="editor") - assert workflow_contents['steps']['0']['inputs'][0]['optional'] + assert workflow_contents["steps"]["0"]["inputs"][0]["optional"] def test_not_importable_prevents_import(self): workflow_id = self.workflow_populator.simple_workflow("test_not_importportable") @@ -652,18 +684,18 @@ steps: for step in downloaded_workflow["steps"].values(): self._assert_has_keys( step, - 'id', - 'type', - 'tool_id', - 'tool_version', - 'name', - 'tool_state', - 'annotation', - 'inputs', - 'workflow_outputs', - 'outputs' + "id", + "type", + "tool_id", + "tool_version", + "name", + "tool_state", + "annotation", + "inputs", + "workflow_outputs", + "outputs", ) - if step['type'] == "tool": + if step["type"] == "tool": self._assert_has_keys(step, "post_job_actions") def test_export_format2(self): @@ -678,23 +710,23 @@ steps: for step in downloaded_workflow["steps"].values(): self._assert_has_keys( step, - 'id', - 'type', - 'content_id', - 'name', - 'tool_state', - 'tooltip', - 'inputs', - 'outputs', - 'config_form', - 'annotation', - 'post_job_actions', - 'workflow_outputs', - 'uuid', - 'label', + "id", + "type", + "content_id", + "name", + "tool_state", + "tooltip", + "inputs", + "outputs", + "config_form", + "annotation", + "post_job_actions", + "workflow_outputs", + "uuid", + "label", ) - @skip_without_tool('output_filter_with_input') + @skip_without_tool("output_filter_with_input") def test_export_editor_filtered_outputs(self): template = """ class: GalaxyWorkflow @@ -706,41 +738,44 @@ steps: produce_collection: false produce_paired_collection: false """ - workflow_id = self._upload_yaml_workflow(template.format(produce_out_1='false', filter_text_1='false')) + workflow_id = self._upload_yaml_workflow(template.format(produce_out_1="false", filter_text_1="false")) downloaded_workflow = self._download_workflow(workflow_id, style="editor") - outputs = downloaded_workflow['steps']['0']['outputs'] + outputs = downloaded_workflow["steps"]["0"]["outputs"] assert len(outputs) == 1 - assert outputs[0]['name'] == 'out_3' - workflow_id = self._upload_yaml_workflow(template.format(produce_out_1='true', filter_text_1='false')) + assert outputs[0]["name"] == "out_3" + workflow_id = self._upload_yaml_workflow(template.format(produce_out_1="true", filter_text_1="false")) downloaded_workflow = self._download_workflow(workflow_id, style="editor") - outputs = downloaded_workflow['steps']['0']['outputs'] + outputs = downloaded_workflow["steps"]["0"]["outputs"] assert len(outputs) == 2 - assert outputs[0]['name'] == 'out_1' - assert outputs[1]['name'] == 'out_3' - workflow_id = self._upload_yaml_workflow(template.format(produce_out_1='true', filter_text_1='foo')) + assert outputs[0]["name"] == "out_1" + assert outputs[1]["name"] == "out_3" + workflow_id = self._upload_yaml_workflow(template.format(produce_out_1="true", filter_text_1="foo")) downloaded_workflow = self._download_workflow(workflow_id, style="editor") - outputs = downloaded_workflow['steps']['0']['outputs'] + outputs = downloaded_workflow["steps"]["0"]["outputs"] assert len(outputs) == 3 - assert outputs[0]['name'] == 'out_1' - assert outputs[1]['name'] == 'out_2' - assert outputs[2]['name'] == 'out_3' + assert outputs[0]["name"] == "out_1" + assert outputs[1]["name"] == "out_2" + assert outputs[2]["name"] == "out_3" - @skip_without_tool('output_filter_exception_1') + @skip_without_tool("output_filter_exception_1") def test_export_editor_filtered_outputs_exception_handling(self): - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow steps: - tool_id: output_filter_exception_1 -""") +""" + ) downloaded_workflow = self._download_workflow(workflow_id, style="editor") - outputs = downloaded_workflow['steps']['0']['outputs'] + outputs = downloaded_workflow["steps"]["0"]["outputs"] assert len(outputs) == 2 - assert outputs[0]['name'] == 'out_1' - assert outputs[1]['name'] == 'out_2' + assert outputs[0]["name"] == "out_1" + assert outputs[1]["name"] == "out_2" - @skip_without_tool('collection_type_source') + @skip_without_tool("collection_type_source") def test_export_editor_collection_type_source(self): - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow inputs: - id: text_input1 @@ -750,17 +785,19 @@ steps: - tool_id: collection_type_source in: input_collect: text_input1 -""") +""" + ) downloaded_workflow = self._download_workflow(workflow_id, style="editor") - steps = downloaded_workflow['steps'] + steps = downloaded_workflow["steps"] assert len(steps) == 2 # Non-subworkflow collection_type_source tools will be handled by the client, # so collection_type should be None here. - assert steps['1']['outputs'][0]['collection_type'] is None + assert steps["1"]["outputs"][0]["collection_type"] is None - @skip_without_tool('collection_type_source') + @skip_without_tool("collection_type_source") def test_export_editor_subworkflow_collection_type_source(self): - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow inputs: outer_input: data @@ -782,19 +819,20 @@ steps: input_collect: inner_input in: inner_input: outer_input -""") +""" + ) downloaded_workflow = self._download_workflow(workflow_id, style="editor") - steps = downloaded_workflow['steps'] + steps = downloaded_workflow["steps"] assert len(steps) == 2 - assert steps['1']['type'] == 'subworkflow' - assert steps['1']['outputs'][0]['collection_type'] == 'list:paired' + assert steps["1"]["type"] == "subworkflow" + assert steps["1"]["outputs"][0]["collection_type"] == "list:paired" def test_import_missing_tool(self): workflow = self.workflow_populator.load_workflow_from_resource(name="test_workflow_missing_tool") workflow_id = self.workflow_populator.create_workflow(workflow) workflow_description = self._show_workflow(workflow_id) steps = workflow_description["steps"] - missing_tool_steps = [v for v in steps.values() if v['tool_id'] == 'cat_missing_tool'] + missing_tool_steps = [v for v in steps.values() if v["tool_id"] == "cat_missing_tool"] assert len(missing_tool_steps) == 1 def test_import_no_tool_id(self): @@ -822,28 +860,29 @@ steps: @skip_without_tool("cat1") def test_run_workflow_by_index(self): - self.__run_cat_workflow(inputs_by='step_index') + self.__run_cat_workflow(inputs_by="step_index") @skip_without_tool("cat1") def test_run_workflow_by_uuid(self): - self.__run_cat_workflow(inputs_by='step_uuid') + self.__run_cat_workflow(inputs_by="step_uuid") @skip_without_tool("cat1") def test_run_workflow_by_uuid_implicitly(self): - self.__run_cat_workflow(inputs_by='uuid_implicitly') + self.__run_cat_workflow(inputs_by="uuid_implicitly") @skip_without_tool("cat1") def test_run_workflow_by_name(self): - self.__run_cat_workflow(inputs_by='name') + self.__run_cat_workflow(inputs_by="name") @skip_without_tool("cat1") def test_run_workflow(self): - self.__run_cat_workflow(inputs_by='step_id') + self.__run_cat_workflow(inputs_by="step_id") @skip_without_tool("multiple_versions") def test_run_versioned_tools(self): with self.dataset_populator.test_history() as history_01_id: - workflow_version_01 = self._upload_yaml_workflow(""" + workflow_version_01 = self._upload_yaml_workflow( + """ class: GalaxyWorkflow steps: multiple: @@ -851,11 +890,13 @@ steps: tool_version: "0.1" state: inttest: 0 -""") +""" + ) self.workflow_populator.invoke_workflow_and_wait(workflow_version_01, history_id=history_01_id) with self.dataset_populator.test_history() as history_02_id: - workflow_version_02 = self._upload_yaml_workflow(""" + workflow_version_02 = self._upload_yaml_workflow( + """ class: GalaxyWorkflow steps: multiple: @@ -863,7 +904,8 @@ steps: tool_version: "0.2" state: inttest: 1 -""") +""" + ) self.workflow_populator.invoke_workflow_and_wait(workflow_version_02, history_id=history_02_id) def __run_cat_workflow(self, inputs_by): @@ -871,22 +913,29 @@ steps: workflow["steps"]["0"]["uuid"] = str(uuid4()) workflow["steps"]["1"]["uuid"] = str(uuid4()) workflow_request, _, workflow_id = self._setup_workflow_run(workflow, inputs_by=inputs_by) - invocation_id = self.workflow_populator.invoke_workflow_and_wait(workflow_id, request=workflow_request, assert_ok=True) + invocation_id = self.workflow_populator.invoke_workflow_and_wait( + workflow_id, request=workflow_request, assert_ok=True + ) invocation = self._invocation_details(workflow_id, invocation_id) assert invocation["state"] == "scheduled", invocation def test_run_workflow_with_missing_tool(self): with self.dataset_populator.test_history() as history_id: - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow steps: step1: tool_id: nonexistent_tool tool_version: "0.1" -""") +""" + ) invocation_response = self.__invoke_workflow(workflow_id, history_id=history_id, assert_ok=False) self._assert_status_code_is(invocation_response, 400) - self.assertEqual(invocation_response.json().get('err_msg'), "Workflow was not invoked; the following required tools are not installed: nonexistent_tool") + self.assertEqual( + invocation_response.json().get("err_msg"), + "Workflow was not invoked; the following required tools are not installed: nonexistent_tool", + ) @skip_without_tool("collection_creates_pair") def test_workflow_run_output_collections(self) -> None: @@ -897,7 +946,8 @@ steps: @skip_without_tool("job_properties") @skip_without_tool("identifier_multiple_in_conditional") def test_workflow_resume_from_failed_step(self): - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow steps: job_props: @@ -920,31 +970,39 @@ steps: in: input1: identifier/output1 queries_0|input2: identifier/output1 -""") +""" + ) with self.dataset_populator.test_history() as history_id: self.workflow_populator.invoke_workflow_and_wait(workflow_id, history_id=history_id, assert_ok=False) - failed_dataset_one = self.dataset_populator.get_history_dataset_details(history_id, hid=1, wait=True, assert_ok=False) - assert failed_dataset_one['state'] == 'error', failed_dataset_one - paused_dataset = self.dataset_populator.get_history_dataset_details(history_id, hid=5, wait=True, assert_ok=False) - assert paused_dataset['state'] == 'paused', paused_dataset - inputs = {"thebool": "false", - "failbool": "false", - "rerun_remap_job_id": failed_dataset_one['creating_job']} + failed_dataset_one = self.dataset_populator.get_history_dataset_details( + history_id, hid=1, wait=True, assert_ok=False + ) + assert failed_dataset_one["state"] == "error", failed_dataset_one + paused_dataset = self.dataset_populator.get_history_dataset_details( + history_id, hid=5, wait=True, assert_ok=False + ) + assert paused_dataset["state"] == "paused", paused_dataset + inputs = {"thebool": "false", "failbool": "false", "rerun_remap_job_id": failed_dataset_one["creating_job"]} self.dataset_populator.run_tool( - tool_id='job_properties', + tool_id="job_properties", inputs=inputs, history_id=history_id, ) - unpaused_dataset_1 = self.dataset_populator.get_history_dataset_details(history_id, hid=5, wait=True, assert_ok=False) - assert unpaused_dataset_1['state'] == 'ok' + unpaused_dataset_1 = self.dataset_populator.get_history_dataset_details( + history_id, hid=5, wait=True, assert_ok=False + ) + assert unpaused_dataset_1["state"] == "ok" self.dataset_populator.wait_for_history(history_id, assert_ok=False) - unpaused_dataset_2 = self.dataset_populator.get_history_dataset_details(history_id, hid=6, wait=True, assert_ok=False) - assert unpaused_dataset_2['state'] == 'ok' + unpaused_dataset_2 = self.dataset_populator.get_history_dataset_details( + history_id, hid=6, wait=True, assert_ok=False + ) + assert unpaused_dataset_2["state"] == "ok" @skip_without_tool("job_properties") @skip_without_tool("collection_creates_list") def test_workflow_resume_from_failed_step_with_hdca_input(self): - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow steps: job_props: @@ -960,39 +1018,50 @@ steps: tool_id: identifier_collection in: input1: list_in_list_out/list_output -""") +""" + ) with self.dataset_populator.test_history() as history_id: invocation_id = self.__invoke_workflow(workflow_id, history_id=history_id) - self.workflow_populator.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id, assert_ok=False) - failed_dataset_one = self.dataset_populator.get_history_dataset_details(history_id, hid=1, wait=True, assert_ok=False) - assert failed_dataset_one['state'] == 'error', failed_dataset_one - paused_colletion = self.dataset_populator.get_history_collection_details(history_id, hid=7, wait=True, assert_ok=False) - first_paused_element = paused_colletion['elements'][0]['object'] - assert first_paused_element['state'] == 'paused', first_paused_element - dependent_dataset = self.dataset_populator.get_history_dataset_details(history_id, hid=8, wait=True, - assert_ok=False) - assert dependent_dataset['state'] == 'paused' - inputs = {"thebool": "false", - "failbool": "false", - "rerun_remap_job_id": failed_dataset_one['creating_job']} + self.workflow_populator.wait_for_invocation_and_jobs( + history_id, workflow_id, invocation_id, assert_ok=False + ) + failed_dataset_one = self.dataset_populator.get_history_dataset_details( + history_id, hid=1, wait=True, assert_ok=False + ) + assert failed_dataset_one["state"] == "error", failed_dataset_one + paused_colletion = self.dataset_populator.get_history_collection_details( + history_id, hid=7, wait=True, assert_ok=False + ) + first_paused_element = paused_colletion["elements"][0]["object"] + assert first_paused_element["state"] == "paused", first_paused_element + dependent_dataset = self.dataset_populator.get_history_dataset_details( + history_id, hid=8, wait=True, assert_ok=False + ) + assert dependent_dataset["state"] == "paused" + inputs = {"thebool": "false", "failbool": "false", "rerun_remap_job_id": failed_dataset_one["creating_job"]} self.dataset_populator.run_tool( - tool_id='job_properties', + tool_id="job_properties", inputs=inputs, history_id=history_id, ) - paused_colletion = self.dataset_populator.get_history_collection_details(history_id, hid=7, wait=True, assert_ok=False) - first_paused_element = paused_colletion['elements'][0]['object'] - assert first_paused_element['state'] == 'ok' + paused_colletion = self.dataset_populator.get_history_collection_details( + history_id, hid=7, wait=True, assert_ok=False + ) + first_paused_element = paused_colletion["elements"][0]["object"] + assert first_paused_element["state"] == "ok" self.dataset_populator.wait_for_history(history_id, assert_ok=False) - dependent_dataset = self.dataset_populator.get_history_dataset_details(history_id, hid=8, wait=True, assert_ok=False) - assert dependent_dataset['name'].startswith('identifier_collection') - assert dependent_dataset['state'] == 'ok' + dependent_dataset = self.dataset_populator.get_history_dataset_details( + history_id, hid=8, wait=True, assert_ok=False + ) + assert dependent_dataset["name"].startswith("identifier_collection") + assert dependent_dataset["state"] == "ok" @skip_without_tool("fail_identifier") @skip_without_tool("identifier_collection") def test_workflow_resume_with_mapped_over_input(self): with self.dataset_populator.test_history() as history_id: - self._run_workflow(""" + self._run_workflow( + """ class: GalaxyWorkflow inputs: input_datasets: collection @@ -1017,36 +1086,45 @@ test_data: - identifier: success value: 1.fastq type: File -""", history_id=history_id, assert_ok=False, wait=True) +""", + history_id=history_id, + assert_ok=False, + wait=True, + ) history_contents = self.dataset_populator._get_contents_request(history_id=history_id).json() first_input = history_contents[1] - assert first_input['history_content_type'] == 'dataset' + assert first_input["history_content_type"] == "dataset" paused_dataset = history_contents[-1] failed_dataset = self.dataset_populator.get_history_dataset_details(history_id, hid=5, assert_ok=False) - assert paused_dataset['state'] == 'paused', paused_dataset - assert failed_dataset['state'] == 'error', failed_dataset - inputs = {"input1": {'values': [{'src': 'hda', - 'id': first_input['id']}] - }, - "failbool": "false", - "rerun_remap_job_id": failed_dataset['creating_job']} + assert paused_dataset["state"] == "paused", paused_dataset + assert failed_dataset["state"] == "error", failed_dataset + inputs = { + "input1": {"values": [{"src": "hda", "id": first_input["id"]}]}, + "failbool": "false", + "rerun_remap_job_id": failed_dataset["creating_job"], + } run_dict = self.dataset_populator.run_tool( - tool_id='fail_identifier', + tool_id="fail_identifier", inputs=inputs, history_id=history_id, ) - unpaused_dataset = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=False) - assert unpaused_dataset['state'] == 'ok' + unpaused_dataset = self.dataset_populator.get_history_dataset_details( + history_id, wait=True, assert_ok=False + ) + assert unpaused_dataset["state"] == "ok" contents = self.dataset_populator.get_history_dataset_content(history_id, hid=7, assert_ok=False) - assert contents == 'fail\nsuccess\n', contents - replaced_hda_id = run_dict['outputs'][0]['id'] - replaced_hda = self.dataset_populator.get_history_dataset_details(history_id, dataset_id=replaced_hda_id, wait=True, assert_ok=False) - assert not replaced_hda['visible'], replaced_hda + assert contents == "fail\nsuccess\n", contents + replaced_hda_id = run_dict["outputs"][0]["id"] + replaced_hda = self.dataset_populator.get_history_dataset_details( + history_id, dataset_id=replaced_hda_id, wait=True, assert_ok=False + ) + assert not replaced_hda["visible"], replaced_hda def test_workflow_resume_with_mapped_over_collection_input(self): # Test that replacement and resume also works if the failed job re-run works on a input DCE with self.dataset_populator.test_history() as history_id: - job_summary = self._run_workflow(""" + job_summary = self._run_workflow( + """ class: GalaxyWorkflow inputs: input_collection: collection @@ -1066,43 +1144,60 @@ steps: test_data: input_collection: collection_type: "list:list:paired" -""", history_id=history_id, assert_ok=False, wait=True) +""", + history_id=history_id, + assert_ok=False, + wait=True, + ) invocation = self.workflow_populator.get_invocation(job_summary.invocation_id, step_details=True) # TODO: return steps sorted by order_index ? Why don't we do that ?? - invocation['steps'].sort(key=lambda step: step['order_index']) - failed_step = invocation['steps'][1] - assert failed_step['jobs'][0]['state'] == 'error' - failed_hdca_id = failed_step['output_collections']['list_output']['id'] - failed_hdca = self.dataset_populator.get_history_collection_details(history_id=history_id, content_id=failed_hdca_id, assert_ok=False) - assert failed_hdca['elements'][0]['object']['elements'][0]['object']['elements'][0]['object']['state'] == 'error' - paused_step = invocation['steps'][2] + invocation["steps"].sort(key=lambda step: step["order_index"]) + failed_step = invocation["steps"][1] + assert failed_step["jobs"][0]["state"] == "error" + failed_hdca_id = failed_step["output_collections"]["list_output"]["id"] + failed_hdca = self.dataset_populator.get_history_collection_details( + history_id=history_id, content_id=failed_hdca_id, assert_ok=False + ) + assert ( + failed_hdca["elements"][0]["object"]["elements"][0]["object"]["elements"][0]["object"]["state"] + == "error" + ) + paused_step = invocation["steps"][2] # job not created, input in error state - assert paused_step['jobs'][0]['state'] == 'paused' - input_hdca = self.dataset_populator.get_history_collection_details(history_id=history_id, content_id=job_summary.inputs['input_collection']['id'], assert_ok=False) + assert paused_step["jobs"][0]["state"] == "paused" + input_hdca = self.dataset_populator.get_history_collection_details( + history_id=history_id, content_id=job_summary.inputs["input_collection"]["id"], assert_ok=False + ) # now re-run errored job - inputs = {"input1": {'values': [{'src': 'dce', - 'id': input_hdca['elements'][0]['id']}] - }, - "failbool": "false", - "rerun_remap_job_id": failed_step['jobs'][0]['id']} + inputs = { + "input1": {"values": [{"src": "dce", "id": input_hdca["elements"][0]["id"]}]}, + "failbool": "false", + "rerun_remap_job_id": failed_step["jobs"][0]["id"], + } run_response = self.dataset_populator.run_tool( - tool_id='collection_creates_list_of_pairs', + tool_id="collection_creates_list_of_pairs", inputs=inputs, history_id=history_id, ) assert not run_response["output_collections"][0]["visible"] - self.dataset_populator.wait_for_job(paused_step['jobs'][0]['id']) + self.dataset_populator.wait_for_job(paused_step["jobs"][0]["id"]) invocation = self.workflow_populator.get_invocation(job_summary.invocation_id, step_details=True) - rerun_step = invocation['steps'][1] - assert rerun_step['jobs'][0]['state'] == 'ok' - replaced_hdca = self.dataset_populator.get_history_collection_details(history_id=history_id, content_id=failed_hdca_id, assert_ok=False) - assert replaced_hdca['elements'][0]['object']['elements'][0]['object']['elements'][0]['object']['state'] == 'ok' + rerun_step = invocation["steps"][1] + assert rerun_step["jobs"][0]["state"] == "ok" + replaced_hdca = self.dataset_populator.get_history_collection_details( + history_id=history_id, content_id=failed_hdca_id, assert_ok=False + ) + assert ( + replaced_hdca["elements"][0]["object"]["elements"][0]["object"]["elements"][0]["object"]["state"] + == "ok" + ) - @skip_without_tool('multi_data_optional') + @skip_without_tool("multi_data_optional") def test_workflow_list_list_multi_data_map_over(self): # Test that a list:list is reduced to list with a multiple="true" data input with self.dataset_populator.test_history() as history_id: - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow inputs: input_datasets: collection @@ -1111,32 +1206,37 @@ steps: tool_id: multi_data_optional in: input1: input_datasets -""") +""" + ) with self.dataset_populator.test_history() as history_id: hdca_id = self.dataset_collection_populator.create_list_of_list_in_history(history_id).json() self.dataset_populator.wait_for_history(history_id, assert_ok=True) inputs = { - '0': self._ds_entry(hdca_id), + "0": self._ds_entry(hdca_id), } invocation_id = self.__invoke_workflow(workflow_id, inputs=inputs, history_id=history_id) self.workflow_populator.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) output_collection = self.dataset_populator.get_history_collection_details(history_id, hid=6) - assert output_collection['collection_type'] == 'list' - assert output_collection['job_source_type'] == 'ImplicitCollectionJobs' + assert output_collection["collection_type"] == "list" + assert output_collection["job_source_type"] == "ImplicitCollectionJobs" @skip_without_tool("cat_list") @skip_without_tool("collection_creates_pair") def test_workflow_run_output_collection_mapping(self): workflow_id = self._upload_yaml_workflow(WORKFLOW_WITH_OUTPUT_COLLECTION_MAPPING) with self.dataset_populator.test_history() as history_id: - hdca1 = self.dataset_collection_populator.create_list_in_history(history_id, contents=["a\nb\nc\nd\n", "e\nf\ng\nh\n"]).json() + hdca1 = self.dataset_collection_populator.create_list_in_history( + history_id, contents=["a\nb\nc\nd\n", "e\nf\ng\nh\n"] + ).json() self.dataset_populator.wait_for_history(history_id, assert_ok=True) inputs = { - '0': self._ds_entry(hdca1), + "0": self._ds_entry(hdca1), } invocation_id = self.__invoke_workflow(workflow_id, inputs=inputs, history_id=history_id) self.workflow_populator.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) - self.assertEqual("a\nc\nb\nd\ne\ng\nf\nh\n", self.dataset_populator.get_history_dataset_content(history_id, hid=0)) + self.assertEqual( + "a\nc\nb\nd\ne\ng\nf\nh\n", self.dataset_populator.get_history_dataset_content(history_id, hid=0) + ) @skip_without_tool("cat_list") @skip_without_tool("collection_split_on_column") @@ -1155,7 +1255,8 @@ steps: # A more advanced output collection workflow, testing regression of # https://github.com/galaxyproject/galaxy/issues/776 with self.dataset_populator.test_history() as history_id: - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow inputs: test_input_1: data @@ -1171,15 +1272,16 @@ steps: in: queries_0|input: test_input_1 queries2_0|input2: split_up/split_output -""") +""" + ) hda1 = self.dataset_populator.new_dataset(history_id, content="samp1\t10.0\nsamp2\t20.0\n") hda2 = self.dataset_populator.new_dataset(history_id, content="samp1\t20.0\nsamp2\t40.0\n") hda3 = self.dataset_populator.new_dataset(history_id, content="samp1\t30.0\nsamp2\t60.0\n") self.dataset_populator.wait_for_history(history_id, assert_ok=True) inputs = { - '0': self._ds_entry(hda1), - '1': self._ds_entry(hda2), - '2': self._ds_entry(hda3), + "0": self._ds_entry(hda1), + "1": self._ds_entry(hda2), + "2": self._ds_entry(hda3), } invocation_id = self.__invoke_workflow(workflow_id, inputs=inputs, history_id=history_id) self.workflow_populator.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) @@ -1193,7 +1295,8 @@ steps: def test_workflow_run_dynamic_output_collections_3(self): # Test a workflow that create a list:list:list followed by a mapping step. with self.dataset_populator.test_history() as history_id: - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow inputs: text_input1: data @@ -1216,13 +1319,14 @@ steps: tool_id: cat in: input1: split_up_2/split_output -""") +""" + ) hda1 = self.dataset_populator.new_dataset(history_id, content="samp1\t10.0\nsamp2\t20.0\n") hda2 = self.dataset_populator.new_dataset(history_id, content="samp1\t30.0\nsamp2\t40.0\n") self.dataset_populator.wait_for_history(history_id, assert_ok=True) inputs = { - '0': self._ds_entry(hda1), - '1': self._ds_entry(hda2), + "0": self._ds_entry(hda1), + "1": self._ds_entry(hda2), } invocation_id = self.__invoke_workflow(workflow_id, inputs=inputs, history_id=history_id) self.workflow_populator.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) @@ -1240,11 +1344,12 @@ steps: assert current["tag"] == tag_test[count] count += 1 - @skip_without_tool('column_param') + @skip_without_tool("column_param") def test_empty_file_data_column_specified(self): # Regression test for https://github.com/galaxyproject/galaxy/pull/10981 with self.dataset_populator.test_history() as history_id: - self._run_jobs("""class: GalaxyWorkflow + self._run_jobs( + """class: GalaxyWorkflow steps: empty_output: tool_id: empty_output @@ -1258,13 +1363,16 @@ steps: state: col: 2 col_names: 'B' -""", history_id=history_id) +""", + history_id=history_id, + ) - @skip_without_tool('column_param_list') + @skip_without_tool("column_param_list") def test_comma_separated_columns(self): # Regression test for https://github.com/galaxyproject/galaxy/pull/10981 with self.dataset_populator.test_history() as history_id: - self._run_jobs("""class: GalaxyWorkflow + self._run_jobs( + """class: GalaxyWorkflow steps: empty_output: tool_id: empty_output @@ -1278,12 +1386,15 @@ steps: state: col: '2,3' col_names: 'B' -""", history_id=history_id) +""", + history_id=history_id, + ) - @skip_without_tool('column_param') + @skip_without_tool("column_param") def test_runtime_data_column_parameter(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs("""class: GalaxyWorkflow + self._run_jobs( + """class: GalaxyWorkflow inputs: bed_input: data steps: @@ -1307,7 +1418,9 @@ test_data: value: 1.bed file_type: bed type: File -""", history_id=history_id) +""", + history_id=history_id, + ) @skip_without_tool("mapper") @skip_without_tool("pileup") @@ -1315,7 +1428,8 @@ test_data: # Testing regression of # https://github.com/galaxyproject/galaxy/issues/1514 with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input_fastqs: collection @@ -1344,25 +1458,40 @@ test_data: reference: value: 1.fasta type: File -""", history_id=history_id) +""", + history_id=history_id, + ) def test_run_subworkflow_simple(self) -> None: with self.dataset_populator.test_history() as history_id: - summary = self._run_workflow(WORKFLOW_NESTED_SIMPLE, test_data=""" + summary = self._run_workflow( + WORKFLOW_NESTED_SIMPLE, + test_data=""" outer_input: value: 1.bed type: File -""", history_id=history_id) +""", + history_id=history_id, + ) invocation_id = summary.invocation_id content = self.dataset_populator.get_history_dataset_content(history_id) - self.assertEqual("chrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\nchrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\n", content) - steps = self.workflow_populator.get_invocation(invocation_id)['steps'] - assert sum(1 for step in steps if step['subworkflow_invocation_id'] is None) == 3 - subworkflow_invocation_id = [step['subworkflow_invocation_id'] for step in steps if step['subworkflow_invocation_id']][0] + self.assertEqual( + "chrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\nchrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\n", + content, + ) + steps = self.workflow_populator.get_invocation(invocation_id)["steps"] + assert sum(1 for step in steps if step["subworkflow_invocation_id"] is None) == 3 + subworkflow_invocation_id = [ + step["subworkflow_invocation_id"] for step in steps if step["subworkflow_invocation_id"] + ][0] subworkflow_invocation = self.workflow_populator.get_invocation(subworkflow_invocation_id) - assert [step for step in subworkflow_invocation['steps'] if step['order_index'] == 0][0]['workflow_step_label'] == 'inner_input' - assert [step for step in subworkflow_invocation['steps'] if step['order_index'] == 1][0]['workflow_step_label'] == 'random_lines' + assert [step for step in subworkflow_invocation["steps"] if step["order_index"] == 0][0][ + "workflow_step_label" + ] == "inner_input" + assert [step for step in subworkflow_invocation["steps"] if step["order_index"] == 1][0][ + "workflow_step_label" + ] == "random_lines" bco = self.workflow_populator.get_biocompute_object(invocation_id) self.workflow_populator.validate_biocompute_object(bco) @@ -1370,14 +1499,18 @@ outer_input: @skip_without_tool("random_lines1") def test_run_subworkflow_runtime_parameters(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(WORKFLOW_NESTED_RUNTIME_PARAMETER, test_data=""" + self._run_jobs( + WORKFLOW_NESTED_RUNTIME_PARAMETER, + test_data=""" step_parameters: '1': '1|num_lines': 2 outer_input: value: 1.bed type: File -""", history_id=history_id) +""", + history_id=history_id, + ) content = self.dataset_populator.get_history_dataset_content(history_id) assert len([x for x in content.split("\n") if x]) == 2 @@ -1475,7 +1608,7 @@ test_data: # Wait for the workflow to finish scheduling and ensure both the invocation # and the history are in valid states. - invocation_scheduled = self._wait_for_invocation_state(uploaded_workflow_id, invocation_id, 'scheduled') + invocation_scheduled = self._wait_for_invocation_state(uploaded_workflow_id, invocation_id, "scheduled") assert invocation_scheduled, "Workflow state is not scheduled..." self.dataset_populator.wait_for_history(history_id, assert_ok=True) @@ -1497,7 +1630,8 @@ test_data: content = self.dataset_populator.get_history_dataset_content(history_id) self.assertEqual( "chrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\nchrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\n", - content) + content, + ) run_test(NESTED_WORKFLOW_AUTO_LABELS_MODERN_SYNTAX) @@ -1505,7 +1639,8 @@ test_data: @skip_without_tool("collection_paired_test") def test_workflow_run_zip_collections(self): with self.dataset_populator.test_history() as history_id: - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow inputs: test_input_1: data @@ -1524,13 +1659,14 @@ steps: tool_id: collection_paired_test in: f1: zip_it/output -""") +""" + ) hda1 = self.dataset_populator.new_dataset(history_id, content="samp1\t10.0\nsamp2\t20.0\n") hda2 = self.dataset_populator.new_dataset(history_id, content="samp1\t20.0\nsamp2\t40.0\n") self.dataset_populator.wait_for_history(history_id, assert_ok=True) inputs = { - '0': self._ds_entry(hda1), - '1': self._ds_entry(hda2), + "0": self._ds_entry(hda1), + "1": self._ds_entry(hda2), } invocation_id = self.__invoke_workflow(workflow_id, inputs=inputs, history_id=history_id) self.workflow_populator.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) @@ -1540,7 +1676,8 @@ steps: @skip_without_tool("collection_paired_test") def test_workflow_flatten(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow steps: nested: @@ -1554,18 +1691,22 @@ steps: input: $link: nested/list_output join_identifier: '-' -""", test_data={}, history_id=history_id) +""", + test_data={}, + history_id=history_id, + ) details = self.dataset_populator.get_history_collection_details(history_id, hid=14) - assert details['collection_type'] == "list" + assert details["collection_type"] == "list" elements = details["elements"] - identifiers = [e['element_identifier'] for e in elements] + identifiers = [e["element_identifier"] for e in elements] assert len(identifiers) == 6 assert "oe1-ie1" in identifiers @skip_without_tool("collection_paired_test") def test_workflow_flatten_with_mapped_over_execution(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(r""" + self._run_jobs( + r""" class: GalaxyWorkflow inputs: input_fastqs: collection @@ -1585,17 +1726,19 @@ test_data: elements: - identifier: samp1 content: "0\n1" -""", history_id=history_id) - history = self._get(f'histories/{history_id}/contents').json() +""", + history_id=history_id, + ) + history = self._get(f"histories/{history_id}/contents").json() flattened_collection = history[-1] - assert flattened_collection['history_content_type'] == 'dataset_collection' - assert flattened_collection['collection_type'] == 'list' - assert flattened_collection['element_count'] == 2 + assert flattened_collection["history_content_type"] == "dataset_collection" + assert flattened_collection["collection_type"] == "list" + assert flattened_collection["element_count"] == 2 nested_collection = self.dataset_populator.get_history_collection_details(history_id, hid=3) - assert nested_collection['collection_type'] == 'list:list' - assert nested_collection['element_count'] == 1 - assert nested_collection['elements'][0]['object']['populated'] - assert nested_collection['elements'][0]['object']['element_count'] == 2 + assert nested_collection["collection_type"] == "list:list" + assert nested_collection["element_count"] == 1 + assert nested_collection["elements"][0]["object"]["populated"] + assert nested_collection["elements"][0]["object"]["element_count"] == 2 @skip_without_tool("cat") def test_workflow_invocation_report_1(self): @@ -1605,7 +1748,8 @@ input_1: type: File """ with self.dataset_populator.test_history() as history_id: - summary = self._run_workflow(""" + summary = self._run_workflow( + """ class: GalaxyWorkflow inputs: input_1: data @@ -1617,7 +1761,10 @@ steps: tool_id: cat in: input1: input_1 -""", test_data=test_data, history_id=history_id) +""", + test_data=test_data, + history_id=history_id, + ) workflow_id = summary.workflow_id invocation_id = summary.invocation_id report_json = self.workflow_populator.workflow_report_json(workflow_id, invocation_id) @@ -1633,9 +1780,7 @@ steps: def test_workflow_invocation_report_custom(self): with self.dataset_populator.test_history() as history_id: summary = self._run_workflow( - WORKFLOW_WITH_CUSTOM_REPORT_1, - test_data=WORKFLOW_WITH_CUSTOM_REPORT_1_TEST_DATA, - history_id=history_id + WORKFLOW_WITH_CUSTOM_REPORT_1, test_data=WORKFLOW_WITH_CUSTOM_REPORT_1_TEST_DATA, history_id=history_id ) workflow_id = summary.workflow_id invocation_id = summary.invocation_id @@ -1660,18 +1805,25 @@ steps: invocation_id = summary.invocation_id bco = self.workflow_populator.get_biocompute_object(invocation_id) self.workflow_populator.validate_biocompute_object(bco) - self.assertEqual(bco['provenance_domain']['name'], "Simple Workflow") + self.assertEqual(bco["provenance_domain"]["name"], "Simple Workflow") @skip_without_tool("__APPLY_RULES__") def test_workflow_run_apply_rules(self): with self.dataset_populator.test_history() as history_id: - self._run_workflow(WORKFLOW_WITH_RULES_1, history_id=history_id, wait=True, assert_ok=True, round_trip_format_conversion=True) + self._run_workflow( + WORKFLOW_WITH_RULES_1, + history_id=history_id, + wait=True, + assert_ok=True, + round_trip_format_conversion=True, + ) output_content = self.dataset_populator.get_history_collection_details(history_id, hid=6) rules_test_data.check_example_2(output_content, self.dataset_populator) def test_filter_failed_mapping(self): with self.dataset_populator.test_history() as history_id: - summary = self._run_workflow(""" + summary = self._run_workflow( + """ class: GalaxyWorkflow inputs: input_c: collection @@ -1694,7 +1846,8 @@ steps: state: input1: $link: filtered_collection -""", test_data=""" +""", + test_data=""" input_c: collection_type: list elements: @@ -1702,7 +1855,11 @@ input_c: content: "0" - identifier: i2 content: "1" -""", history_id=history_id, wait=True, assert_ok=False) +""", + history_id=history_id, + wait=True, + assert_ok=False, + ) jobs = summary.jobs def filter_jobs_by_tool(tool_id): @@ -1717,15 +1874,21 @@ input_c: def test_workflow_request(self): workflow = self.workflow_populator.load_workflow(name="test_for_queue") workflow_request, history_id, workflow_id = self._setup_workflow_run(workflow) - run_workflow_response = self.workflow_populator.invoke_workflow_raw(workflow_id, workflow_request, assert_ok=True) + run_workflow_response = self.workflow_populator.invoke_workflow_raw( + workflow_id, workflow_request, assert_ok=True + ) invocation_id = run_workflow_response.json()["id"] self.workflow_populator.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) def test_workflow_new_autocreated_history(self): workflow = self.workflow_populator.load_workflow(name="test_for_new_autocreated_history") workflow_request, history_id, workflow_id = self._setup_workflow_run(workflow) - del workflow_request['history'] # Not passing a history param means asking for a new history to be automatically created - run_workflow_dict = self.workflow_populator.invoke_workflow_raw(workflow_id, workflow_request, assert_ok=True).json() + del workflow_request[ + "history" + ] # Not passing a history param means asking for a new history to be automatically created + run_workflow_dict = self.workflow_populator.invoke_workflow_raw( + workflow_id, workflow_request, assert_ok=True + ).json() new_history_id = run_workflow_dict["history_id"] assert history_id != new_history_id invocation_id = run_workflow_dict["id"] @@ -1742,7 +1905,9 @@ input_c: self._assert_has_keys(invocation, "id", "outputs", "output_collections") assert len(invocation["output_collections"]) == 0 assert len(invocation["outputs"]) == 1 - output_content = self.dataset_populator.get_history_dataset_content(history_id, dataset_id=invocation["outputs"]["wf_output_1"]["id"]) + output_content = self.dataset_populator.get_history_dataset_content( + history_id, dataset_id=invocation["outputs"]["wf_output_1"]["id"] + ) assert "hello world" == output_content.strip() @skip_without_tool("cat") @@ -1757,7 +1922,9 @@ input_c: self._assert_has_keys(invocation, "id", "outputs", "output_collections") assert len(invocation["output_collections"]) == 1 assert len(invocation["outputs"]) == 0 - output_content = self.dataset_populator.get_history_collection_details(history_id, content_id=invocation["output_collections"]["wf_output_1"]["id"]) + output_content = self.dataset_populator.get_history_collection_details( + history_id, content_id=invocation["output_collections"]["wf_output_1"]["id"] + ) self._assert_has_keys(output_content, "id", "elements") assert output_content["collection_type"] == "list" elements = output_content["elements"] @@ -1766,7 +1933,8 @@ input_c: assert elements0["element_identifier"] == "el1" def _run_workflow_with_output_collections(self, history_id) -> RunJobsSummary: - summary = self._run_workflow(""" + summary = self._run_workflow( + """ class: GalaxyWorkflow inputs: input1: @@ -1780,7 +1948,8 @@ steps: tool_id: cat in: input1: input1 -""", test_data=""" +""", + test_data=""" input1: collection_type: list name: the_dataset_list @@ -1788,11 +1957,15 @@ input1: - identifier: el1 value: 1.fastq type: File -""", history_id=history_id, round_trip_format_conversion=True) +""", + history_id=history_id, + round_trip_format_conversion=True, + ) return summary def _run_workflow_with_inputs_as_outputs(self, history_id) -> RunJobsSummary: - summary = self._run_workflow(""" + summary = self._run_workflow( + """ class: GalaxyWorkflow inputs: input1: data @@ -1803,7 +1976,10 @@ outputs: wf_output_param: outputSource: text_input steps: [] -""", test_data={"input1": "hello world", "text_input": {"value": "A text variable", "type": "raw"}}, history_id=history_id) +""", + test_data={"input1": "hello world", "text_input": {"value": "A text variable", "type": "raw"}}, + history_id=history_id, + ) return summary def test_workflow_input_as_output(self): @@ -1820,12 +1996,15 @@ steps: [] assert len(invocation["output_values"]) == 1 assert "wf_output_param" in invocation["output_values"] assert invocation["output_values"]["wf_output_param"] == "A text variable", invocation["output_values"] - output_content = self.dataset_populator.get_history_dataset_content(history_id, content_id=invocation["outputs"]["wf_output_1"]["id"]) + output_content = self.dataset_populator.get_history_dataset_content( + history_id, content_id=invocation["outputs"]["wf_output_1"]["id"] + ) assert output_content == "hello world\n" def test_subworkflow_output_as_output(self): with self.dataset_populator.test_history() as history_id: - summary = self._run_workflow(""" + summary = self._run_workflow( + """ class: GalaxyWorkflow inputs: input1: data @@ -1844,7 +2023,10 @@ steps: steps: [] in: inner_input: input1 -""", test_data={"input1": "hello world"}, history_id=history_id) +""", + test_data={"input1": "hello world"}, + history_id=history_id, + ) workflow_id = summary.workflow_id invocation_id = summary.invocation_id invocation_response = self._get(f"workflows/{workflow_id}/invocations/{invocation_id}") @@ -1853,13 +2035,16 @@ steps: self._assert_has_keys(invocation, "id", "outputs", "output_collections") assert len(invocation["output_collections"]) == 0 assert len(invocation["outputs"]) == 1 - output_content = self.dataset_populator.get_history_dataset_content(history_id, content_id=invocation["outputs"]["wf_output_1"]["id"]) + output_content = self.dataset_populator.get_history_dataset_content( + history_id, content_id=invocation["outputs"]["wf_output_1"]["id"] + ) assert output_content == "hello world\n" @skip_without_tool("cat") def test_workflow_input_mapping(self): with self.dataset_populator.test_history() as history_id: - summary = self._run_workflow(""" + summary = self._run_workflow( + """ class: GalaxyWorkflow inputs: input1: data @@ -1871,7 +2056,8 @@ steps: tool_id: cat in: input1: input1 -""", test_data=""" +""", + test_data=""" input1: collection_type: list name: the_dataset_list @@ -1882,7 +2068,9 @@ input1: - identifier: el2 value: 1.fastq type: File -""", history_id=history_id) +""", + history_id=history_id, + ) workflow_id = summary.workflow_id invocation_id = summary.invocation_id invocation_response = self._get(f"workflows/{workflow_id}/invocations/{invocation_id}") @@ -1891,7 +2079,9 @@ input1: self._assert_has_keys(invocation, "id", "outputs", "output_collections") assert len(invocation["output_collections"]) == 1 assert len(invocation["outputs"]) == 0 - output_content = self.dataset_populator.get_history_collection_details(history_id, content_id=invocation["output_collections"]["wf_output_1"]["id"]) + output_content = self.dataset_populator.get_history_collection_details( + history_id, content_id=invocation["output_collections"]["wf_output_1"]["id"] + ) self._assert_has_keys(output_content, "id", "elements") elements = output_content["elements"] assert len(elements) == 2 @@ -1901,7 +2091,8 @@ input1: @skip_without_tool("collection_creates_pair") def test_workflow_run_input_mapping_with_output_collections(self): with self.dataset_populator.test_history() as history_id: - summary = self._run_workflow(""" + summary = self._run_workflow( + """ class: GalaxyWorkflow inputs: text_input: data @@ -1913,7 +2104,8 @@ steps: tool_id: collection_creates_pair in: input1: text_input -""", test_data=""" +""", + test_data=""" text_input: collection_type: list name: the_dataset_list @@ -1924,7 +2116,9 @@ text_input: - identifier: el2 value: 1.fastq type: File -""", history_id=history_id) +""", + history_id=history_id, + ) workflow_id = summary.workflow_id invocation_id = summary.invocation_id invocation_response = self._get(f"workflows/{workflow_id}/invocations/{invocation_id}") @@ -1933,7 +2127,9 @@ text_input: self._assert_has_keys(invocation, "id", "outputs", "output_collections") assert len(invocation["output_collections"]) == 1 assert len(invocation["outputs"]) == 0 - output_content = self.dataset_populator.get_history_collection_details(history_id, content_id=invocation["output_collections"]["wf_output_1"]["id"]) + output_content = self.dataset_populator.get_history_collection_details( + history_id, content_id=invocation["output_collections"]["wf_output_1"]["id"] + ) self._assert_has_keys(output_content, "id", "elements") assert output_content["collection_type"] == "list:paired", output_content elements = output_content["elements"] @@ -1946,24 +2142,24 @@ text_input: jobs_summary_response = self._get(f"workflows/{workflow_id}/invocations/{invocation_id}/jobs_summary") self._assert_status_code_is(jobs_summary_response, 200) jobs_summary = jobs_summary_response.json() - assert 'states' in jobs_summary + assert "states" in jobs_summary - invocation_states = jobs_summary['states'] - assert invocation_states and 'ok' in invocation_states, jobs_summary - assert invocation_states['ok'] == 2, jobs_summary - assert jobs_summary['model'] == 'WorkflowInvocation', jobs_summary + invocation_states = jobs_summary["states"] + assert invocation_states and "ok" in invocation_states, jobs_summary + assert invocation_states["ok"] == 2, jobs_summary + assert jobs_summary["model"] == "WorkflowInvocation", jobs_summary jobs_summary_response = self._get(f"workflows/{workflow_id}/invocations/{invocation_id}/step_jobs_summary") self._assert_status_code_is(jobs_summary_response, 200) jobs_summary = jobs_summary_response.json() assert len(jobs_summary) == 1 collection_summary = jobs_summary[0] - assert 'states' in collection_summary + assert "states" in collection_summary - collection_states = collection_summary['states'] - assert collection_states and 'ok' in collection_states, collection_states - assert collection_states['ok'] == 2, collection_summary - assert collection_summary['model'] == 'ImplicitCollectionJobs', collection_summary + collection_states = collection_summary["states"] + assert collection_states and "ok" in collection_states, collection_states + assert collection_states["ok"] == 2, collection_summary + assert collection_summary["model"] == "ImplicitCollectionJobs", collection_summary def test_workflow_run_input_mapping_with_subworkflows(self): with self.dataset_populator.test_history() as history_id: @@ -1990,7 +2186,9 @@ outer_input: self._assert_has_keys(invocation, "id", "outputs", "output_collections") assert len(invocation["output_collections"]) == 1, invocation assert len(invocation["outputs"]) == 0 - output_content = self.dataset_populator.get_history_collection_details(history_id, content_id=invocation["output_collections"]["outer_output"]["id"]) + output_content = self.dataset_populator.get_history_collection_details( + history_id, content_id=invocation["output_collections"]["outer_output"]["id"] + ) self._assert_has_keys(output_content, "id", "elements") assert output_content["collection_type"] == "list", output_content elements = output_content["elements"] @@ -2007,7 +2205,8 @@ outer_input: # evaluation. Testing rescheduling and propagating connections within a subworkflow # is handled by the next test case. with self.dataset_populator.test_history() as history_id: - self._run_workflow(""" + self._run_workflow( + """ class: GalaxyWorkflow inputs: outer_input: data @@ -2052,8 +2251,15 @@ test_data: outer_input: value: 1.bed type: File -""", history_id=history_id, wait=True, round_trip_format_conversion=True) - self.assertEqual("chr6\t108722976\t108723115\tCCDS5067.1_cds_0_0_chr6_108722977_f\t0\t+\nchrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\n", self.dataset_populator.get_history_dataset_content(history_id)) +""", + history_id=history_id, + wait=True, + round_trip_format_conversion=True, + ) + self.assertEqual( + "chr6\t108722976\t108723115\tCCDS5067.1_cds_0_0_chr6_108722977_f\t0\t+\nchrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\n", + self.dataset_populator.get_history_dataset_content(history_id), + ) # self.assertEqual("chr16\t142908\t143003\tCCDS10397.1_cds_0_0_chr16_142909_f\t0\t+\nchrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\n", self.dataset_populator.get_history_dataset_content(history_id)) @skip_without_tool("cat_list") @@ -2065,7 +2271,8 @@ test_data: # delayed, but this also tests recovering and handling scheduling within the subworkflow # since the delayed step (split) isn't the last step of the subworkflow. with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: outer_input: data @@ -2110,19 +2317,28 @@ steps: tool_id: cat_list in: input1: nested_workflow/workflow_output -""", test_data=""" +""", + test_data=""" outer_input: value: 1.bed type: File -""", history_id=history_id, wait=True, round_trip_format_conversion=True) - self.assertEqual("chr6\t108722976\t108723115\tCCDS5067.1_cds_0_0_chr6_108722977_f\t0\t+\nchrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\n", self.dataset_populator.get_history_dataset_content(history_id)) +""", + history_id=history_id, + wait=True, + round_trip_format_conversion=True, + ) + self.assertEqual( + "chr6\t108722976\t108723115\tCCDS5067.1_cds_0_0_chr6_108722977_f\t0\t+\nchrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\n", + self.dataset_populator.get_history_dataset_content(history_id), + ) @skip_without_tool("cat_list") @skip_without_tool("random_lines1") @skip_without_tool("split") def test_recover_mapping_in_subworkflow(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: outer_input: data @@ -2162,19 +2378,28 @@ steps: tool_id: cat_list in: input1: nested_workflow/workflow_output -""", test_data=""" +""", + test_data=""" outer_input: value: 1.bed type: File -""", history_id=history_id, wait=True, round_trip_format_conversion=True) - self.assertEqual("chr6\t108722976\t108723115\tCCDS5067.1_cds_0_0_chr6_108722977_f\t0\t+\nchrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\n", self.dataset_populator.get_history_dataset_content(history_id)) +""", + history_id=history_id, + wait=True, + round_trip_format_conversion=True, + ) + self.assertEqual( + "chr6\t108722976\t108723115\tCCDS5067.1_cds_0_0_chr6_108722977_f\t0\t+\nchrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\n", + self.dataset_populator.get_history_dataset_content(history_id), + ) @skip_without_tool("empty_list") @skip_without_tool("count_list") @skip_without_tool("random_lines1") def test_empty_list_mapping(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -2199,17 +2424,22 @@ steps: tool_id: count_list in: input1: random_lines/out_file1 -""", test_data=""" +""", + test_data=""" input1: value: 1.bed type: File -""", history_id=history_id, wait=True) +""", + history_id=history_id, + wait=True, + ) self.assertEqual("0\n", self.dataset_populator.get_history_dataset_content(history_id)) @skip_without_tool("random_lines1") def test_change_datatype_collection_map_over(self): with self.dataset_populator.test_history() as history_id: - jobs_summary = self._run_workflow(""" + jobs_summary = self._run_workflow( + """ class: GalaxyWorkflow inputs: text_input1: collection @@ -2221,21 +2451,25 @@ steps: outputs: out_file1: change_datatype: csv -""", test_data=""" +""", + test_data=""" text_input1: collection_type: "list:paired" -""", history_id=history_id) +""", + history_id=history_id, + ) hdca = self.dataset_populator.get_history_collection_details(history_id=jobs_summary.history_id, hid=4) - assert hdca['collection_type'] == 'list:paired' - assert len(hdca['elements'][0]['object']["elements"]) == 2 - forward, reverse = hdca['elements'][0]['object']["elements"] - assert forward['object']['file_ext'] == 'csv' - assert reverse['object']['file_ext'] == 'csv' + assert hdca["collection_type"] == "list:paired" + assert len(hdca["elements"][0]["object"]["elements"]) == 2 + forward, reverse = hdca["elements"][0]["object"]["elements"] + assert forward["object"]["file_ext"] == "csv" + assert reverse["object"]["file_ext"] == "csv" @skip_without_tool("collection_type_source_map_over") def test_mapping_and_subcollection_mapping(self): with self.dataset_populator.test_history() as history_id: - jobs_summary = self._run_workflow(""" + jobs_summary = self._run_workflow( + """ class: GalaxyWorkflow inputs: text_input1: collection @@ -2244,20 +2478,24 @@ steps: tool_id: collection_type_source_map_over in: input_collect: text_input1 -""", test_data=""" +""", + test_data=""" text_input1: collection_type: "list:paired" -""", history_id=history_id) +""", + history_id=history_id, + ) hdca = self.dataset_populator.get_history_collection_details(history_id=jobs_summary.history_id, hid=1) - assert hdca['collection_type'] == 'list:paired' - assert len(hdca['elements'][0]['object']["elements"]) == 2 + assert hdca["collection_type"] == "list:paired" + assert len(hdca["elements"][0]["object"]["elements"]) == 2 @skip_without_tool("empty_list") @skip_without_tool("count_multi_file") @skip_without_tool("random_lines1") def test_empty_list_reduction(self): with self.dataset_populator.test_history() as history_id: - self._run_workflow(""" + self._run_workflow( + """ class: GalaxyWorkflow inputs: input1: data @@ -2282,11 +2520,16 @@ steps: tool_id: count_multi_file in: input1: random_lines/out_file1 -""", test_data=""" +""", + test_data=""" input1: value: 1.bed type: File -""", history_id=history_id, wait=True, round_trip_format_conversion=True) +""", + history_id=history_id, + wait=True, + round_trip_format_conversion=True, + ) self.assertEqual("0\n", self.dataset_populator.get_history_dataset_content(history_id)) @skip_without_tool("cat") @@ -2305,7 +2548,7 @@ input1: self._delete(f"histories/{history_id}") - invocation_cancelled = self._wait_for_invocation_state(uploaded_workflow_id, invocation_id, 'cancelled') + invocation_cancelled = self._wait_for_invocation_state(uploaded_workflow_id, invocation_id, "cancelled") assert invocation_cancelled, "Workflow state is not cancelled..." @skip_without_tool("cat") @@ -2325,7 +2568,7 @@ input1: self._delete(f"histories/{history_id}") - invocation_cancelled = self._wait_for_invocation_state(uploaded_workflow_id, invocation_id, 'cancelled') + invocation_cancelled = self._wait_for_invocation_state(uploaded_workflow_id, invocation_id, "cancelled") assert invocation_cancelled, "Workflow state is not cancelled..." @skip_without_tool("cat") @@ -2349,7 +2592,7 @@ input1: # Wait for the workflow to finish scheduling and ensure both the invocation # and the history are in valid states. - invocation_scheduled = self._wait_for_invocation_state(uploaded_workflow_id, invocation_id, 'scheduled') + invocation_scheduled = self._wait_for_invocation_state(uploaded_workflow_id, invocation_id, "scheduled") assert invocation_scheduled, "Workflow state is not scheduled..." self.dataset_populator.wait_for_history(history_id, assert_ok=True) @@ -2373,7 +2616,7 @@ input1: self.__review_paused_steps(uploaded_workflow_id, invocation_id, order_index=2, action=False) # Ensure the workflow eventually becomes cancelled. - invocation_cancelled = self._wait_for_invocation_state(uploaded_workflow_id, invocation_id, 'cancelled') + invocation_cancelled = self._wait_for_invocation_state(uploaded_workflow_id, invocation_id, "cancelled") assert invocation_cancelled, "Workflow state is not cancelled..." @skip_without_tool("head") @@ -2382,10 +2625,12 @@ input1: workflow = self.workflow_populator.load_workflow_from_resource("test_workflow_map_reduce_pause") uploaded_workflow_id = self.workflow_populator.create_workflow(workflow) hda1 = self.dataset_populator.new_dataset(history_id, content="reviewed\nunreviewed") - hdca1 = self.dataset_collection_populator.create_list_in_history(history_id, contents=["1\n2\n3", "4\n5\n6"]).json() + hdca1 = self.dataset_collection_populator.create_list_in_history( + history_id, contents=["1\n2\n3", "4\n5\n6"] + ).json() index_map = { - '0': self._ds_entry(hda1), - '1': self._ds_entry(hdca1), + "0": self._ds_entry(hda1), + "1": self._ds_entry(hdca1), } invocation_id = self.__invoke_workflow(uploaded_workflow_id, inputs=index_map, history_id=history_id) @@ -2402,8 +2647,10 @@ input1: self.__review_paused_steps(uploaded_workflow_id, invocation_id, order_index=4, action=True) self.workflow_populator.wait_for_invocation_and_jobs(history_id, uploaded_workflow_id, invocation_id) invocation = self._invocation_details(uploaded_workflow_id, invocation_id) - assert invocation['state'] == 'scheduled' - self.assertEqual("reviewed\n1\nreviewed\n4\n", self.dataset_populator.get_history_dataset_content(history_id)) + assert invocation["state"] == "scheduled" + self.assertEqual( + "reviewed\n1\nreviewed\n4\n", self.dataset_populator.get_history_dataset_content(history_id) + ) @skip_without_tool("cat") def test_cancel_workflow_invocation(self): @@ -2426,7 +2673,7 @@ input1: self._assert_status_code_is(delete_response, 200) invocation = self._invocation_details(uploaded_workflow_id, invocation_id) - assert invocation['state'] == 'cancelled' + assert invocation["state"] == "cancelled" @skip_without_tool("cat") def test_pause_outputs_with_deleted_inputs(self): @@ -2439,7 +2686,8 @@ input1: def _deleted_inputs_workflow(self, purge): # We run a workflow on a collection with a deleted element. with self.dataset_populator.test_history() as history_id: - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow inputs: input1: @@ -2454,14 +2702,16 @@ steps: tool_id: cat in: input1: first_cat/out_file1 -""") +""" + ) DELETED = 0 PAUSED_1 = 3 PAUSED_2 = 5 - hdca1 = self.dataset_collection_populator.create_list_in_history(history_id, - contents=[("sample1-1", "1 2 3")]).json() + hdca1 = self.dataset_collection_populator.create_list_in_history( + history_id, contents=[("sample1-1", "1 2 3")] + ).json() self.dataset_populator.wait_for_history(history_id, assert_ok=True) - deleted_id = hdca1['elements'][DELETED]['object']['id'] + deleted_id = hdca1["elements"][DELETED]["object"]["id"] r = self._delete(f"histories/{history_id}/contents/{deleted_id}?purge={purge}") label_map = {"input1": self._ds_entry(hdca1)} workflow_request = dict( @@ -2473,19 +2723,22 @@ steps: invocation_id = r.json()["id"] # If this starts failing we may have prevented running workflows on collections with deleted members, # in which case we can disable this test. - self.workflow_populator.wait_for_invocation_and_jobs(workflow_id, history_id, invocation_id, assert_ok=False) + self.workflow_populator.wait_for_invocation_and_jobs( + workflow_id, history_id, invocation_id, assert_ok=False + ) # Why is this sleep needed? -John if not purge: time.sleep(5) contents = self.__history_contents(history_id) - assert contents[DELETED]['deleted'] - state = 'error' if purge else 'paused' - assert contents[PAUSED_1]['state'] == state - assert contents[PAUSED_2]['state'] == 'paused' + assert contents[DELETED]["deleted"] + state = "error" if purge else "paused" + assert contents[PAUSED_1]["state"] == state + assert contents[PAUSED_2]["state"] == "paused" def test_run_with_implicit_connection(self): with self.dataset_populator.test_history() as history_id: - run_summary = self._run_workflow(""" + run_summary = self._run_workflow( + """ class: GalaxyWorkflow inputs: test_input: data @@ -2513,7 +2766,12 @@ steps: seed_source: seed_source_selector: set_seed seed: asdf -""", test_data={"test_input": "hello world"}, history_id=history_id, wait=False, round_trip_format_conversion=True) +""", + test_data={"test_input": "hello world"}, + history_id=history_id, + wait=False, + round_trip_format_conversion=True, + ) history_id = run_summary.history_id workflow_id = run_summary.workflow_id invocation_id = run_summary.invocation_id @@ -2521,7 +2779,7 @@ steps: wait_on(lambda: len(self._history_jobs(history_id)) >= 2 or None, "history jobs") self.dataset_populator.wait_for_history(history_id, assert_ok=True) invocation = self._invocation_details(workflow_id, invocation_id) - assert invocation['state'] != 'scheduled', invocation + assert invocation["state"] != "scheduled", invocation # Expect two jobs - the upload and first cat. randomlines shouldn't run # it is implicitly dependent on second cat. self._assert_history_job_count(history_id, 2) @@ -2532,28 +2790,45 @@ steps: def test_run_with_optional_data_specified_to_multi_data(self): with self.dataset_populator.test_history() as history_id: - self._run_workflow(WORKFLOW_OPTIONAL_TRUE_INPUT_DATA, test_data=""" + self._run_workflow( + WORKFLOW_OPTIONAL_TRUE_INPUT_DATA, + test_data=""" input1: value: 1.bed type: File -""", history_id=history_id, wait=True, assert_ok=True) +""", + history_id=history_id, + wait=True, + assert_ok=True, + ) content = self.dataset_populator.get_history_dataset_content(history_id) assert "CCDS989.1_cds_0_0_chr1_147962193_r" in content def test_run_with_optional_data_unspecified_to_multi_data(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(WORKFLOW_OPTIONAL_TRUE_INPUT_DATA, test_data={}, history_id=history_id, wait=True, assert_ok=True) + self._run_jobs( + WORKFLOW_OPTIONAL_TRUE_INPUT_DATA, test_data={}, history_id=history_id, wait=True, assert_ok=True + ) content = self.dataset_populator.get_history_dataset_content(history_id) assert "No input selected" in content def test_run_with_non_optional_data_unspecified_fails_invocation(self): with self.dataset_populator.test_history() as history_id: - error = self._run_jobs(WORKFLOW_OPTIONAL_FALSE_INPUT_DATA, test_data={}, history_id=history_id, wait=False, assert_ok=False, expected_response=400) + error = self._run_jobs( + WORKFLOW_OPTIONAL_FALSE_INPUT_DATA, + test_data={}, + history_id=history_id, + wait=False, + assert_ok=False, + expected_response=400, + ) self._assert_failed_on_non_optional_input(error, "input1") def test_run_with_optional_collection_specified(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(WORKFLOW_OPTIONAL_TRUE_INPUT_COLLECTION, test_data=""" + self._run_jobs( + WORKFLOW_OPTIONAL_TRUE_INPUT_COLLECTION, + test_data=""" input1: collection_type: paired name: the_dataset_pair @@ -2564,19 +2839,32 @@ input1: - identifier: reverse value: 1.fastq type: File -""", history_id=history_id, wait=True, assert_ok=True) +""", + history_id=history_id, + wait=True, + assert_ok=True, + ) content = self.dataset_populator.get_history_dataset_content(history_id) assert "GAATTGATCAGGACATAGGACAACTGTAGGCACCAT" in content def test_run_with_optional_collection_unspecified(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(WORKFLOW_OPTIONAL_TRUE_INPUT_COLLECTION, test_data={}, history_id=history_id, wait=True, assert_ok=True) + self._run_jobs( + WORKFLOW_OPTIONAL_TRUE_INPUT_COLLECTION, test_data={}, history_id=history_id, wait=True, assert_ok=True + ) content = self.dataset_populator.get_history_dataset_content(history_id) assert "No input specified." in content def test_run_with_non_optional_collection_unspecified_fails_invocation(self): with self.dataset_populator.test_history() as history_id: - error = self._run_jobs(WORKFLOW_OPTIONAL_FALSE_INPUT_COLLECTION, test_data={}, history_id=history_id, wait=False, assert_ok=False, expected_response=400) + error = self._run_jobs( + WORKFLOW_OPTIONAL_FALSE_INPUT_COLLECTION, + test_data={}, + history_id=history_id, + wait=False, + assert_ok=False, + expected_response=400, + ) self._assert_failed_on_non_optional_input(error, "input1") def _assert_failed_on_non_optional_input(self, error, input_name): @@ -2587,7 +2875,8 @@ input1: def test_run_with_validated_parameter_connection_optional(self): with self.dataset_populator.test_history() as history_id: - self._run_workflow(""" + self._run_workflow( + """ class: GalaxyWorkflow inputs: text_input: text @@ -2598,77 +2887,109 @@ steps: r2: - text: $link: text_input -""", test_data=""" +""", + test_data=""" text_input: value: "abd" type: raw -""", history_id=history_id, wait=True, round_trip_format_conversion=True) +""", + history_id=history_id, + wait=True, + round_trip_format_conversion=True, + ) jobs = self._history_jobs(history_id) assert len(jobs) == 1 def test_run_with_required_text_parameter_not_provided_fails(self): with self.dataset_populator.test_history() as history_id: try: - self._run_workflow(""" + self._run_workflow( + """ class: GalaxyWorkflow inputs: text_input: text steps: [] -""", assert_ok=True, history_id=history_id) +""", + assert_ok=True, + history_id=history_id, + ) except AssertionError as e: - assert '(text_input) is not optional' in str(e) + assert "(text_input) is not optional" in str(e) def test_run_with_required_text_parameter_null_fails(self): with self.dataset_populator.test_history() as history_id: try: - self._run_workflow(""" + self._run_workflow( + """ class: GalaxyWorkflow inputs: text_input: text steps: [] -""", test_data=""" +""", + test_data=""" text_input: value: null type: raw -""", assert_ok=True, history_id=history_id) +""", + assert_ok=True, + history_id=history_id, + ) except AssertionError as e: - assert '(text_input) is not optional' in str(e) + assert "(text_input) is not optional" in str(e) def test_run_with_int_parameter(self): with self.dataset_populator.test_history() as history_id: failed = False try: - self._run_jobs(WORKFLOW_PARAMETER_INPUT_INTEGER_REQUIRED, test_data=""" + self._run_jobs( + WORKFLOW_PARAMETER_INPUT_INTEGER_REQUIRED, + test_data=""" data_input: value: 1.bed type: File -""", history_id=history_id, wait=True, assert_ok=True) +""", + history_id=history_id, + wait=True, + assert_ok=True, + ) except AssertionError as e: - assert '(int_input) is not optional' in str(e) + assert "(int_input) is not optional" in str(e) failed = True assert failed - run_response = self._run_workflow(WORKFLOW_PARAMETER_INPUT_INTEGER_REQUIRED, test_data=""" + run_response = self._run_workflow( + WORKFLOW_PARAMETER_INPUT_INTEGER_REQUIRED, + test_data=""" data_input: value: 1.bed type: File int_input: value: 1 type: raw -""", history_id=history_id, wait=True, assert_ok=True) +""", + history_id=history_id, + wait=True, + assert_ok=True, + ) # self.dataset_populator.wait_for_history(history_id, assert_ok=True) content = self.dataset_populator.get_history_dataset_content(history_id) assert len(content.splitlines()) == 1, content invocation = self.workflow_populator.get_invocation(run_response.invocation_id) - assert invocation['input_step_parameters']['int_input']['parameter_value'] == 1 + assert invocation["input_step_parameters"]["int_input"]["parameter_value"] == 1 - run_response = self._run_workflow(WORKFLOW_PARAMETER_INPUT_INTEGER_OPTIONAL, test_data=""" + run_response = self._run_workflow( + WORKFLOW_PARAMETER_INPUT_INTEGER_OPTIONAL, + test_data=""" data_input: value: 1.bed type: File -""", history_id=history_id, wait=True, assert_ok=True) +""", + history_id=history_id, + wait=True, + assert_ok=True, + ) invocation = self.workflow_populator.get_invocation(run_response.invocation_id) # Optional step parameter without default value will not be recorded. - assert 'int_input' not in invocation['input_step_parameters'] + assert "int_input" not in invocation["input_step_parameters"] def test_run_with_int_parameter_nested(self): with self.dataset_populator.test_history() as history_id: @@ -2676,29 +2997,38 @@ data_input: workflow_id = self.workflow_populator.create_workflow(workflow) hda: dict = self.dataset_populator.new_dataset(history_id, content="1 2 3") workflow_request = { - 'history_id': history_id, - 'inputs_by': 'name', - 'inputs': json.dumps({ - 'input_dataset': {'src': 'hda', 'id': hda['id']}, - 'int_parameter': 1, - }) + "history_id": history_id, + "inputs_by": "name", + "inputs": json.dumps( + { + "input_dataset": {"src": "hda", "id": hda["id"]}, + "int_parameter": 1, + } + ), } self.workflow_populator.invoke_workflow_and_wait(workflow_id, request=workflow_request, assert_ok=True) def test_run_with_validated_parameter_connection_default_values(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(WORKFLOW_PARAMETER_INPUT_INTEGER_DEFAULT, test_data=""" + self._run_jobs( + WORKFLOW_PARAMETER_INPUT_INTEGER_DEFAULT, + test_data=""" data_input: value: 1.bed type: File -""", history_id=history_id, wait=True, assert_ok=True) +""", + history_id=history_id, + wait=True, + assert_ok=True, + ) self.dataset_populator.wait_for_history(history_id, assert_ok=True) content = self.dataset_populator.get_history_dataset_content(history_id) assert len(content.splitlines()) == 3, content def test_run_with_validated_parameter_connection_invalid(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: text_input: text @@ -2709,15 +3039,21 @@ steps: r2: - text: $link: text_input -""", test_data=""" +""", + test_data=""" text_input: value: "" type: raw -""", history_id=history_id, wait=True, assert_ok=False) +""", + history_id=history_id, + wait=True, + assert_ok=False, + ) def test_run_with_text_input_connection(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: data_input: data @@ -2733,14 +3069,17 @@ steps: seed_source_selector: set_seed seed: $link: text_input -""", test_data=""" +""", + test_data=""" data_input: value: 1.bed type: File text_input: value: asdf type: raw -""", history_id=history_id) +""", + history_id=history_id, + ) self.dataset_populator.wait_for_history(history_id, assert_ok=True) content = self.dataset_populator.get_history_dataset_content(history_id) @@ -2748,7 +3087,8 @@ text_input: def test_run_with_numeric_input_connection(self): history_id = self.dataset_populator.new_history() - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow steps: - label: forty_two @@ -2761,7 +3101,9 @@ steps: inttest: $link: forty_two/out1 test_data: {} -""", history_id=history_id) +""", + history_id=history_id, + ) self.dataset_populator.wait_for_history(history_id, assert_ok=True) content = self.dataset_populator.get_history_dataset_content(history_id) @@ -2771,12 +3113,13 @@ test_data: {} str_4point14 = lines[2] assert lines[3] == "" assert int(str_43) == 43 - assert abs(float(str_4point14) - 4.14) < .0001 + assert abs(float(str_4point14) - 4.14) < 0.0001 @skip_without_tool("param_value_from_file") def test_expression_tool_map_over(self): history_id = self.dataset_populator.new_history() - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: text_input1: collection @@ -2800,23 +3143,34 @@ test_data: content: A - identifier: B content: B -""", history_id=history_id) - history_contents = self._get(f'histories/{history_id}/contents').json() - collection = [c for c in history_contents if c['history_content_type'] == 'dataset_collection' and c['name'] == 'replaced_param_collection'][0] - collection_details = self._get(collection['url']).json() - assert collection_details['element_count'] == 2 - elements = collection_details['elements'] - assert elements[0]['element_identifier'] == 'A' - assert elements[1]['element_identifier'] == 'B' - element_a_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=elements[0]['object']) - element_b_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=elements[1]['object']) - assert element_a_content.strip() == 'A' - assert element_b_content.strip() == 'B' +""", + history_id=history_id, + ) + history_contents = self._get(f"histories/{history_id}/contents").json() + collection = [ + c + for c in history_contents + if c["history_content_type"] == "dataset_collection" and c["name"] == "replaced_param_collection" + ][0] + collection_details = self._get(collection["url"]).json() + assert collection_details["element_count"] == 2 + elements = collection_details["elements"] + assert elements[0]["element_identifier"] == "A" + assert elements[1]["element_identifier"] == "B" + element_a_content = self.dataset_populator.get_history_dataset_content( + history_id, dataset=elements[0]["object"] + ) + element_b_content = self.dataset_populator.get_history_dataset_content( + history_id, dataset=elements[1]["object"] + ) + assert element_a_content.strip() == "A" + assert element_b_content.strip() == "B" - @skip_without_tool('create_input_collection') + @skip_without_tool("create_input_collection") def test_workflow_optional_input_text_parameter_reevaluation(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: text_input: @@ -2859,28 +3213,34 @@ steps: outputs: out_file1: rename: "#{inner_text_input} suffix" - """, history_id=history_id) + """, + history_id=history_id, + ) - @skip_without_tool('cat1') + @skip_without_tool("cat1") def test_workflow_rerun_with_use_cached_job(self): workflow = self.workflow_populator.load_workflow(name="test_for_run") # We launch a workflow with self.dataset_populator.test_history() as history_id_one, self.dataset_populator.test_history() as history_id_two: workflow_request, _, workflow_id = self._setup_workflow_run(workflow, history_id=history_id_one) - invocation_id = self.workflow_populator.invoke_workflow_and_wait(workflow_id, request=workflow_request, assert_ok=True) + invocation_id = self.workflow_populator.invoke_workflow_and_wait( + workflow_id, request=workflow_request, assert_ok=True + ) invocation_1 = self.workflow_populator.get_invocation(invocation_id) # We copy the workflow inputs to a new history new_workflow_request = workflow_request.copy() - new_ds_map = json.loads(new_workflow_request['ds_map']) - for key, input_values in invocation_1['inputs'].items(): - copy_payload = {"content": input_values['id'], "source": "hda", "type": "dataset"} + new_ds_map = json.loads(new_workflow_request["ds_map"]) + for key, input_values in invocation_1["inputs"].items(): + copy_payload = {"content": input_values["id"], "source": "hda", "type": "dataset"} copy_response = self._post(f"histories/{history_id_two}/contents", data=copy_payload, json=True).json() - new_ds_map[key]['id'] = copy_response['id'] - new_workflow_request['ds_map'] = json.dumps(new_ds_map, sort_keys=True) - new_workflow_request['history'] = f"hist_id={history_id_two}" - new_workflow_request['use_cached_job'] = True + new_ds_map[key]["id"] = copy_response["id"] + new_workflow_request["ds_map"] = json.dumps(new_ds_map, sort_keys=True) + new_workflow_request["history"] = f"hist_id={history_id_two}" + new_workflow_request["use_cached_job"] = True # We run the workflow again, it should not produce any new outputs - new_workflow_response = self.workflow_populator.invoke_workflow_raw(workflow_id, new_workflow_request, assert_ok=True).json() + new_workflow_response = self.workflow_populator.invoke_workflow_raw( + workflow_id, new_workflow_request, assert_ok=True + ).json() invocation_id = new_workflow_response["id"] self.workflow_populator.wait_for_invocation_and_jobs(history_id_two, workflow_id, invocation_id) @@ -2892,10 +3252,11 @@ steps: first_wf_output = self._get(f"datasets/{first_wf_output_hda['id']}").json() second_wf_output = self._get(f"datasets/{second_wf_output_hda['id']}").json() - assert first_wf_output['file_name'] == second_wf_output['file_name'], \ - f"first output:\n{first_wf_output}\nsecond output:\n{second_wf_output}" + assert ( + first_wf_output["file_name"] == second_wf_output["file_name"] + ), f"first output:\n{first_wf_output}\nsecond output:\n{second_wf_output}" - @skip_without_tool('cat1') + @skip_without_tool("cat1") def test_nested_workflow_rerun_with_use_cached_job(self): with self.dataset_populator.test_history() as history_id_one, self.dataset_populator.test_history() as history_id_two: test_data = """ @@ -2903,31 +3264,38 @@ outer_input: value: 1.bed type: File """ - run_jobs_summary = self._run_workflow(WORKFLOW_NESTED_SIMPLE, test_data=test_data, history_id=history_id_one) + run_jobs_summary = self._run_workflow( + WORKFLOW_NESTED_SIMPLE, test_data=test_data, history_id=history_id_one + ) workflow_id = run_jobs_summary.workflow_id workflow_request = run_jobs_summary.workflow_request # We copy the inputs to a new history and re-run the workflow - inputs = json.loads(workflow_request['inputs']) - dataset_type = inputs['outer_input']['src'] - dataset_id = inputs['outer_input']['id'] + inputs = json.loads(workflow_request["inputs"]) + dataset_type = inputs["outer_input"]["src"] + dataset_id = inputs["outer_input"]["id"] copy_payload = {"content": dataset_id, "source": dataset_type, "type": "dataset"} copy_response = self._post(f"histories/{history_id_two}/contents", data=copy_payload, json=True) self._assert_status_code_is(copy_response, 200) - new_dataset_id = copy_response.json()['id'] - inputs['outer_input']['id'] = new_dataset_id - workflow_request['use_cached_job'] = True - workflow_request['history'] = f"hist_id={history_id_two}" - workflow_request['inputs'] = json.dumps(inputs) - self.workflow_populator.invoke_workflow_and_wait(workflow_id, request=run_jobs_summary.workflow_request, assert_ok=True) + new_dataset_id = copy_response.json()["id"] + inputs["outer_input"]["id"] = new_dataset_id + workflow_request["use_cached_job"] = True + workflow_request["history"] = f"hist_id={history_id_two}" + workflow_request["inputs"] = json.dumps(inputs) + self.workflow_populator.invoke_workflow_and_wait( + workflow_id, request=run_jobs_summary.workflow_request, assert_ok=True + ) # Now make sure that the HDAs in each history point to the same dataset instances history_one_contents = self.__history_contents(history_id_one) history_two_contents = self.__history_contents(history_id_two) assert len(history_one_contents) == len(history_two_contents) for i, (item_one, item_two) in enumerate(zip(history_one_contents, history_two_contents)): - assert item_one['dataset_id'] == item_two['dataset_id'], \ - 'Dataset ids should match, but "%s" and "%s" are not the same for History item %i.' % (item_one['dataset_id'], - item_two['dataset_id'], - i + 1) + assert ( + item_one["dataset_id"] == item_two["dataset_id"] + ), 'Dataset ids should match, but "%s" and "%s" are not the same for History item %i.' % ( + item_one["dataset_id"], + item_two["dataset_id"], + i + 1, + ) def test_cannot_run_inaccessible_workflow(self): workflow = self.workflow_populator.load_workflow(name="test_for_run_cannot_access") @@ -2963,15 +3331,23 @@ outer_input: workflow = self.workflow_populator.load_workflow_from_resource("test_workflow_matching_lists") workflow_id = self.workflow_populator.create_workflow(workflow) with self.dataset_populator.test_history() as history_id: - hdca1 = self.dataset_collection_populator.create_list_in_history(history_id, contents=[("sample1-1", "1 2 3"), ("sample2-1", "7 8 9")]).json() - hdca2 = self.dataset_collection_populator.create_list_in_history(history_id, contents=[("sample1-2", "4 5 6"), ("sample2-2", "0 a b")]).json() + hdca1 = self.dataset_collection_populator.create_list_in_history( + history_id, contents=[("sample1-1", "1 2 3"), ("sample2-1", "7 8 9")] + ).json() + hdca2 = self.dataset_collection_populator.create_list_in_history( + history_id, contents=[("sample1-2", "4 5 6"), ("sample2-2", "0 a b")] + ).json() self.dataset_populator.wait_for_history(history_id, assert_ok=True) label_map = {"list1": self._ds_entry(hdca1), "list2": self._ds_entry(hdca2)} workflow_request = dict( ds_map=self.workflow_populator.build_ds_map(workflow_id, label_map), ) - self.workflow_populator.invoke_workflow_and_wait(workflow_id, history_id=history_id, request=workflow_request, assert_ok=True) - self.assertEqual("1 2 3\n4 5 6\n7 8 9\n0 a b\n", self.dataset_populator.get_history_dataset_content(history_id)) + self.workflow_populator.invoke_workflow_and_wait( + workflow_id, history_id=history_id, request=workflow_request, assert_ok=True + ) + self.assertEqual( + "1 2 3\n4 5 6\n7 8 9\n0 a b\n", self.dataset_populator.get_history_dataset_content(history_id) + ) def test_workflow_stability(self): # Run this index stability test with following command: @@ -3001,10 +3377,7 @@ outer_input: self._assert_error_code_is(response, error_codes.error_codes_by_name["USER_REQUEST_MISSING_PARAMETER"]) def test_invalid_create_multiple_types(self): - data = { - 'shared_workflow_id': '1234567890abcdef', - 'from_history_id': '1234567890abcdef' - } + data = {"shared_workflow_id": "1234567890abcdef", "from_history_id": "1234567890abcdef"} response = self._post("workflows", data) self._assert_status_code_is(response, 400) self._assert_error_code_is(response, error_codes.error_codes_by_name["USER_REQUEST_INVALID_PARAMETER"]) @@ -3012,9 +3385,11 @@ outer_input: @skip_without_tool("cat1") def test_run_with_pja(self): workflow = self.workflow_populator.load_workflow(name="test_for_pja_run", add_pja=True) - workflow_request, history_id, workflow_id = self._setup_workflow_run(workflow, inputs_by='step_index') + workflow_request, history_id, workflow_id = self._setup_workflow_run(workflow, inputs_by="step_index") workflow_request["replacement_params"] = dumps(dict(replaceme="was replaced")) - run_workflow_response = self.workflow_populator.invoke_workflow_raw(workflow_id, workflow_request, assert_ok=True) + run_workflow_response = self.workflow_populator.invoke_workflow_raw( + workflow_id, workflow_request, assert_ok=True + ) invocation_id = run_workflow_response.json()["id"] self.workflow_populator.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id, assert_ok=True) content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) @@ -3023,24 +3398,32 @@ outer_input: @skip_without_tool("hidden_param") def test_hidden_param_in_workflow(self): with self.dataset_populator.test_history() as history_id: - run_object = self._run_workflow(""" + run_object = self._run_workflow( + """ class: GalaxyWorkflow steps: step1: tool_id: hidden_param -""", test_data={}, history_id=history_id, wait=False) - self.workflow_populator.wait_for_invocation_and_jobs(history_id, run_object.workflow_id, run_object.invocation_id) +""", + test_data={}, + history_id=history_id, + wait=False, + ) + self.workflow_populator.wait_for_invocation_and_jobs( + history_id, run_object.workflow_id, run_object.invocation_id + ) contents = self.__history_contents(history_id) assert len(contents) == 1 okay_dataset = contents[0] assert okay_dataset["state"] == "ok" content = self.dataset_populator.get_history_dataset_content(history_id, hid=1) - assert content == '1\n' + assert content == "1\n" @skip_without_tool("output_filter") def test_optional_workflow_output(self): with self.dataset_populator.test_history() as history_id: - run_object = self._run_workflow(""" + run_object = self._run_workflow( + """ class: GalaxyWorkflow inputs: [] outputs: @@ -3053,8 +3436,14 @@ steps: produce_out_1: False filter_text_1: '1' produce_collection: False -""", test_data={}, history_id=history_id, wait=False) - self.workflow_populator.wait_for_invocation_and_jobs(history_id, run_object.workflow_id, run_object.invocation_id) +""", + test_data={}, + history_id=history_id, + wait=False, + ) + self.workflow_populator.wait_for_invocation_and_jobs( + history_id, run_object.workflow_id, run_object.invocation_id + ) contents = self.__history_contents(history_id) assert len(contents) == 1 okay_dataset = contents[0] @@ -3070,7 +3459,8 @@ input1: - identifier: A content: A """ - run_object = self._run_workflow(""" + run_object = self._run_workflow( + """ class: GalaxyWorkflow inputs: input1: @@ -3084,8 +3474,14 @@ steps: tool_id: output_filter_with_input_optional in: input_1: input1 -""", test_data=test_data, history_id=history_id, wait=False) - self.workflow_populator.wait_for_invocation_and_jobs(history_id, run_object.workflow_id, run_object.invocation_id) +""", + test_data=test_data, + history_id=history_id, + wait=False, + ) + self.workflow_populator.wait_for_invocation_and_jobs( + history_id, run_object.workflow_id, run_object.invocation_id + ) contents = self.__history_contents(history_id) assert len(contents) == 4 for content in contents: @@ -3098,7 +3494,8 @@ steps: @skip_without_tool("cat") def test_run_rename_on_mapped_over_collection(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: @@ -3112,7 +3509,8 @@ steps: outputs: out_file1: rename: "my new name" -""", test_data=""" +""", + test_data=""" input1: collection_type: list name: the_dataset_list @@ -3120,12 +3518,16 @@ input1: - identifier: el1 value: 1.fastq type: File -""", history_id=history_id) +""", + history_id=history_id, + ) content = self.dataset_populator.get_history_dataset_details(history_id, hid=4, wait=True, assert_ok=True) name = content["name"] assert name == "my new name", name assert content["history_content_type"] == "dataset" - content = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, assert_ok=True) + content = self.dataset_populator.get_history_collection_details( + history_id, hid=3, wait=True, assert_ok=True + ) name = content["name"] assert content["history_content_type"] == "dataset_collection", content assert name == "my new name", name @@ -3133,7 +3535,8 @@ input1: @skip_without_tool("cat") def test_run_rename_based_on_inputs_on_mapped_over_collection(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: @@ -3147,7 +3550,8 @@ steps: outputs: out_file1: rename: "#{input1} suffix" -""", test_data=""" +""", + test_data=""" input1: collection_type: list name: the_dataset_list @@ -3155,8 +3559,12 @@ input1: - identifier: el1 value: 1.fastq type: File -""", history_id=history_id) - content = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, assert_ok=True) +""", + history_id=history_id, + ) + content = self.dataset_populator.get_history_collection_details( + history_id, hid=3, wait=True, assert_ok=True + ) name = content["name"] assert content["history_content_type"] == "dataset_collection", content assert name == "the_dataset_list suffix", name @@ -3164,7 +3572,8 @@ input1: @skip_without_tool("collection_creates_pair") def test_run_rename_collection_output(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3175,21 +3584,27 @@ steps: outputs: paired_output: rename: "my new name" -""", test_data=""" +""", + test_data=""" input1: value: 1.fasta type: File name: fasta1 -""", history_id=history_id) - details1 = self.dataset_populator.get_history_collection_details(history_id, hid=4, wait=True, assert_ok=True) - assert details1['elements'][0]['object']['visible'] is False +""", + history_id=history_id, + ) + details1 = self.dataset_populator.get_history_collection_details( + history_id, hid=4, wait=True, assert_ok=True + ) + assert details1["elements"][0]["object"]["visible"] is False assert details1["name"] == "my new name", details1 assert details1["history_content_type"] == "dataset_collection" @skip_without_tool("__BUILD_LIST__") def test_run_build_list_hide_collection_output(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3204,21 +3619,27 @@ steps: outputs: output: hide: true -""", test_data=""" +""", + test_data=""" input1: value: 1.fasta type: File name: fasta1 -""", history_id=history_id) - details1 = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, assert_ok=True) - assert details1['elements'][0]['object']['visible'] is False +""", + history_id=history_id, + ) + details1 = self.dataset_populator.get_history_collection_details( + history_id, hid=3, wait=True, assert_ok=True + ) + assert details1["elements"][0]["object"]["visible"] is False assert details1["name"] == "data 1 (as list)", details1 assert details1["visible"] is False @skip_without_tool("__BUILD_LIST__") def test_run_build_list_delete_intermediate_collection_output(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3233,15 +3654,19 @@ steps: outputs: output: delete_intermediate_datasets: true -""", test_data=""" +""", + test_data=""" input1: value: 1.fasta type: File name: fasta1 -""", history_id=history_id) - details1 = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, - assert_ok=True) - assert details1['elements'][0]['object']['visible'] is False +""", + history_id=history_id, + ) + details1 = self.dataset_populator.get_history_collection_details( + history_id, hid=3, wait=True, assert_ok=True + ) + assert details1["elements"][0]["object"]["visible"] is False assert details1["name"] == "data 1 (as list)", details1 # FIXME: this doesn't work because the workflow is still being scheduled # TODO: Implement a way to run PJAs that couldn't be run during/after the job @@ -3251,7 +3676,8 @@ input1: @skip_without_tool("__BUILD_LIST__") def test_run_build_list_change_datatype_collection_output(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3273,27 +3699,33 @@ steps: datasets: - id_cond: id_select: idx -""", test_data=""" +""", + test_data=""" input1: value: 1.fasta type: File file_type: fasta name: fasta1 -""", history_id=history_id) - details1 = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, - assert_ok=True) +""", + history_id=history_id, + ) + details1 = self.dataset_populator.get_history_collection_details( + history_id, hid=3, wait=True, assert_ok=True + ) assert details1["name"] == "data 1 (as list)", details1 - assert details1['elements'][0]['object']['visible'] is False - assert details1['elements'][0]['object']['file_ext'] == 'txt' - details2 = self.dataset_populator.get_history_collection_details(history_id, hid=5, wait=True, - assert_ok=True) + assert details1["elements"][0]["object"]["visible"] is False + assert details1["elements"][0]["object"]["file_ext"] == "txt" + details2 = self.dataset_populator.get_history_collection_details( + history_id, hid=5, wait=True, assert_ok=True + ) # Also check that we don't overwrite the original HDA's datatype - assert details2['elements'][0]['object']['file_ext'] == 'fasta' + assert details2["elements"][0]["object"]["file_ext"] == "fasta" @skip_without_tool("__BUILD_LIST__") def test_run_build_list_rename_collection_output(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3308,22 +3740,27 @@ steps: outputs: output: rename: "my new name" -""", test_data=""" +""", + test_data=""" input1: value: 1.fasta type: File name: fasta1 -""", history_id=history_id) - details1 = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, - assert_ok=True) - assert details1['elements'][0]['object']['visible'] is False +""", + history_id=history_id, + ) + details1 = self.dataset_populator.get_history_collection_details( + history_id, hid=3, wait=True, assert_ok=True + ) + assert details1["elements"][0]["object"]["visible"] is False assert details1["name"] == "my new name", details1 assert details1["history_content_type"] == "dataset_collection" @skip_without_tool("create_2") def test_run_rename_multiple_outputs(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: [] steps: @@ -3336,7 +3773,10 @@ steps: rename: "my new name" out_file2: rename: "my other new name" -""", test_data={}, history_id=history_id) +""", + test_data={}, + history_id=history_id, + ) details1 = self.dataset_populator.get_history_dataset_details(history_id, hid=1, wait=True, assert_ok=True) details2 = self.dataset_populator.get_history_dataset_details(history_id, hid=2) @@ -3355,7 +3795,8 @@ steps: @skip_without_tool("cat") def test_run_rename_when_resuming_jobs(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3376,35 +3817,43 @@ steps: outputs: out_file1: rename: "#{input1} suffix" -""", test_data=""" +""", + test_data=""" input1: value: 1.fasta type: File name: fail -""", history_id=history_id, wait=True, assert_ok=False) +""", + history_id=history_id, + wait=True, + assert_ok=False, + ) content = self.dataset_populator.get_history_dataset_details(history_id, hid=2, wait=True, assert_ok=False) name = content["name"] - assert content['state'] == 'error', content + assert content["state"] == "error", content input1 = self.dataset_populator.get_history_dataset_details(history_id, hid=1, wait=True, assert_ok=False) - job_id = content['creating_job'] - inputs = {"input1": {'values': [{'src': 'hda', - 'id': input1['id']}] - }, - "failbool": "false", - "rerun_remap_job_id": job_id} + job_id = content["creating_job"] + inputs = { + "input1": {"values": [{"src": "hda", "id": input1["id"]}]}, + "failbool": "false", + "rerun_remap_job_id": job_id, + } self.dataset_populator.run_tool( - tool_id='fail_identifier', + tool_id="fail_identifier", inputs=inputs, history_id=history_id, ) - unpaused_dataset = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=False) - assert unpaused_dataset['state'] == 'ok' - assert unpaused_dataset['name'] == f"{name} suffix" + unpaused_dataset = self.dataset_populator.get_history_dataset_details( + history_id, wait=True, assert_ok=False + ) + assert unpaused_dataset["state"] == "ok" + assert unpaused_dataset["name"] == f"{name} suffix" @skip_without_tool("cat") def test_run_rename_based_on_input_recursive(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3416,12 +3865,15 @@ steps: outputs: out_file1: rename: "#{input1} #{input1 | upper} suffix" -""", test_data=""" +""", + test_data=""" input1: value: 1.fasta type: File name: '#{input1}' -""", history_id=history_id) +""", + history_id=history_id, + ) content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) name = content["name"] assert name == "#{input1} #{INPUT1} suffix", name @@ -3429,7 +3881,8 @@ input1: @skip_without_tool("cat") def test_run_rename_based_on_input_repeat(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3446,7 +3899,8 @@ steps: outputs: out_file1: rename: "#{queries_0.input2| basename} suffix" -""", test_data=""" +""", + test_data=""" input1: value: 1.fasta type: File @@ -3455,7 +3909,9 @@ input2: value: 1.fasta type: File name: fasta2 -""", history_id=history_id) +""", + history_id=history_id, + ) content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) name = content["name"] assert name == "fasta2 suffix", name @@ -3463,7 +3919,8 @@ input2: @skip_without_tool("mapper2") def test_run_rename_based_on_input_conditional(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: fasta_input: data @@ -3483,7 +3940,8 @@ steps: # Wish it was qualified for conditionals but it doesn't seem to be. -John # rename: "#{fastq_input.fastq_input1 | basename} suffix" rename: "#{fastq_input1 | basename} suffix" -""", test_data=""" +""", + test_data=""" fasta_input: value: 1.fasta type: File @@ -3494,7 +3952,9 @@ fastq_input: type: File name: fastq1 file_type: fastqsanger -""", history_id=history_id) +""", + history_id=history_id, + ) content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) name = content["name"] assert name == "fastq1 suffix", name @@ -3502,7 +3962,8 @@ fastq_input: @skip_without_tool("mapper2") def test_run_rename_based_on_input_collection(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: fasta_input: data @@ -3522,7 +3983,8 @@ steps: # Wish it was qualified for conditionals but it doesn't seem to be. -John # rename: "#{fastq_input.fastq_input1 | basename} suffix" rename: "#{fastq_input1} suffix" -""", test_data=""" +""", + test_data=""" fasta_input: value: 1.fasta type: File @@ -3538,7 +4000,9 @@ fastq_inputs: - identifier: reverse value: 1.fastq type: File -""", history_id=history_id) +""", + history_id=history_id, + ) content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) name = content["name"] assert name == "the_dataset_pair suffix", name @@ -3546,7 +4010,8 @@ fastq_inputs: @skip_without_tool("collection_creates_pair") def test_run_hide_on_collection_output(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3559,13 +4024,18 @@ steps: outputs: paired_output: hide: true -""", test_data=""" +""", + test_data=""" input1: value: 1.fasta type: File name: fasta1 -""", history_id=history_id) - details1 = self.dataset_populator.get_history_collection_details(history_id, hid=4, wait=True, assert_ok=True) +""", + history_id=history_id, + ) + details1 = self.dataset_populator.get_history_collection_details( + history_id, hid=4, wait=True, assert_ok=True + ) assert details1["history_content_type"] == "dataset_collection" assert not details1["visible"], details1 @@ -3573,7 +4043,8 @@ input1: @skip_without_tool("cat") def test_run_hide_on_mapped_over_collection(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: - id: input1 @@ -3587,7 +4058,8 @@ steps: outputs: out_file1: hide: true -""", test_data=""" +""", + test_data=""" input1: collection_type: list name: the_dataset_list @@ -3595,20 +4067,25 @@ input1: - identifier: el1 value: 1.fastq type: File -""", history_id=history_id) +""", + history_id=history_id, + ) content = self.dataset_populator.get_history_dataset_details(history_id, hid=4, wait=True, assert_ok=True) assert content["history_content_type"] == "dataset" assert not content["visible"] - content = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, assert_ok=True) + content = self.dataset_populator.get_history_collection_details( + history_id, hid=3, wait=True, assert_ok=True + ) assert content["history_content_type"] == "dataset_collection", content assert not content["visible"] @skip_without_tool("cat") def test_tag_auto_propagation(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3628,12 +4105,16 @@ steps: tool_id: cat in: input1: first_cat/out_file1 -""", test_data=""" +""", + test_data=""" input1: value: 1.fasta type: File name: fasta1 -""", history_id=history_id, round_trip_format_conversion=True) +""", + history_id=history_id, + round_trip_format_conversion=True, + ) details0 = self.dataset_populator.get_history_dataset_details(history_id, hid=2, wait=True, assert_ok=True) tags = details0["tags"] @@ -3651,7 +4132,8 @@ input1: @skip_without_tool("collection_creates_pair") def test_run_add_tag_on_collection_output(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3664,13 +4146,19 @@ steps: paired_output: add_tags: - "name:foo" -""", test_data=""" +""", + test_data=""" input1: value: 1.fasta type: File name: fasta1 -""", history_id=history_id, round_trip_format_conversion=True) - details1 = self.dataset_populator.get_history_collection_details(history_id, hid=4, wait=True, assert_ok=True) +""", + history_id=history_id, + round_trip_format_conversion=True, + ) + details1 = self.dataset_populator.get_history_collection_details( + history_id, hid=4, wait=True, assert_ok=True + ) assert details1["history_content_type"] == "dataset_collection" assert details1["tags"][0] == "name:foo", details1 @@ -3678,7 +4166,8 @@ input1: @skip_without_tool("cat") def test_run_add_tag_on_mapped_over_collection(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: @@ -3693,7 +4182,8 @@ steps: out_file1: add_tags: - "name:foo" -""", test_data=""" +""", + test_data=""" input1: collection_type: list name: the_dataset_list @@ -3701,8 +4191,13 @@ input1: - identifier: el1 value: 1.fastq type: File -""", history_id=history_id, round_trip_format_conversion=True) - details1 = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, assert_ok=True) +""", + history_id=history_id, + round_trip_format_conversion=True, + ) + details1 = self.dataset_populator.get_history_collection_details( + history_id, hid=3, wait=True, assert_ok=True + ) assert details1["history_content_type"] == "dataset_collection" assert details1["tags"][0] == "name:foo", details1 @@ -3711,7 +4206,8 @@ input1: @skip_without_tool("cat") def test_run_remove_tag_on_collection_output(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3732,19 +4228,29 @@ steps: paired_output: remove_tags: - "name:foo" -""", test_data=""" +""", + test_data=""" input1: value: 1.fasta type: File name: fasta1 -""", history_id=history_id, round_trip_format_conversion=True) - details_dataset_with_tag = self.dataset_populator.get_history_dataset_details(history_id, hid=2, wait=True, assert_ok=True) +""", + history_id=history_id, + round_trip_format_conversion=True, + ) + details_dataset_with_tag = self.dataset_populator.get_history_dataset_details( + history_id, hid=2, wait=True, assert_ok=True + ) assert details_dataset_with_tag["history_content_type"] == "dataset", details_dataset_with_tag assert details_dataset_with_tag["tags"][0] == "name:foo", details_dataset_with_tag - details_collection_without_tag = self.dataset_populator.get_history_collection_details(history_id, hid=5, wait=True, assert_ok=True) - assert details_collection_without_tag["history_content_type"] == "dataset_collection", details_collection_without_tag + details_collection_without_tag = self.dataset_populator.get_history_collection_details( + history_id, hid=5, wait=True, assert_ok=True + ) + assert ( + details_collection_without_tag["history_content_type"] == "dataset_collection" + ), details_collection_without_tag assert len(details_collection_without_tag["tags"]) == 0, details_collection_without_tag @skip_without_tool("cat1") @@ -3754,7 +4260,7 @@ input1: workflow["steps"]["0"]["uuid"] = uuid0 workflow["steps"]["1"]["uuid"] = uuid1 workflow["steps"]["2"]["uuid"] = uuid2 - workflow_request, history_id, workflow_id = self._setup_workflow_run(workflow, inputs_by='step_index') + workflow_request, history_id, workflow_id = self._setup_workflow_run(workflow, inputs_by="step_index") workflow_request["replacement_params"] = dumps(dict(replaceme="was replaced")) pja_map = { "RenameDatasetActionout_file1": dict( @@ -3763,9 +4269,7 @@ input1: action_arguments=dict(newname="foo ${replaceme}"), ) } - workflow_request["parameters"] = dumps({ - uuid2: {"__POST_JOB_ACTIONS__": pja_map} - }) + workflow_request["parameters"] = dumps({uuid2: {"__POST_JOB_ACTIONS__": pja_map}}) self.workflow_populator.invoke_workflow_and_wait(workflow_id, request=workflow_request, assert_ok=True) content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) @@ -3779,7 +4283,8 @@ input1: @skip_without_tool("cat1") def test_run_with_delayed_runtime_pja(self): - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow inputs: test_input: data @@ -3796,14 +4301,16 @@ steps: tool_id: cat1 in: input1: the_pause -""", round_trip_format_conversion=True) +""", + round_trip_format_conversion=True, + ) downloaded_workflow = self._download_workflow(workflow_id) uuid_dict = {int(index): step["uuid"] for index, step in downloaded_workflow["steps"].items()} with self.dataset_populator.test_history() as history_id: hda = self.dataset_populator.new_dataset(history_id, content="1 2 3") self.dataset_populator.wait_for_history(history_id) inputs = { - '0': self._ds_entry(hda), + "0": self._ds_entry(hda), } uuid2 = uuid_dict[3] workflow_request = {} @@ -3815,10 +4322,10 @@ steps: action_arguments=dict(newname="foo ${replaceme}"), ) } - workflow_request["parameters"] = dumps({ - uuid2: {"__POST_JOB_ACTIONS__": pja_map} - }) - invocation_id = self.__invoke_workflow(workflow_id, inputs=inputs, request=workflow_request, history_id=history_id) + workflow_request["parameters"] = dumps({uuid2: {"__POST_JOB_ACTIONS__": pja_map}}) + invocation_id = self.__invoke_workflow( + workflow_id, inputs=inputs, request=workflow_request, history_id=history_id + ) time.sleep(2) self.dataset_populator.wait_for_history(history_id) @@ -3832,7 +4339,8 @@ steps: @skip_without_tool("cat1") def test_delete_intermediate_datasets_pja_1(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3855,7 +4363,10 @@ steps: outputs: out_file1: delete_intermediate_datasets: true -""", test_data={"input1": "hello world"}, history_id=history_id) +""", + test_data={"input1": "hello world"}, + history_id=history_id, + ) hda1 = self.dataset_populator.get_history_dataset_details(history_id, hid=1) hda2 = self.dataset_populator.get_history_dataset_details(history_id, hid=2) hda3 = self.dataset_populator.get_history_dataset_details(history_id, hid=3) @@ -3871,7 +4382,8 @@ steps: @skip_without_tool("cat1") def test_validated_post_job_action_validated(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3886,21 +4398,29 @@ steps: post_job_actions: ValidateOutputsAction: action_type: ValidateOutputsAction -""", test_data={"input1": {"type": "File", "file_type": "fastqsanger", "value": "1.fastqsanger"}}, history_id=history_id) +""", + test_data={"input1": {"type": "File", "file_type": "fastqsanger", "value": "1.fastqsanger"}}, + history_id=history_id, + ) hda2 = self.dataset_populator.get_history_dataset_details(history_id, hid=2) assert hda2["validated_state"] == "ok" @skip_without_tool("cat1") def test_validated_post_job_action_unvalidated_default(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(WORKFLOW_SIMPLE, test_data={"input1": {"type": "File", "file_type": "fastqsanger", "value": "1.fastqsanger"}}, history_id=history_id) + self._run_jobs( + WORKFLOW_SIMPLE, + test_data={"input1": {"type": "File", "file_type": "fastqsanger", "value": "1.fastqsanger"}}, + history_id=history_id, + ) hda2 = self.dataset_populator.get_history_dataset_details(history_id, hid=2) assert hda2["validated_state"] == "unknown" @skip_without_tool("cat1") def test_validated_post_job_action_invalid(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input1: data @@ -3915,12 +4435,16 @@ steps: post_job_actions: ValidateOutputsAction: action_type: ValidateOutputsAction -""", test_data={"input1": {"type": "File", "file_type": "fastqcssanger", "value": "1.fastqsanger"}}, history_id=history_id) +""", + test_data={"input1": {"type": "File", "file_type": "fastqcssanger", "value": "1.fastqsanger"}}, + history_id=history_id, + ) hda2 = self.dataset_populator.get_history_dataset_details(history_id, hid=2) assert hda2["validated_state"] == "invalid" def test_value_restriction_with_select_and_text_param(self): - workflow_id = self.workflow_populator.upload_yaml_workflow(""" + workflow_id = self.workflow_populator.upload_yaml_workflow( + """ class: GalaxyWorkflow inputs: select_text: @@ -3935,12 +4459,13 @@ steps: tool_id: param_text_option in: text_param: select_text -""") +""" + ) with self.dataset_populator.test_history() as history_id: - run_workflow = self._download_workflow(workflow_id, style='run', history_id=history_id) - options = run_workflow['steps'][0]['inputs'][0]['options'] + run_workflow = self._download_workflow(workflow_id, style="run", history_id=history_id) + options = run_workflow["steps"][0]["inputs"][0]["options"] assert len(options) == 5 - assert options[0] == ['Ex1', '--ex1', False] + assert options[0] == ["Ex1", "--ex1", False] @skip_without_tool("random_lines1") def test_run_replace_params_by_tool(self): @@ -3954,10 +4479,12 @@ steps: @skip_without_tool("random_lines1") def test_run_replace_params_by_uuid(self): workflow_request, history_id, workflow_id = self._setup_random_x2_workflow("test_for_replace_") - workflow_request["parameters"] = dumps({ - "58dffcc9-bcb7-4117-a0e1-61513524b3b1": dict(num_lines=4), - "58dffcc9-bcb7-4117-a0e1-61513524b3b2": dict(num_lines=3), - }) + workflow_request["parameters"] = dumps( + { + "58dffcc9-bcb7-4117-a0e1-61513524b3b1": dict(num_lines=4), + "58dffcc9-bcb7-4117-a0e1-61513524b3b2": dict(num_lines=3), + } + ) self.workflow_populator.invoke_workflow_and_wait(workflow_id, request=workflow_request, assert_ok=True) # Would be 8 and 6 without modification self.__assert_lines_hid_line_count_is(history_id, 2, 4) @@ -3974,11 +4501,22 @@ steps: hda3 = self.dataset_populator.new_dataset(history_id, content="7 8 9") hda4 = self.dataset_populator.new_dataset(history_id, content="10 11 12") parameters = { - "0": {"input": {"batch": True, "values": [{"id": hda1.get("id"), "hid": hda1.get("hid"), "src": "hda"}, - {"id": hda2.get("id"), "hid": hda2.get("hid"), "src": "hda"}, - {"id": hda3.get("id"), "hid": hda2.get("hid"), "src": "hda"}, - {"id": hda4.get("id"), "hid": hda2.get("hid"), "src": "hda"}]}}, - "1": {"input": {"batch": False, "values": [{"id": hda1.get("id"), "hid": hda1.get("hid"), "src": "hda"}]}, "exp": "2"}} + "0": { + "input": { + "batch": True, + "values": [ + {"id": hda1.get("id"), "hid": hda1.get("hid"), "src": "hda"}, + {"id": hda2.get("id"), "hid": hda2.get("hid"), "src": "hda"}, + {"id": hda3.get("id"), "hid": hda2.get("hid"), "src": "hda"}, + {"id": hda4.get("id"), "hid": hda2.get("hid"), "src": "hda"}, + ], + } + }, + "1": { + "input": {"batch": False, "values": [{"id": hda1.get("id"), "hid": hda1.get("hid"), "src": "hda"}]}, + "exp": "2", + }, + } workflow_request = { "history_id": history_id, "batch": True, @@ -4013,13 +4551,21 @@ steps: hda3 = self.dataset_populator.new_dataset(history_id, content="7 8 9") hda4 = self.dataset_populator.new_dataset(history_id, content="10 11 12") inputs = { - "coolinput": {"batch": True, "values": [{"id": hda1.get("id"), "hid": hda1.get("hid"), "src": "hda"}, - {"id": hda2.get("id"), "hid": hda2.get("hid"), "src": "hda"}, - {"id": hda3.get("id"), "hid": hda2.get("hid"), "src": "hda"}, - {"id": hda4.get("id"), "hid": hda2.get("hid"), "src": "hda"}]} + "coolinput": { + "batch": True, + "values": [ + {"id": hda1.get("id"), "hid": hda1.get("hid"), "src": "hda"}, + {"id": hda2.get("id"), "hid": hda2.get("hid"), "src": "hda"}, + {"id": hda3.get("id"), "hid": hda2.get("hid"), "src": "hda"}, + {"id": hda4.get("id"), "hid": hda2.get("hid"), "src": "hda"}, + ], + } } parameters = { - "1": {"input": {"batch": False, "values": [{"id": hda1.get("id"), "hid": hda1.get("hid"), "src": "hda"}]}, "exp": "2"} + "1": { + "input": {"batch": False, "values": [{"id": hda1.get("id"), "hid": hda1.get("hid"), "src": "hda"}]}, + "exp": "2", + } } workflow_request = { "history_id": history_id, @@ -4048,16 +4594,19 @@ steps: @skip_without_tool("validation_default") def test_parameter_substitution_sanitization(self): - substitions = dict(input1="\" ; echo \"moo") + substitions = dict(input1='" ; echo "moo') run_workflow_response, history_id = self._run_validation_workflow_with_substitions(substitions) self.dataset_populator.wait_for_history(history_id, assert_ok=True) - self.assertEqual("__dq__ X echo __dq__moo\n", self.dataset_populator.get_history_dataset_content(history_id, hid=1)) + self.assertEqual( + "__dq__ X echo __dq__moo\n", self.dataset_populator.get_history_dataset_content(history_id, hid=1) + ) @skip_without_tool("validation_repeat") def test_parameter_substitution_validation_value_errors_0(self): with self.dataset_populator.test_history() as history_id: - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow steps: validation: @@ -4065,10 +4614,10 @@ steps: state: r2: - text: "abd" -""") +""" + ) workflow_request = dict( - history=f"hist_id={history_id}", - parameters=dumps(dict(validation_repeat={"r2_0|text": ""})) + history=f"hist_id={history_id}", parameters=dumps(dict(validation_repeat={"r2_0|text": ""})) ) url = f"workflows/{workflow_id}/invocations" invocation_response = self._post(url, data=workflow_request) @@ -4077,7 +4626,7 @@ steps: @skip_without_tool("validation_default") def test_parameter_substitution_validation_value_errors_1(self): - substitions = dict(select_param="\" ; echo \"moo") + substitions = dict(select_param='" ; echo "moo') run_workflow_response, history_id = self._run_validation_workflow_with_substitions(substitions) self._assert_status_code_is(run_workflow_response, 400) @@ -4085,7 +4634,8 @@ steps: @skip_without_tool("validation_repeat") def test_workflow_import_state_validation_1(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow steps: validation: @@ -4093,7 +4643,12 @@ steps: state: r2: - text: "" -""", history_id=history_id, wait=False, expected_response=400, assert_ok=False) +""", + history_id=history_id, + wait=False, + expected_response=400, + assert_ok=False, + ) def _run_validation_workflow_with_substitions(self, substitions): workflow = self.workflow_populator.load_workflow_from_resource("test_workflow_validation_1") @@ -4102,14 +4657,16 @@ steps: workflow_request = dict( history=f"hist_id={history_id}", workflow_id=uploaded_workflow_id, - parameters=dumps(dict(validation_default=substitions)) + parameters=dumps(dict(validation_default=substitions)), ) run_workflow_response = self.workflow_populator.invoke_workflow_raw(uploaded_workflow_id, workflow_request) return run_workflow_response, history_id @skip_without_tool("random_lines1") def test_run_replace_params_by_steps(self): - workflow_request, history_id, workflow_id, steps = self._setup_random_x2_workflow_steps("test_for_replace_step_params") + workflow_request, history_id, workflow_id, steps = self._setup_random_x2_workflow_steps( + "test_for_replace_step_params" + ) params = dumps({str(steps[1]["id"]): dict(num_lines=5)}) workflow_request["parameters"] = params self.workflow_populator.invoke_workflow_and_wait(workflow_id, request=workflow_request, assert_ok=True) @@ -4119,27 +4676,34 @@ steps: @skip_without_tool("random_lines1") def test_run_replace_params_nested(self): - workflow_request, history_id, workflow_id, steps = self._setup_random_x2_workflow_steps("test_for_replace_step_params_nested") + workflow_request, history_id, workflow_id, steps = self._setup_random_x2_workflow_steps( + "test_for_replace_step_params_nested" + ) seed_source = dict( seed_source_selector="set_seed", seed="moo", ) - params = dumps({str(steps[0]["id"]): dict(num_lines=1, seed_source=seed_source), - str(steps[1]["id"]): dict(num_lines=1, seed_source=seed_source)}) + params = dumps( + { + str(steps[0]["id"]): dict(num_lines=1, seed_source=seed_source), + str(steps[1]["id"]): dict(num_lines=1, seed_source=seed_source), + } + ) workflow_request["parameters"] = params self.workflow_populator.invoke_workflow_and_wait(workflow_id, request=workflow_request, assert_ok=True) self.assertEqual("2\n", self.dataset_populator.get_history_dataset_content(history_id)) @skip_without_tool("random_lines1") def test_run_replace_params_nested_normalized(self): - workflow_request, history_id, workflow_id, steps = self._setup_random_x2_workflow_steps("test_for_replace_step_normalized_params_nested") + workflow_request, history_id, workflow_id, steps = self._setup_random_x2_workflow_steps( + "test_for_replace_step_normalized_params_nested" + ) parameters = { "num_lines": 1, "seed_source|seed_source_selector": "set_seed", "seed_source|seed": "moo", } - params = dumps({str(steps[0]["id"]): parameters, - str(steps[1]["id"]): parameters}) + params = dumps({str(steps[0]["id"]): parameters, str(steps[1]["id"]): parameters}) workflow_request["parameters"] = params workflow_request["parameters_normalized"] = False self.workflow_populator.invoke_workflow_and_wait(workflow_id, request=workflow_request, assert_ok=True) @@ -4148,14 +4712,21 @@ steps: @skip_without_tool("random_lines1") def test_run_replace_params_over_default(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(WORKFLOW_ONE_STEP_DEFAULT, test_data=""" + self._run_jobs( + WORKFLOW_ONE_STEP_DEFAULT, + test_data=""" step_parameters: '1': num_lines: 4 input: value: 1.bed type: File -""", history_id=history_id, wait=True, assert_ok=True, round_trip_format_conversion=True) +""", + history_id=history_id, + wait=True, + assert_ok=True, + round_trip_format_conversion=True, + ) result = self.dataset_populator.get_history_dataset_content(history_id) assert result.count("\n") == 4 @@ -4169,7 +4740,8 @@ input: @skip_without_tool("random_lines1") def test_run_replace_params_over_default_delayed(self): with self.dataset_populator.test_history() as history_id: - run_summary = self._run_workflow(""" + run_summary = self._run_workflow( + """ class: GalaxyWorkflow inputs: input: data @@ -4188,14 +4760,18 @@ steps: input: the_pause num_lines: default: 6 -""", test_data=""" +""", + test_data=""" step_parameters: '3': num_lines: 4 input: value: 1.bed type: File -""", history_id=history_id, wait=False) +""", + history_id=history_id, + wait=False, + ) wait_on(lambda: len(self._history_jobs(history_id)) >= 2 or None, "history jobs") self.dataset_populator.wait_for_history(history_id, assert_ok=True) @@ -4221,39 +4797,54 @@ input: def test_invocation_filtering(self): with self._different_user(email=f"{uuid4()}@test.com"): # new user, start with no invocations - assert not self._assert_invocation_for_url_is('invocations') - self._run_jobs(""" + assert not self._assert_invocation_for_url_is("invocations") + self._run_jobs( + """ class: GalaxyWorkflow inputs: input: type: data optional: true steps: [] -""", wait=False) - first_invocation = self._assert_invocation_for_url_is('invocations') +""", + wait=False, + ) + first_invocation = self._assert_invocation_for_url_is("invocations") new_history_id = self.dataset_populator.new_history() # new history has no invocations assert not self._assert_invocation_for_url_is(f"invocations?history_id={new_history_id}") - self._run_jobs(""" + self._run_jobs( + """ class: GalaxyWorkflow inputs: input: type: data optional: true steps: [] -""", history_id=new_history_id, wait=False) +""", + history_id=new_history_id, + wait=False, + ) # new history has one invocation now new_invocation = self._assert_invocation_for_url_is(f"invocations?history_id={new_history_id}") # filter invocation by workflow instance id - self._assert_invocation_for_url_is(f"invocations?workflow_id={first_invocation['workflow_id']}&instance=true", first_invocation) + self._assert_invocation_for_url_is( + f"invocations?workflow_id={first_invocation['workflow_id']}&instance=true", first_invocation + ) # limit to 1, newest invocation first by default self._assert_invocation_for_url_is("invocations?limit=1", target_invocation=new_invocation) # limit to 1, descending sort on date - self._assert_invocation_for_url_is("invocations?limit=1&sort_by=create_time&sort_desc=true", target_invocation=new_invocation) + self._assert_invocation_for_url_is( + "invocations?limit=1&sort_by=create_time&sort_desc=true", target_invocation=new_invocation + ) # limit to 1, ascending sort on date - self._assert_invocation_for_url_is("invocations?limit=1&sort_by=create_time&sort_desc=false", target_invocation=first_invocation) + self._assert_invocation_for_url_is( + "invocations?limit=1&sort_by=create_time&sort_desc=false", target_invocation=first_invocation + ) # limit to 1, ascending sort on date, offset 1 - self._assert_invocation_for_url_is("invocations?limit=1&sort_by=create_time&sort_desc=false&offset=1", target_invocation=new_invocation) + self._assert_invocation_for_url_is( + "invocations?limit=1&sort_by=create_time&sort_desc=false&offset=1", target_invocation=new_invocation + ) def _assert_invocation_for_url_is(self, route, target_invocation=None): response = self._get(route) @@ -4261,7 +4852,7 @@ steps: [] invocations = response.json() if target_invocation: assert len(invocations) == 1 - assert invocations[0]['id'] == target_invocation['id'] + assert invocations[0]["id"] == target_invocation["id"] if invocations: assert len(invocations) == 1 return invocations[0] @@ -4302,7 +4893,7 @@ steps: [] assert invocation_id in invocation_ids # Wait for the invocation to be fully scheduled, so we have details on all steps. - self._wait_for_invocation_state(workflow_id, invocation_id, 'scheduled') + self._wait_for_invocation_state(workflow_id, invocation_id, "scheduled") usage_details = self._invocation_details(workflow_id, invocation_id) invocation_steps = usage_details["steps"] @@ -4322,7 +4913,9 @@ steps: [] assert job_id is not None invocation_tool_step_id = invocation_tool_step["id"] - invocation_tool_step_response = self._get(f"workflows/{workflow_id}/invocations/{invocation_id}/steps/{invocation_tool_step_id}") + invocation_tool_step_response = self._get( + f"workflows/{workflow_id}/invocations/{invocation_id}/steps/{invocation_tool_step_id}" + ) self._assert_status_code_is(invocation_tool_step_response, 200) self._assert_has_keys(invocation_tool_step_response.json(), "id", "order_index", "job_id") @@ -4378,7 +4971,8 @@ steps: [] def _run_mapping_workflow(self): history_id = self.dataset_populator.new_history() - summary = self._run_workflow(""" + summary = self._run_workflow( + """ class: GalaxyWorkflow inputs: input_c: collection @@ -4387,7 +4981,8 @@ steps: tool_id: cat1 in: input1: input_c -""", test_data=""" +""", + test_data=""" input_c: collection_type: list elements: @@ -4395,7 +4990,11 @@ input_c: content: "0" - identifier: i2 content: "1" -""", history_id=history_id, wait=True, assert_ok=True) +""", + history_id=history_id, + wait=True, + assert_ok=True, + ) workflow_id = summary.workflow_id invocation_id = summary.invocation_id return workflow_id, invocation_id @@ -4411,9 +5010,11 @@ input_c: response = self._get(f"workflows/{other_id}/usage") self._assert_status_code_is(response, 200) assert len(response.json()) == 0 - run_workflow_response = self.workflow_populator.invoke_workflow_raw(workflow_id, workflow_request, assert_ok=True) + run_workflow_response = self.workflow_populator.invoke_workflow_raw( + workflow_id, workflow_request, assert_ok=True + ) run_workflow_dict = run_workflow_response.json() - invocation_id = run_workflow_dict['id'] + invocation_id = run_workflow_dict["id"] usage_details_response = self._get(f"workflows/{other_id}/usage/{invocation_id}") self._assert_status_code_is(usage_details_response, 200) @@ -4425,9 +5026,11 @@ input_c: response = self._get(f"workflows/{workflow_id}/usage") self._assert_status_code_is(response, 200) assert len(response.json()) == 0 - run_workflow_response = self.workflow_populator.invoke_workflow_raw(workflow_id, workflow_request, assert_ok=True) + run_workflow_response = self.workflow_populator.invoke_workflow_raw( + workflow_id, workflow_request, assert_ok=True + ) run_workflow_dict = run_workflow_response.json() - invocation_id = run_workflow_dict['id'] + invocation_id = run_workflow_dict["id"] usage_details_response = self._get(f"workflows/{workflow_id}/usage/{invocation_id}") self._assert_status_code_is(usage_details_response, 200) @@ -4438,9 +5041,11 @@ input_c: response = self._get(f"workflows/{workflow_id}/usage") self._assert_status_code_is(response, 200) assert len(response.json()) == 0 - run_workflow_response = self.workflow_populator.invoke_workflow_raw(workflow_id, workflow_request, assert_ok=True) + run_workflow_response = self.workflow_populator.invoke_workflow_raw( + workflow_id, workflow_request, assert_ok=True + ) run_workflow_dict = run_workflow_response.json() - invocation_id = run_workflow_dict['id'] + invocation_id = run_workflow_dict["id"] with self._different_user(): usage_details_response = self._get(f"workflows/{workflow_id}/usage/{invocation_id}") self._assert_status_code_is(usage_details_response, 403) @@ -4448,11 +5053,11 @@ input_c: def test_workflow_publishing(self): workflow_id = self.workflow_populator.simple_workflow("dummy") response = self._show_workflow(workflow_id) - assert not response['published'] - published_worklow = self._put(f'workflows/{workflow_id}', data={'published': True}, json=True).json() - assert published_worklow['published'] - unpublished_worklow = self._put(f'workflows/{workflow_id}', data={'published': False}, json=True).json() - assert not unpublished_worklow['published'] + assert not response["published"] + published_worklow = self._put(f"workflows/{workflow_id}", data={"published": True}, json=True).json() + assert published_worklow["published"] + unpublished_worklow = self._put(f"workflows/{workflow_id}", data={"published": False}, json=True).json() + assert not unpublished_worklow["published"] def test_workflow_from_path_requires_admin(self): # There are two ways to import workflows from paths, just verify both require an admin. @@ -4478,7 +5083,7 @@ input_c: workflow_id = self.workflow_populator.create_workflow(workflow) hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3") index_map = { - '0': self._ds_entry(hda1), + "0": self._ds_entry(hda1), } invocation_id = self.__invoke_workflow( workflow_id, @@ -4491,27 +5096,27 @@ input_c: target_state_reached = False for _ in range(50): invocation = self._invocation_details(workflow_id, invocation_id) - if invocation['state'] != 'new': + if invocation["state"] != "new": target_state_reached = True break - time.sleep(.25) + time.sleep(0.25) return target_state_reached def _assert_invocation_non_terminal(self, workflow_id, invocation_id): invocation = self._invocation_details(workflow_id, invocation_id) - assert invocation['state'] in ['ready', 'new'], invocation + assert invocation["state"] in ["ready", "new"], invocation def _wait_for_invocation_state(self, workflow_id, invocation_id, target_state): target_state_reached = False for _ in range(25): invocation = self._invocation_details(workflow_id, invocation_id) - if invocation['state'] == target_state: + if invocation["state"] == target_state: target_state_reached = True break - time.sleep(.5) + time.sleep(0.5) return target_state_reached @@ -4557,7 +5162,9 @@ input_c: workflow_summary_response = self._get(f"workflows/{workflow_id}") self._assert_status_code_is(workflow_summary_response, 200) steps = workflow_summary_response.json()["steps"] - return sorted((step for step in steps.values() if step["tool_id"] == "random_lines1"), key=lambda step: step["id"]) + return sorted( + (step for step in steps.values() if step["tool_id"] == "random_lines1"), key=lambda step: step["id"] + ) def _setup_random_x2_workflow(self, name: str): workflow = self.workflow_populator.load_random_x2_workflow(name) @@ -4569,18 +5176,20 @@ input_c: hda1 = self.dataset_populator.new_dataset(history_id, content=ten_lines) workflow_request = dict( history=f"hist_id={history_id}", - ds_map=dumps({ - key: self._ds_entry(hda1), - }), + ds_map=dumps( + { + key: self._ds_entry(hda1), + } + ), ) return workflow_request, history_id, uploaded_workflow_id def __review_paused_steps(self, uploaded_workflow_id, invocation_id, order_index, action=True): invocation = self._invocation_details(uploaded_workflow_id, invocation_id) invocation_steps = invocation["steps"] - pause_steps = [s for s in invocation_steps if s['order_index'] == order_index] + pause_steps = [s for s in invocation_steps if s["order_index"] == order_index] for pause_step in pause_steps: - pause_step_id = pause_step['id'] + pause_step_id = pause_step["id"] self._execute_invocation_step_action(uploaded_workflow_id, invocation_id, pause_step_id, action=action) @@ -4620,24 +5229,17 @@ input_c: return show_response.json() def _assert_looks_like_instance_workflow_representation(self, workflow): - self._assert_has_keys( - workflow, - 'url', - 'owner', - 'inputs', - 'annotation', - 'steps' - ) + self._assert_has_keys(workflow, "url", "owner", "inputs", "annotation", "steps") for step in workflow["steps"].values(): self._assert_has_keys( step, - 'id', - 'type', - 'tool_id', - 'tool_version', - 'annotation', - 'tool_inputs', - 'input_steps', + "id", + "type", + "tool_id", + "tool_version", + "annotation", + "tool_inputs", + "input_steps", ) def _all_user_invocation_ids(self): @@ -4652,7 +5254,8 @@ class AdminWorkflowsApiTestCase(BaseWorkflowsApiTestCase): require_admin_user = True def test_import_export_dynamic_tools(self): - workflow_id = self._upload_yaml_workflow(""" + workflow_id = self._upload_yaml_workflow( + """ class: GalaxyWorkflow steps: - type: input @@ -4678,7 +5281,8 @@ steps: $link: embed1/output1 test_data: input1: "hello world" -""") +""" + ) downloaded_workflow = self._download_workflow(workflow_id) response = self.workflow_populator.create_workflow_response(downloaded_workflow) workflow_id = response.json()["id"] @@ -4686,8 +5290,13 @@ test_data: hda1 = self.dataset_populator.new_dataset(history_id, content="Hello World Second!") workflow_request = dict( inputs_by="name", - inputs=json.dumps({'input1': self._ds_entry(hda1)}), + inputs=json.dumps({"input1": self._ds_entry(hda1)}), + ) + invocation_id = self.workflow_populator.invoke_workflow( + workflow_id, history_id=history_id, request=workflow_request, assert_ok=True ) - invocation_id = self.workflow_populator.invoke_workflow(workflow_id, history_id=history_id, request=workflow_request, assert_ok=True) self.workflow_populator.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) - self.assertEqual("Hello World Second!\nhello world 2\n", self.dataset_populator.get_history_dataset_content(history_id, hid=4)) + self.assertEqual( + "Hello World Second!\nhello world 2\n", + self.dataset_populator.get_history_dataset_content(history_id, hid=4), + ) diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 9183483c2c5..18152fd3b41 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -45,7 +45,10 @@ import string import time import unittest import urllib.parse -from abc import ABCMeta, abstractmethod +from abc import ( + ABCMeta, + abstractmethod, +) from functools import wraps from io import StringIO from operator import itemgetter @@ -78,8 +81,8 @@ from galaxy.tool_util.verify.test_data import TestDataResolver from galaxy.tool_util.verify.wait import ( timeout_type, TimeoutAssertionError, - wait_on as tool_util_wait_on, ) +from galaxy.tool_util.verify.wait import wait_on as tool_util_wait_on from galaxy.util import ( DEFAULT_SOCKET_TIMEOUT, galaxy_root_path, @@ -88,7 +91,6 @@ from galaxy.util import ( from . import api_asserts from .api import ApiTestInteractor - CWL_TOOL_DIRECTORY = os.path.join(galaxy_root_path, "test", "functional", "tools", "cwl_tools") # Simple workflow that takes an input and call cat wrapper on it. @@ -104,7 +106,6 @@ SKIP_FLAKEY_TESTS_ON_ERROR = os.environ.get("GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ER def flakey(method): - @wraps(method) def wrapped_method(test_case, *args, **kwargs): try: @@ -127,7 +128,6 @@ def skip_without_tool(tool_id): """ def method_wrapper(method): - def get_tool_ids(api_test_case): index = api_test_case.galaxy_interactor.get("tools", data=dict(in_panel=False)) tools = index.json() @@ -178,7 +178,6 @@ def is_site_up(url): def skip_if_site_down(url): - def method_wrapper(method): @wraps(method) def wrapped_method(api_test_case, *args, **kwargs): @@ -207,8 +206,7 @@ def summarize_instance_history_on_error(method): def uses_test_history(**test_history_kwd): - """Can override require_new and cancel_executions using kwds to decorator. - """ + """Can override require_new and cancel_executions using kwds to decorator.""" def method_wrapper(method): @wraps(method) @@ -224,6 +222,7 @@ def uses_test_history(**test_history_kwd): def _raise_skip_if(check, *args): if check: from nose.plugins.skip import SkipTest + raise SkipTest(*args) @@ -242,14 +241,12 @@ def conformance_tests_gen(directory, filename="conformance_tests.yaml"): class CwlRun: - def __init__(self, dataset_populator, history_id): self.dataset_populator = dataset_populator self.history_id = history_id class CwlToolRun(CwlRun): - def __init__(self, dataset_populator, history_id, run_response): super().__init__(dataset_populator, history_id) self.run_response = run_response @@ -263,7 +260,6 @@ class CwlToolRun(CwlRun): class CwlWorkflowRun(CwlRun): - def __init__(self, dataset_populator, workflow_populator, history_id, workflow_id, invocation_id): super().__init__(dataset_populator, history_id) self.workflow_populator = workflow_populator @@ -271,13 +267,10 @@ class CwlWorkflowRun(CwlRun): self.invocation_id = invocation_id def wait(self): - self.workflow_populator.wait_for_invocation_and_jobs( - self.history_id, self.workflow_id, self.invocation_id - ) + self.workflow_populator.wait_for_invocation_and_jobs(self.history_id, self.workflow_id, self.invocation_id) class CwlPopulator: - def __init__(self, dataset_populator, workflow_populator): self.dataset_populator = dataset_populator self.workflow_populator = workflow_populator @@ -331,7 +324,9 @@ class CwlPopulator: # workflows as well, and then make it the default. Or decide they are safe. "allow_tool_state_corrections": True, } - invocation_id = self.workflow_populator.invoke_workflow(workflow_id, history_id=history_id, inputs=job, request=request, inputs_by="name") + invocation_id = self.workflow_populator.invoke_workflow( + workflow_id, history_id=history_id, inputs=job, request=request, inputs_by="name" + ) return CwlWorkflowRun(self.dataset_populator, self.workflow_populator, history_id, workflow_id, invocation_id) def run_cwl_job( @@ -452,7 +447,7 @@ class BasePopulator(metaclass=ABCMeta): class BaseDatasetPopulator(BasePopulator): - """ Abstract description of API operations optimized for testing + """Abstract description of API operations optimized for testing Galaxy - implementations must implement _get, _post and _delete. """ @@ -466,17 +461,22 @@ class BaseDatasetPopulator(BasePopulator): return run_response.json()["outputs"][0] def new_dataset_request(self, history_id: str, content=None, wait: bool = False, **kwds) -> requests.Response: - """Lower-level dataset creation that returns the upload tool response object. - """ + """Lower-level dataset creation that returns the upload tool response object.""" if content is None and "ftp_files" not in kwds: content = "TestData123" payload = self.upload_payload(history_id, content=content, **kwds) run_response = self.tools_post(payload) if wait: - self.wait_for_tool_run(history_id, run_response, assert_ok=kwds.get('assert_ok', True)) + self.wait_for_tool_run(history_id, run_response, assert_ok=kwds.get("assert_ok", True)) return run_response - def fetch(self, payload: dict, assert_ok: bool = True, timeout: timeout_type = DEFAULT_TIMEOUT, wait: Optional[bool] = None): + def fetch( + self, + payload: dict, + assert_ok: bool = True, + timeout: timeout_type = DEFAULT_TIMEOUT, + wait: Optional[bool] = None, + ): tool_response = self._post("tools/fetch", data=payload) if wait is None: wait = assert_ok @@ -492,10 +492,12 @@ class BaseDatasetPopulator(BasePopulator): def fetch_hdas(self, history_id: str, items: List[Dict[str, Any]], wait: bool = True) -> List[Dict[str, Any]]: destination = {"type": "hdas"} - targets = [{ - "destination": destination, - "items": items, - }] + targets = [ + { + "destination": destination, + "items": items, + } + ] payload = { "history_id": history_id, "targets": json.dumps(targets), @@ -512,11 +514,17 @@ class BaseDatasetPopulator(BasePopulator): def tag_dataset(self, history_id, hda_id, tags): url = f"histories/{history_id}/contents/{hda_id}" - response = self._put(url, {'tags': tags}, json=True) + response = self._put(url, {"tags": tags}, json=True) response.raise_for_status() return response.json() - def wait_for_tool_run(self, history_id: str, run_response: requests.Response, timeout: timeout_type = DEFAULT_TIMEOUT, assert_ok: bool = True): + def wait_for_tool_run( + self, + history_id: str, + run_response: requests.Response, + timeout: timeout_type = DEFAULT_TIMEOUT, + assert_ok: bool = True, + ): job = self.check_run(run_response) self.wait_for_job(job["id"], timeout=timeout) self.wait_for_history(history_id, assert_ok=assert_ok, timeout=timeout) @@ -528,15 +536,18 @@ class BaseDatasetPopulator(BasePopulator): job = run["jobs"][0] return job - def wait_for_history(self, history_id: str, assert_ok: bool = False, timeout: timeout_type = DEFAULT_TIMEOUT) -> str: + def wait_for_history( + self, history_id: str, assert_ok: bool = False, timeout: timeout_type = DEFAULT_TIMEOUT + ) -> str: try: - return wait_on_state(lambda: self._get(f"histories/{history_id}"), desc="history state", assert_ok=assert_ok, timeout=timeout) + return wait_on_state( + lambda: self._get(f"histories/{history_id}"), desc="history state", assert_ok=assert_ok, timeout=timeout + ) except AssertionError: self._summarize_history(history_id) raise def wait_for_history_jobs(self, history_id: str, assert_ok: bool = False, timeout: timeout_type = DEFAULT_TIMEOUT): - def has_active_jobs(): active_jobs = self.active_history_jobs(history_id) if len(active_jobs) == 0: @@ -555,7 +566,9 @@ class BaseDatasetPopulator(BasePopulator): self.wait_for_history(history_id, assert_ok=True, timeout=timeout) def wait_for_job(self, job_id: str, assert_ok: bool = False, timeout: timeout_type = DEFAULT_TIMEOUT): - return wait_on_state(lambda: self.get_job_details(job_id, full=True), desc="job state", assert_ok=assert_ok, timeout=timeout) + return wait_on_state( + lambda: self.get_job_details(job_id, full=True), desc="job state", assert_ok=assert_ok, timeout=timeout + ) def get_job_details(self, job_id: str, full: bool = False) -> Response: return self._get(f"jobs/{job_id}?full={full}") @@ -584,7 +597,7 @@ class BaseDatasetPopulator(BasePopulator): delete_response.raise_for_status() def delete_dataset(self, history_id: str, content_id: str, purge: bool = False) -> Response: - delete_response = self._delete(f"histories/{history_id}/contents/{content_id}", {'purge': purge}, json=True) + delete_response = self._delete(f"histories/{history_id}/contents/{content_id}", {"purge": purge}, json=True) return delete_response def create_tool_from_path(self, tool_path: str) -> Dict[str, Any]: @@ -664,20 +677,20 @@ class BaseDatasetPopulator(BasePopulator): def upload_payload(self, history_id: str, content: Optional[str] = None, **kwds) -> dict: name = kwds.get("name", "Test_Dataset") dbkey = kwds.get("dbkey", "?") - file_type = kwds.get("file_type", 'txt') + file_type = kwds.get("file_type", "txt") upload_params = { - 'files_0|NAME': name, - 'dbkey': dbkey, - 'file_type': file_type, + "files_0|NAME": name, + "dbkey": dbkey, + "file_type": file_type, } if dbkey is None: del upload_params["dbkey"] if content is None: upload_params["files_0|ftp_files"] = kwds.get("ftp_files") - elif hasattr(content, 'read'): + elif hasattr(content, "read"): upload_params["files_0|file_data"] = content else: - upload_params['files_0|url_paste'] = content + upload_params["files_0|url_paste"] = content if "to_posix_lines" in kwds: upload_params["files_0|to_posix_lines"] = kwds["to_posix_lines"] @@ -687,10 +700,7 @@ class BaseDatasetPopulator(BasePopulator): upload_params["files_0|auto_decompress"] = kwds["auto_decompress"] upload_params.update(kwds.get("extra_inputs", {})) return self.run_tool_payload( - tool_id='upload1', - inputs=upload_params, - history_id=history_id, - upload_type='upload_dataset' + tool_id="upload1", inputs=upload_params, history_id=history_id, upload_type="upload_dataset" ) def get_remote_files(self, target: str = "ftp") -> dict: @@ -708,12 +718,7 @@ class BaseDatasetPopulator(BasePopulator): kwds["__files"][key] = value del inputs[key] - return dict( - tool_id=tool_id, - inputs=json.dumps(inputs), - history_id=history_id, - **kwds - ) + return dict(tool_id=tool_id, inputs=json.dumps(inputs), history_id=history_id, **kwds) def build_tool_state(self, tool_id: str, history_id: str): response = self._post(f"tools/{tool_id}/build?history_id={history_id}") @@ -733,16 +738,16 @@ class BaseDatasetPopulator(BasePopulator): tool_response = self._post(url, data=payload) return tool_response - def get_history_dataset_content(self, history_id: str, wait=True, filename=None, type='text', raw=False, **kwds): + def get_history_dataset_content(self, history_id: str, wait=True, filename=None, type="text", raw=False, **kwds): dataset_id = self.__history_content_id(history_id, wait=wait, **kwds) data = {} if filename: data["filename"] = filename if raw: - data['raw'] = True + data["raw"] = True display_response = self._get_contents_request(history_id, f"/{dataset_id}/display", data=data) assert display_response.status_code == 200, display_response.text - if type == 'text': + if type == "text": return display_response.text else: return display_response.content @@ -795,7 +800,7 @@ class BaseDatasetPopulator(BasePopulator): def run_exit_code_from_file(self, history_id: str, hdca_id: str) -> dict: exit_code_inputs = { - "input": {'batch': True, 'values': [{"src": "hdca", "id": hdca_id}]}, + "input": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}, } response = self.run_tool("exit_code_from_file", exit_code_inputs, history_id) self.wait_for_history(history_id, assert_ok=False) @@ -825,8 +830,10 @@ class BaseDatasetPopulator(BasePopulator): raise Exception(f"Could not find content with HID [{hid}] in [{history_contents}]") else: # No hid specified - just grab most recent element of correct content type - if kwds.get('history_content_type'): - history_contents = [c for c in history_contents if c['history_content_type'] == kwds['history_content_type']] + if kwds.get("history_content_type"): + history_contents = [ + c for c in history_contents if c["history_content_type"] == kwds["history_content_type"] + ] history_content_id = history_contents[-1]["id"] return history_content_id @@ -839,9 +846,12 @@ class BaseDatasetPopulator(BasePopulator): return self._get(url, data=data) def ds_entry(self, history_content: dict) -> dict: - src = 'hda' - if 'history_content_type' in history_content and history_content['history_content_type'] == "dataset_collection": - src = 'hdca' + src = "hda" + if ( + "history_content_type" in history_content + and history_content["history_content_type"] == "dataset_collection" + ): + src = "hdca" return dict(src=src, id=history_content["id"]) def dataset_storage_info(self, dataset_id: str) -> dict: @@ -929,23 +939,20 @@ class BaseDatasetPopulator(BasePopulator): def validated(): metadata = self.get_history_dataset_details(history_id, dataset_id=dataset_id) - validated_state = metadata['validated_state'] - if validated_state == 'unknown': + validated_state = metadata["validated_state"] + if validated_state == "unknown": return else: return validated_state - return wait_on( - validated, - "dataset validation" - ) + return wait_on(validated, "dataset validation") def setup_history_for_export_testing(self, history_name): history_id = self.new_history(name=history_name) hda = self.new_dataset(history_id, content="1 2 3") - tags = ['name:name'] - response = self.tag_dataset(history_id, hda['id'], tags=tags) - assert response['tags'] == tags + tags = ["name:name"] + response = self.tag_dataset(history_id, hda["id"], tags=tags) + assert response["tags"] == tags deleted_hda = self.new_dataset(history_id, content="1 2 3", wait=True) self.delete_dataset(history_id, deleted_hda["id"]) deleted_details = self.get_history_dataset_details(history_id, id=deleted_hda["id"]) @@ -958,6 +965,7 @@ class BaseDatasetPopulator(BasePopulator): put_response.raise_for_status() if put_response.status_code == 202: + def export_ready_response(): put_response = self._put(url) if put_response.status_code == 202: @@ -976,7 +984,7 @@ class BaseDatasetPopulator(BasePopulator): put_response = self.prepare_export(history_id, data) response = put_response.json() api_asserts.assert_has_keys(response, "download_url") - download_url = urllib.parse.urljoin(self.galaxy_interactor.api_url, response["download_url"].strip('/')) + download_url = urllib.parse.urljoin(self.galaxy_interactor.api_url, response["download_url"].strip("/")) if check_download: self.get_export_url(download_url) @@ -995,7 +1003,7 @@ class BaseDatasetPopulator(BasePopulator): files["archive_file"] = archive_file import_response = self._post("histories", data=import_data, files=files) api_asserts.assert_status_code_is(import_response, 200) - return import_response.json()['id'] + return import_response.json()["id"] def import_history_and_wait_for_name(self, import_data, history_name): def history_names(): @@ -1027,7 +1035,6 @@ class BaseDatasetPopulator(BasePopulator): return history_index_response.json() def wait_on_history_length(self, history_id, wait_on_history_length): - def history_has_length(): history_length = self.history_length(history_id) return None if history_length != wait_on_history_length else True @@ -1057,14 +1064,19 @@ class BaseDatasetPopulator(BasePopulator): def get_random_name(self, prefix=None, suffix=None, len=10): # stolen from navigates_galaxy.py - return '{}{}{}'.format( - prefix or '', - ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(len)), - suffix or '', + return "{}{}{}".format( + prefix or "", + "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(len)), + suffix or "", ) def wait_for_dataset(self, history_id, dataset_id, assert_ok=False, timeout=DEFAULT_TIMEOUT): - return wait_on_state(lambda: self._get(f"histories/{history_id}/contents/{dataset_id}"), desc="dataset state", assert_ok=assert_ok, timeout=timeout) + return wait_on_state( + lambda: self._get(f"histories/{history_id}/contents/{dataset_id}"), + desc="dataset state", + assert_ok=assert_ok, + timeout=timeout, + ) class GalaxyInteractorHttpMixin: @@ -1094,7 +1106,6 @@ class GalaxyInteractorHttpMixin: class DatasetPopulator(GalaxyInteractorHttpMixin, BaseDatasetPopulator): - def __init__(self, galaxy_interactor): self.galaxy_interactor = galaxy_interactor @@ -1104,7 +1115,7 @@ class DatasetPopulator(GalaxyInteractorHttpMixin, BaseDatasetPopulator): class BaseWorkflowPopulator(BasePopulator): dataset_populator: BaseDatasetPopulator - dataset_collection_populator: 'BaseDatasetCollectionPopulator' + dataset_collection_populator: "BaseDatasetCollectionPopulator" def load_workflow(self, name: str, content: str = workflow_str, add_pja=False) -> dict: workflow = json.loads(content) @@ -1150,10 +1161,7 @@ class BaseWorkflowPopulator(BasePopulator): return uploaded_workflow_id def create_workflow_response(self, workflow: Dict[str, Any], **create_kwds) -> Response: - data = dict( - workflow=json.dumps(workflow), - **create_kwds - ) + data = dict(workflow=json.dumps(workflow), **create_kwds) upload_response = self._post("workflows/upload", data=data) return upload_response @@ -1167,11 +1175,15 @@ class BaseWorkflowPopulator(BasePopulator): workflow_yaml_wrapped = self.download_workflow(workflow_id, style="format2_wrapped_yaml") assert "yaml_content" in workflow_yaml_wrapped, workflow_yaml_wrapped round_trip_converted_content = workflow_yaml_wrapped["yaml_content"] - workflow_id = self.upload_yaml_workflow(round_trip_converted_content, client_convert=False, round_trip_conversion=False) + workflow_id = self.upload_yaml_workflow( + round_trip_converted_content, client_convert=False, round_trip_conversion=False + ) return workflow_id - def wait_for_invocation(self, workflow_id: str, invocation_id: str, timeout: timeout_type = DEFAULT_TIMEOUT, assert_ok: bool = True): + def wait_for_invocation( + self, workflow_id: str, invocation_id: str, timeout: timeout_type = DEFAULT_TIMEOUT, assert_ok: bool = True + ): url = f"workflows/{workflow_id}/usage/{invocation_id}" def workflow_state(): @@ -1184,8 +1196,11 @@ class BaseWorkflowPopulator(BasePopulator): api_asserts.assert_status_code_is(history_invocations_response, 200) return history_invocations_response.json() - def wait_for_history_workflows(self, history_id, assert_ok=True, timeout=DEFAULT_TIMEOUT, expected_invocation_count=None): + def wait_for_history_workflows( + self, history_id, assert_ok=True, timeout=DEFAULT_TIMEOUT, expected_invocation_count=None + ): if expected_invocation_count is not None: + def invocation_count(): invocations = self.history_invocations(history_id) if len(invocations) == expected_invocation_count: @@ -1198,13 +1213,13 @@ class BaseWorkflowPopulator(BasePopulator): self.wait_for_workflow(workflow_id, invocation_id, history_id, timeout=timeout, assert_ok=assert_ok) def wait_for_workflow(self, workflow_id, invocation_id, history_id, assert_ok=True, timeout=DEFAULT_TIMEOUT): - """ Wait for a workflow invocation to completely schedule and then history - to be complete. """ + """Wait for a workflow invocation to completely schedule and then history + to be complete.""" self.wait_for_invocation(workflow_id, invocation_id, timeout=timeout, assert_ok=assert_ok) self.dataset_populator.wait_for_history_jobs(history_id, assert_ok=assert_ok, timeout=timeout) def get_invocation(self, invocation_id, step_details=False): - r = self._get(f"invocations/{invocation_id}", data={'step_details': step_details}) + r = self._get(f"invocations/{invocation_id}", data={"step_details": step_details}) r.raise_for_status() return r.json() @@ -1213,15 +1228,37 @@ class BaseWorkflowPopulator(BasePopulator): bco_response.raise_for_status() return bco_response.json() - def validate_biocompute_object(self, bco, expected_schema_version='https://w3id.org/ieee/ieee-2791-schema/2791object.json'): + def validate_biocompute_object( + self, bco, expected_schema_version="https://w3id.org/ieee/ieee-2791-schema/2791object.json" + ): # TODO: actually use jsonref and jsonschema to validate this someday - api_asserts.assert_has_keys(bco, "object_id", "spec_version", "etag", "provenance_domain", "usability_domain", "description_domain", "execution_domain", "parametric_domain", "io_domain", "error_domain") - assert bco['spec_version'] == expected_schema_version - api_asserts.assert_has_keys(bco['description_domain'], "keywords", "xref", "platform", "pipeline_steps") - api_asserts.assert_has_keys(bco['execution_domain'], "script_access_type", "script", "script_driver", "software_prerequisites", "external_data_endpoints", "environment_variables") - for p in bco['parametric_domain']: + api_asserts.assert_has_keys( + bco, + "object_id", + "spec_version", + "etag", + "provenance_domain", + "usability_domain", + "description_domain", + "execution_domain", + "parametric_domain", + "io_domain", + "error_domain", + ) + assert bco["spec_version"] == expected_schema_version + api_asserts.assert_has_keys(bco["description_domain"], "keywords", "xref", "platform", "pipeline_steps") + api_asserts.assert_has_keys( + bco["execution_domain"], + "script_access_type", + "script", + "script_driver", + "software_prerequisites", + "external_data_endpoints", + "environment_variables", + ) + for p in bco["parametric_domain"]: api_asserts.assert_has_keys(p, "param", "value", "step") - api_asserts.assert_has_keys(bco['io_domain'], "input_subdomain", "output_subdomain") + api_asserts.assert_has_keys(bco["io_domain"], "input_subdomain", "output_subdomain") def invoke_workflow_raw(self, workflow_id: str, request: dict, assert_ok: bool = False) -> Response: url = f"workflows/{workflow_id}/invocations" @@ -1230,7 +1267,15 @@ class BaseWorkflowPopulator(BasePopulator): invocation_response.raise_for_status() return invocation_response - def invoke_workflow(self, workflow_id: str, history_id: Optional[str] = None, inputs: Optional[dict] = None, request: Optional[dict] = None, assert_ok: bool = True, inputs_by: str = 'step_index'): + def invoke_workflow( + self, + workflow_id: str, + history_id: Optional[str] = None, + inputs: Optional[dict] = None, + request: Optional[dict] = None, + assert_ok: bool = True, + inputs_by: str = "step_index", + ): if inputs is None: inputs = {} @@ -1252,8 +1297,17 @@ class BaseWorkflowPopulator(BasePopulator): else: return invocation_response - def invoke_workflow_and_wait(self, workflow_id: str, history_id: Optional[str] = None, inputs: Optional[dict] = None, request: Optional[dict] = None, assert_ok: bool = True): - invoke_return = self.invoke_workflow(workflow_id, history_id=history_id, inputs=inputs, request=request, assert_ok=assert_ok) + def invoke_workflow_and_wait( + self, + workflow_id: str, + history_id: Optional[str] = None, + inputs: Optional[dict] = None, + request: Optional[dict] = None, + assert_ok: bool = True, + ): + invoke_return = self.invoke_workflow( + workflow_id, history_id=history_id, inputs=inputs, request=request, assert_ok=assert_ok + ) if assert_ok: invocation_id = invoke_return else: @@ -1264,7 +1318,7 @@ class BaseWorkflowPopulator(BasePopulator): if history_id is None and request: history_id = request["history"] if history_id.startswith("hist_id="): - history_id = history_id[len("hist_id="):] + history_id = history_id[len("hist_id=") :] self.wait_for_workflow(workflow_id, invocation_id, history_id, assert_ok=assert_ok) return invoke_return @@ -1273,12 +1327,14 @@ class BaseWorkflowPopulator(BasePopulator): api_asserts.assert_status_code_is(response, 200) return response.json() - def download_workflow(self, workflow_id: str, style: Optional[str] = None, history_id: Optional[str] = None) -> dict: + def download_workflow( + self, workflow_id: str, style: Optional[str] = None, history_id: Optional[str] = None + ) -> dict: params = {} if style is not None: params["style"] = style if history_id is not None: - params['history_id'] = history_id + params["history_id"] = history_id response = self._get(f"workflows/{workflow_id}/download", data=params) api_asserts.assert_status_code_is(response, 200) if style != "format2": @@ -1287,14 +1343,14 @@ class BaseWorkflowPopulator(BasePopulator): return ordered_load(response.text) def update_workflow(self, workflow_id: str, workflow_object: dict) -> Response: - data = dict( - workflow=workflow_object - ) - raw_url = f'workflows/{workflow_id}' + data = dict(workflow=workflow_object) + raw_url = f"workflows/{workflow_id}" put_response = self._put(raw_url, data, json=True) return put_response - def refactor_workflow(self, workflow_id: str, actions: list, dry_run: Optional[bool] = None, style: Optional[str] = None) -> Response: + def refactor_workflow( + self, workflow_id: str, actions: list, dry_run: Optional[bool] = None, style: Optional[str] = None + ) -> Response: data: Dict[str, Any] = dict( actions=actions, ) @@ -1302,7 +1358,7 @@ class BaseWorkflowPopulator(BasePopulator): data["style"] = style if dry_run is not None: data["dry_run"] = dry_run - raw_url = f'workflows/{workflow_id}/refactor' + raw_url = f"workflows/{workflow_id}/refactor" put_response = self._put(raw_url, data, json=True) return put_response @@ -1313,7 +1369,20 @@ class BaseWorkflowPopulator(BasePopulator): put_respose = self.update_workflow(workflow_id, workflow_object) put_respose.raise_for_status() - def run_workflow(self, has_workflow, test_data=None, history_id=None, wait=True, source_type=None, jobs_descriptions=None, expected_response=200, assert_ok=True, client_convert=None, round_trip_format_conversion=False, raw_yaml=False): + def run_workflow( + self, + has_workflow, + test_data=None, + history_id=None, + wait=True, + source_type=None, + jobs_descriptions=None, + expected_response=200, + assert_ok=True, + client_convert=None, + round_trip_format_conversion=False, + raw_yaml=False, + ): """High-level wrapper around workflow API, etc. to invoke format 2 workflows.""" workflow_populator = self if client_convert is None: @@ -1324,7 +1393,7 @@ class BaseWorkflowPopulator(BasePopulator): source_type=source_type, client_convert=client_convert, round_trip_format_conversion=round_trip_format_conversion, - raw_yaml=raw_yaml + raw_yaml=raw_yaml, ) if test_data is None: @@ -1337,17 +1406,19 @@ class BaseWorkflowPopulator(BasePopulator): if not isinstance(test_data, dict): test_data = yaml.safe_load(test_data) - parameters = test_data.pop('step_parameters', {}) + parameters = test_data.pop("step_parameters", {}) replacement_parameters = test_data.pop("replacement_parameters", {}) if history_id is None: history_id = self.dataset_populator.new_history() - inputs, label_map, has_uploads = load_data_dict(history_id, test_data, self.dataset_populator, self.dataset_collection_populator) + inputs, label_map, has_uploads = load_data_dict( + history_id, test_data, self.dataset_populator, self.dataset_collection_populator + ) workflow_request: Dict[str, Any] = dict( history=f"hist_id={history_id}", workflow_id=workflow_id, ) workflow_request["inputs"] = json.dumps(label_map) - workflow_request["inputs_by"] = 'name' + workflow_request["inputs_by"] = "name" if parameters: workflow_request["parameters"] = json.dumps(parameters) workflow_request["parameters_normalized"] = True @@ -1361,7 +1432,7 @@ class BaseWorkflowPopulator(BasePopulator): if expected_response != 200: assert not assert_ok return invocation - invocation_id = invocation.get('id') + invocation_id = invocation.get("id") if invocation_id: # Wait for workflow to become fully scheduled and then for all jobs # complete. @@ -1375,7 +1446,7 @@ class BaseWorkflowPopulator(BasePopulator): inputs=inputs, jobs=jobs, invocation=invocation, - workflow_request=workflow_request + workflow_request=workflow_request, ) def dump_workflow(self, workflow_id, style=None): @@ -1400,7 +1471,13 @@ class BaseWorkflowPopulator(BasePopulator): ds_map[key] = label_map[label] return json.dumps(ds_map) - def setup_workflow_run(self, workflow: Optional[Dict[str, Any]] = None, inputs_by: str = 'step_id', history_id: Optional[str] = None, workflow_id: Optional[str] = None) -> Tuple[Dict[str, Any], str, str]: + def setup_workflow_run( + self, + workflow: Optional[Dict[str, Any]] = None, + inputs_by: str = "step_id", + history_id: Optional[str] = None, + workflow_id: Optional[str] = None, + ) -> Tuple[Dict[str, Any], str, str]: ds_entry = self.dataset_populator.ds_entry if not workflow_id: assert workflow, "If workflow_id not specified, must specify a workflow dictionary to load" @@ -1412,23 +1489,17 @@ class BaseWorkflowPopulator(BasePopulator): workflow_request = dict( history=f"hist_id={history_id}", ) - label_map = { - 'WorkflowInput1': ds_entry(hda1), - 'WorkflowInput2': ds_entry(hda2) - } - if inputs_by == 'step_id': + label_map = {"WorkflowInput1": ds_entry(hda1), "WorkflowInput2": ds_entry(hda2)} + if inputs_by == "step_id": ds_map = self.build_ds_map(workflow_id, label_map) workflow_request["ds_map"] = ds_map elif inputs_by == "step_index": - index_map = { - '0': ds_entry(hda1), - '1': ds_entry(hda2) - } + index_map = {"0": ds_entry(hda1), "1": ds_entry(hda2)} workflow_request["inputs"] = json.dumps(index_map) - workflow_request["inputs_by"] = 'step_index' + workflow_request["inputs_by"] = "step_index" elif inputs_by == "name": workflow_request["inputs"] = json.dumps(label_map) - workflow_request["inputs_by"] = 'name' + workflow_request["inputs_by"] = "name" elif inputs_by in ["step_uuid", "uuid_implicitly"]: assert workflow, f"Must specify workflow for this inputs_by {inputs_by} parameter value" uuid_map = { @@ -1445,9 +1516,9 @@ class BaseWorkflowPopulator(BasePopulator): state = self.wait_for_invocation(workflow_id, invocation_id) if assert_ok: assert state == "scheduled", state - time.sleep(.5) + time.sleep(0.5) self.dataset_populator.wait_for_history_jobs(history_id, assert_ok=assert_ok) - time.sleep(.5) + time.sleep(0.5) class RunJobsSummary(NamedTuple): @@ -1461,7 +1532,6 @@ class RunJobsSummary(NamedTuple): class WorkflowPopulator(GalaxyInteractorHttpMixin, BaseWorkflowPopulator, ImporterGalaxyInterface): - def __init__(self, galaxy_interactor): self.galaxy_interactor = galaxy_interactor self.dataset_populator = DatasetPopulator(galaxy_interactor) @@ -1472,7 +1542,7 @@ class WorkflowPopulator(GalaxyInteractorHttpMixin, BaseWorkflowPopulator, Import def import_workflow(self, workflow, **kwds) -> Dict[str, Any]: workflow_str = json.dumps(workflow, indent=4) data = { - 'workflow': workflow_str, + "workflow": workflow_str, } data.update(**kwds) upload_response = self._post("workflows", data=data) @@ -1480,7 +1550,7 @@ class WorkflowPopulator(GalaxyInteractorHttpMixin, BaseWorkflowPopulator, Import return upload_response.json() def import_tool(self, tool) -> Dict[str, Any]: - """ Import a workflow via POST /api/workflows or + """Import a workflow via POST /api/workflows or comparable interface into Galaxy. """ upload_response = self._import_tool_response(tool) @@ -1489,9 +1559,7 @@ class WorkflowPopulator(GalaxyInteractorHttpMixin, BaseWorkflowPopulator, Import def _import_tool_response(self, tool) -> Response: tool_str = json.dumps(tool, indent=4) - data = { - 'representation': tool_str - } + data = {"representation": tool_str} upload_response = self._post("dynamic_tools", data=data, admin=True) return upload_response @@ -1514,7 +1582,7 @@ class WorkflowPopulator(GalaxyInteractorHttpMixin, BaseWorkflowPopulator, Import scale_workflow_steps = [ {"tool_id": "create_input_collection", "state": {"collection_size": collection_size}, "label": "wf_input"}, - {"tool_id": "cat", "state": {"input1": self._link("wf_input", "output")}, "label": "cat_0"} + {"tool_id": "cat", "state": {"input1": self._link("wf_input", "output")}, "label": "cat_0"}, ] for i in range(workflow_depth): @@ -1536,7 +1604,11 @@ class WorkflowPopulator(GalaxyInteractorHttpMixin, BaseWorkflowPopulator, Import scale_workflow_steps = [ {"tool_id": "create_input_collection", "state": {"collection_size": collection_size}, "label": "wf_input"}, - {"tool_id": "cat", "state": {"input1": self._link("wf_input"), "input2": self._link("wf_input")}, "label": "cat_0"} + { + "tool_id": "cat", + "state": {"input1": self._link("wf_input"), "input2": self._link("wf_input")}, + "label": "cat_0", + }, ] for i in range(workflow_depth): @@ -1585,7 +1657,6 @@ class WorkflowPopulator(GalaxyInteractorHttpMixin, BaseWorkflowPopulator, Import class LibraryPopulator: - def __init__(self, galaxy_interactor): self.galaxy_interactor = galaxy_interactor self.dataset_populator = DatasetPopulator(galaxy_interactor) @@ -1615,7 +1686,7 @@ class LibraryPopulator: page: Optional[int] = 1, page_limit: Optional[int] = 1000, q: Optional[str] = None, - admin: Optional[bool] = True + admin: Optional[bool] = True, ): query = f"&q={q}" if q else "" response = self.galaxy_interactor.get( @@ -1669,7 +1740,9 @@ class LibraryPopulator: self._set_permissions(library_id, permissions) def _set_permissions(self, library_id, permissions): - response = self.galaxy_interactor.post(f"libraries/{library_id}/permissions", data=permissions, admin=True, json=True) + response = self.galaxy_interactor.post( + f"libraries/{library_id}/permissions", data=permissions, admin=True, json=True + ) api_asserts.assert_status_code_is(response, 200) def user_email(self): @@ -1776,10 +1849,9 @@ class BaseDatasetCollectionPopulator: dataset_populator: BaseDatasetPopulator def create_list_from_pairs(self, history_id, pairs, name="Dataset Collection from pairs"): - return self.create_nested_collection(history_id=history_id, - collection=pairs, - collection_type='list:paired', - name=name) + return self.create_nested_collection( + history_id=history_id, collection=pairs, collection_type="list:paired", name=name + ) def nested_collection_identifiers(self, history_id, collection_type): rank_types = list(reversed(collection_type.split(":"))) @@ -1793,16 +1865,20 @@ class BaseDatasetCollectionPopulator: for i, rank_type in enumerate(reversed(rank_types[1:])): name = f"test_level_{i + 1}" if rank_type == "list" else "paired" - identifiers = [dict( - src="new_collection", - name=name, - collection_type=nested_collection_type, - element_identifiers=identifiers, - )] + identifiers = [ + dict( + src="new_collection", + name=name, + collection_type=nested_collection_type, + element_identifiers=identifiers, + ) + ] nested_collection_type = f"{rank_type}:{nested_collection_type}" return identifiers - def create_nested_collection(self, history_id, collection_type, name=None, collection=None, element_identifiers=None): + def create_nested_collection( + self, history_id, collection_type, name=None, collection=None, element_identifiers=None + ): """Create a nested collection either from collection or using collection_type).""" assert collection_type is not None name = name or f"Test {collection_type}" @@ -1810,11 +1886,7 @@ class BaseDatasetCollectionPopulator: assert element_identifiers is None element_identifiers = [] for i, pair in enumerate(collection): - element_identifiers.append(dict( - name=f"test{i}", - src="hdca", - id=pair - )) + element_identifiers.append(dict(name=f"test{i}", src="hdca", id=pair)) if element_identifiers is None: element_identifiers = self.nested_collection_identifiers(history_id, collection_type) @@ -1828,46 +1900,44 @@ class BaseDatasetCollectionPopulator: return self.__create(payload) def create_list_of_pairs_in_history(self, history_id, **kwds): - return self.upload_collection(history_id, "list:paired", elements=[ - { - "name": "test0", - "elements": [ - {"src": "pasted", "paste_content": "TestData123", "name": "forward"}, - {"src": "pasted", "paste_content": "TestData123", "name": "reverse"}, - ] - } - ]) + return self.upload_collection( + history_id, + "list:paired", + elements=[ + { + "name": "test0", + "elements": [ + {"src": "pasted", "paste_content": "TestData123", "name": "forward"}, + {"src": "pasted", "paste_content": "TestData123", "name": "reverse"}, + ], + } + ], + ) def create_list_of_list_in_history(self, history_id, **kwds): # create_nested_collection will generate nested collection from just datasets, # this function uses recursive generation of history hdcas. - collection_type = kwds.pop('collection_type', 'list:list') - collection_types = collection_type.split(':') - list = self.create_list_in_history(history_id, **kwds).json()['id'] - current_collection_type = 'list' + collection_type = kwds.pop("collection_type", "list:list") + collection_types = collection_type.split(":") + list = self.create_list_in_history(history_id, **kwds).json()["id"] + current_collection_type = "list" for collection_type in collection_types[1:]: current_collection_type = f"{current_collection_type}:{collection_type}" - response = self.create_nested_collection(history_id=history_id, - collection_type=current_collection_type, - name=current_collection_type, - collection=[list]) - list = response.json()['id'] + response = self.create_nested_collection( + history_id=history_id, + collection_type=current_collection_type, + name=current_collection_type, + collection=[list], + ) + list = response.json()["id"] return response def create_pair_in_history(self, history_id, **kwds): - payload = self.create_pair_payload( - history_id, - instance_type="history", - **kwds - ) + payload = self.create_pair_payload(history_id, instance_type="history", **kwds) return self.__create(payload) def create_list_in_history(self, history_id, **kwds): - payload = self.create_list_payload( - history_id, - instance_type="history", - **kwds - ) + payload = self.create_list_payload(history_id, instance_type="history", **kwds) return self.__create(payload) def upload_collection(self, history_id, collection_type, elements, **kwds): @@ -1878,7 +1948,9 @@ class BaseDatasetCollectionPopulator: return self.__create_payload(history_id, identifiers_func=self.list_identifiers, collection_type="list", **kwds) def create_pair_payload(self, history_id, **kwds): - return self.__create_payload(history_id, identifiers_func=self.pair_identifiers, collection_type="paired", **kwds) + return self.__create_payload( + history_id, identifiers_func=self.pair_identifiers, collection_type="paired", **kwds + ) def __create_payload(self, *args, **kwds): direct_upload = kwds.pop("direct_upload", False) @@ -1930,12 +2002,14 @@ class BaseDatasetCollectionPopulator: name = kwds.get("name", "Test Dataset Collection") - targets = [{ - "destination": {"type": "hdca"}, - "elements": elements, - "collection_type": collection_type, - "name": name, - }] + targets = [ + { + "destination": {"type": "hdca"}, + "elements": elements, + "collection_type": collection_type, + "name": name, + } + ] payload = dict( history_id=history_id, targets=json.dumps(targets), @@ -1945,7 +2019,9 @@ class BaseDatasetCollectionPopulator: def wait_for_fetched_collection(self, fetch_response): self.dataset_populator.wait_for_job(fetch_response["jobs"][0]["id"], assert_ok=True) initial_dataset_collection = fetch_response["outputs"][0] - dataset_collection = self.dataset_populator.get_history_collection_details(initial_dataset_collection["history_id"], hid=initial_dataset_collection["hid"]) + dataset_collection = self.dataset_populator.get_history_collection_details( + initial_dataset_collection["history_id"], hid=initial_dataset_collection["hid"] + ) return dataset_collection def __create_payload_collection(self, history_id, identifiers_func, collection_type, **kwds): @@ -1960,11 +2036,7 @@ class BaseDatasetCollectionPopulator: if "name" not in kwds: kwds["name"] = "Test Dataset Collection" - payload = dict( - history_id=history_id, - collection_type=collection_type, - **kwds - ) + payload = dict(history_id=history_id, collection_type=collection_type, **kwds) return payload def pair_identifiers(self, history_id, contents=None): @@ -1985,11 +2057,13 @@ class BaseDatasetCollectionPopulator: def hda_to_identifier(i, hda): return dict(name=contents[i][0], src="hda", id=hda["id"]) + else: hdas = self.__datasets(history_id, count=count, contents=contents) def hda_to_identifier(i, hda): return dict(name=f"data{i + 1}", src="hda", id=hda["id"]) + element_identifiers = [hda_to_identifier(i, hda) for (i, hda) in enumerate(hdas)] return element_identifiers @@ -2013,13 +2087,15 @@ class BaseDatasetCollectionPopulator: def wait_for_dataset_collection(self, create_payload, assert_ok=False, timeout=DEFAULT_TIMEOUT): for element in create_payload["elements"]: - if element['element_type'] == 'hda': - self.dataset_populator.wait_for_dataset(history_id=element['object']['history_id'], - dataset_id=element['object']['id'], - assert_ok=assert_ok, - timeout=timeout) - elif element['element_type'] == 'dataset_collection': - self.wait_for_dataset_collection(element['object'], assert_ok=assert_ok, timeout=timeout) + if element["element_type"] == "hda": + self.dataset_populator.wait_for_dataset( + history_id=element["object"]["history_id"], + dataset_id=element["object"]["id"], + assert_ok=assert_ok, + timeout=timeout, + ) + elif element["element_type"] == "dataset_collection": + self.wait_for_dataset_collection(element["object"], assert_ok=assert_ok, timeout=timeout) @abstractmethod def _create_collection(self, payload: dict) -> Response: @@ -2027,7 +2103,6 @@ class BaseDatasetCollectionPopulator: class DatasetCollectionPopulator(BaseDatasetCollectionPopulator): - def __init__(self, galaxy_interactor: ApiTestInteractor): self.galaxy_interactor = galaxy_interactor self.dataset_populator = DatasetPopulator(galaxy_interactor) @@ -2054,7 +2129,7 @@ def load_data_dict(history_id, test_data, dataset_populator, dataset_collection_ for key, value in test_data.items(): is_dict = isinstance(value, dict) - if is_dict and ("elements" in value or value.get('collection_type')): + if is_dict and ("elements" in value or value.get("collection_type")): elements_data = value.get("elements", []) elements = [] for element_data in elements_data: @@ -2075,15 +2150,27 @@ def load_data_dict(history_id, test_data, dataset_populator, dataset_collection_ new_collection_kwds = {} if "name" in value: new_collection_kwds["name"] = value["name"] - collection_type = value.get('collection_type', '') + collection_type = value.get("collection_type", "") if collection_type == "list:paired": - fetch_response = dataset_collection_populator.create_list_of_pairs_in_history(history_id, contents=elements, **new_collection_kwds).json() - elif collection_type and ':' in collection_type: - fetch_response = {'outputs': [dataset_collection_populator.create_nested_collection(history_id, collection_type=collection_type, **new_collection_kwds).json()]} + fetch_response = dataset_collection_populator.create_list_of_pairs_in_history( + history_id, contents=elements, **new_collection_kwds + ).json() + elif collection_type and ":" in collection_type: + fetch_response = { + "outputs": [ + dataset_collection_populator.create_nested_collection( + history_id, collection_type=collection_type, **new_collection_kwds + ).json() + ] + } elif collection_type == "list": - fetch_response = dataset_collection_populator.create_list_in_history(history_id, contents=elements, direct_upload=True, **new_collection_kwds).json() + fetch_response = dataset_collection_populator.create_list_in_history( + history_id, contents=elements, direct_upload=True, **new_collection_kwds + ).json() else: - fetch_response = dataset_collection_populator.create_pair_in_history(history_id, contents=elements or None, direct_upload=True, **new_collection_kwds).json() + fetch_response = dataset_collection_populator.create_pair_in_history( + history_id, contents=elements or None, direct_upload=True, **new_collection_kwds + ).json() hdca_output = fetch_response["outputs"][0] hdca = dataset_populator.ds_entry(hdca_output) hdca["hid"] = hdca_output["hid"] @@ -2094,9 +2181,7 @@ def load_data_dict(history_id, test_data, dataset_populator, dataset_collection_ input_type = value["type"] if input_type == "File": content = open_test_data(value) - new_dataset_kwds = { - "content": content - } + new_dataset_kwds = {"content": content} if "name" in value: new_dataset_kwds["name"] = value["name"] if "file_type" in value: @@ -2126,7 +2211,7 @@ def stage_inputs( use_fetch_api=True, to_posix_lines=True, tool_or_workflow="workflow", - job_dir=None + job_dir=None, ): """Alternative to load_data_dict that uses production-style workflow inputs.""" kwds = dict( @@ -2137,9 +2222,7 @@ def stage_inputs( ) if job_dir is not None: kwds["job_dir"] = job_dir - inputs, datasets = InteractorStaging(galaxy_interactor, use_fetch_api=use_fetch_api).stage( - tool_or_workflow, **kwds - ) + inputs, datasets = InteractorStaging(galaxy_interactor, use_fetch_api=use_fetch_api).stage(tool_or_workflow, **kwds) return inputs, datasets @@ -2152,7 +2235,9 @@ def stage_rules_example(galaxy_interactor, history_id, example): return inputs -def wait_on_state(state_func: Callable, desc="state", skip_states=None, ok_states=None, assert_ok=False, timeout=DEFAULT_TIMEOUT) -> str: +def wait_on_state( + state_func: Callable, desc="state", skip_states=None, ok_states=None, assert_ok=False, timeout=DEFAULT_TIMEOUT +) -> str: def get_state(): response = state_func() assert response.status_code == 200, f"Failed to fetch state update while waiting. [{response.content}]" @@ -2178,6 +2263,7 @@ def wait_on_state(state_func: Callable, desc="state", skip_states=None, ok_state class GiHttpMixin: """Mixin for adapting Galaxy testing populators helpers to bioblend.""" + _gi: GalaxyClient @property @@ -2196,26 +2282,26 @@ class GiHttpMixin: if data is None: data = {} data = data.copy() - data['key'] = self._gi.key + data["key"] = self._gi.key return requests.post(self._url(route), data=data, headers=headers, timeout=DEFAULT_SOCKET_TIMEOUT) def _put(self, route, data=None, headers=None, admin=False, json: bool = False): if data is None: data = {} data = data.copy() - data['key'] = self._gi.key + data["key"] = self._gi.key return requests.put(self._url(route), data=data, headers=headers, timeout=DEFAULT_SOCKET_TIMEOUT) def _delete(self, route, data=None, headers=None, admin=False, json: bool = False): if data is None: data = {} data = data.copy() - data['key'] = self._gi.key + data["key"] = self._gi.key return requests.delete(self._url(route), data=data, headers=headers, timeout=DEFAULT_SOCKET_TIMEOUT) def _url(self, route): if route.startswith("/api/"): - route = route[len("/api/"):] + route = route[len("/api/") :] return f"{self._api_url()}/{route}" diff --git a/packages/app/galaxy/project_galaxy_app.py b/packages/app/galaxy/project_galaxy_app.py index 0f6da972170..97e3da53372 100644 --- a/packages/app/galaxy/project_galaxy_app.py +++ b/packages/app/galaxy/project_galaxy_app.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-app" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Application (backend)' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Application (backend)" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/auth/galaxy/project_galaxy_auth.py b/packages/auth/galaxy/project_galaxy_auth.py index 098069f6278..c9d4a49aa4b 100644 --- a/packages/auth/galaxy/project_galaxy_auth.py +++ b/packages/auth/galaxy/project_galaxy_auth.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-auth" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Auth Framework and Implementations' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Auth Framework and Implementations" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/containers/galaxy/project_galaxy_containers.py b/packages/containers/galaxy/project_galaxy_containers.py index 6556d81944b..03c735ef605 100644 --- a/packages/containers/galaxy/project_galaxy_containers.py +++ b/packages/containers/galaxy/project_galaxy_containers.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-containers" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Container Modeling and Interaction Abstractions' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Container Modeling and Interaction Abstractions" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/data/galaxy/project_galaxy_data.py b/packages/data/galaxy/project_galaxy_data.py index fadc67f1fda..dc1ab8194fd 100644 --- a/packages/data/galaxy/project_galaxy_data.py +++ b/packages/data/galaxy/project_galaxy_data.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-data" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Datatype Framework and Datatypes' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Datatype Framework and Datatypes" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/files/galaxy/project_galaxy_files.py b/packages/files/galaxy/project_galaxy_files.py index c4c7cd092fb..c7bdbe1d5b2 100644 --- a/packages/files/galaxy/project_galaxy_files.py +++ b/packages/files/galaxy/project_galaxy_files.py @@ -1,9 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-files" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy File Source Framework and Default Plugins' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy File Source Framework and Default Plugins" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" RAW_CONTENT_URL = f"https://raw.github.com/{PROJECT_USERAME}/{PROJECT_NAME}/master/" diff --git a/packages/job_execution/galaxy/project_galaxy_job_execution.py b/packages/job_execution/galaxy/project_galaxy_job_execution.py index 206a2c5b72d..15c4c324512 100644 --- a/packages/job_execution/galaxy/project_galaxy_job_execution.py +++ b/packages/job_execution/galaxy/project_galaxy_job_execution.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-job-execution" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Job Execution Runtime Utilities' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Job Execution Runtime Utilities" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/job_metrics/galaxy/project_galaxy_job_metrics.py b/packages/job_metrics/galaxy/project_galaxy_job_metrics.py index 475e83011c0..a90aa4d8477 100644 --- a/packages/job_metrics/galaxy/project_galaxy_job_metrics.py +++ b/packages/job_metrics/galaxy/project_galaxy_job_metrics.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-job-metrics" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Job Metrics' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Job Metrics" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/meta/galaxy/project_galaxy_meta.py b/packages/meta/galaxy/project_galaxy_meta.py index ad070331bcd..22e04986a1b 100644 --- a/packages/meta/galaxy/project_galaxy_meta.py +++ b/packages/meta/galaxy/project_galaxy_meta.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Server Metapackage' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Server Metapackage" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/objectstore/galaxy/project_galaxy_objectstore.py b/packages/objectstore/galaxy/project_galaxy_objectstore.py index 423632d8ef2..9f9be584e9a 100644 --- a/packages/objectstore/galaxy/project_galaxy_objectstore.py +++ b/packages/objectstore/galaxy/project_galaxy_objectstore.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-objectstore" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Objectstore Framework and Plugins' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Objectstore Framework and Plugins" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/selenium/galaxy/project_galaxy_selenium.py b/packages/selenium/galaxy/project_galaxy_selenium.py index 2e0042a5459..8e08c6a47ba 100644 --- a/packages/selenium/galaxy/project_galaxy_selenium.py +++ b/packages/selenium/galaxy/project_galaxy_selenium.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-selenium" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Selenium Interaction Framework' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Selenium Interaction Framework" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/test_api/galaxy/project_galaxy_test_api.py b/packages/test_api/galaxy/project_galaxy_test_api.py index 301d5943661..99427178fec 100644 --- a/packages/test_api/galaxy/project_galaxy_test_api.py +++ b/packages/test_api/galaxy/project_galaxy_test_api.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-test-api" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy API Tests' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy API Tests" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/test_base/galaxy/project_galaxy_test_base.py b/packages/test_base/galaxy/project_galaxy_test_base.py index d6158c2e7f3..0b0d5d94ee1 100644 --- a/packages/test_base/galaxy/project_galaxy_test_base.py +++ b/packages/test_base/galaxy/project_galaxy_test_base.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-test-base" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Testing Utilities' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Testing Utilities" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/test_driver/galaxy/project_galaxy_test_driver.py b/packages/test_driver/galaxy/project_galaxy_test_driver.py index 4470ad07595..edf016a60d4 100644 --- a/packages/test_driver/galaxy/project_galaxy_test_driver.py +++ b/packages/test_driver/galaxy/project_galaxy_test_driver.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-test-driver" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Test Driver' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Test Driver" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/test_selenium/galaxy/project_galaxy_test_selenium.py b/packages/test_selenium/galaxy/project_galaxy_test_selenium.py index 0b17f1627dd..2a04b8b0ce1 100644 --- a/packages/test_selenium/galaxy/project_galaxy_test_selenium.py +++ b/packages/test_selenium/galaxy/project_galaxy_test_selenium.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-test-selenium" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Selenium Tests' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Selenium Tests" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/tool_util/galaxy/project_galaxy_tool_util.py b/packages/tool_util/galaxy/project_galaxy_tool_util.py index c1e09ea37ab..a3d3a33444c 100644 --- a/packages/tool_util/galaxy/project_galaxy_tool_util.py +++ b/packages/tool_util/galaxy/project_galaxy_tool_util.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-tool-util" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Tool and Tool Dependency Utilities' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Tool and Tool Dependency Utilities" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/util/galaxy/project_galaxy_util.py b/packages/util/galaxy/project_galaxy_util.py index cb9df928969..9cbe67d463c 100644 --- a/packages/util/galaxy/project_galaxy_util.py +++ b/packages/util/galaxy/project_galaxy_util.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-util" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Generic Utilities' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Generic Utilities" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/web_framework/galaxy/project_galaxy_web_framework.py b/packages/web_framework/galaxy/project_galaxy_web_framework.py index 462187a6d4c..7b5e0e533a8 100644 --- a/packages/web_framework/galaxy/project_galaxy_web_framework.py +++ b/packages/web_framework/galaxy/project_galaxy_web_framework.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-web-framework" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Web Framework' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Web Framework" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/web_stack/galaxy/project_galaxy_web_stack.py b/packages/web_stack/galaxy/project_galaxy_web_stack.py index b73589ed7f7..a3a5aafe2be 100644 --- a/packages/web_stack/galaxy/project_galaxy_web_stack.py +++ b/packages/web_stack/galaxy/project_galaxy_web_stack.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-web-stack" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Web Strack Abstraction' -PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Web Strack Abstraction" +PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/packages/webapps/galaxy/project_galaxy_webapps.py b/packages/webapps/galaxy/project_galaxy_webapps.py index 660c0b88489..ddaf425c5a8 100644 --- a/packages/webapps/galaxy/project_galaxy_webapps.py +++ b/packages/webapps/galaxy/project_galaxy_webapps.py @@ -1,11 +1,9 @@ -__version__ = "22.1.0rc1" +__version__ = "22.5.0.dev0" PROJECT_NAME = "galaxy-webapps" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" -PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Web Apps' -PROJECT_EMAIL = 'jmchilton@gmail.com' -RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format( - PROJECT_USERAME, PROJECT_NAME -) +PROJECT_AUTHOR = "Galaxy Project and Community" +PROJECT_DESCRIPTION = "Galaxy Web Apps" +PROJECT_EMAIL = "jmchilton@gmail.com" +RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME) diff --git a/pyproject.toml b/pyproject.toml index f0cc84f1e01..fe6c0bd55d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,15 @@ +[tool.black] +line-length = 120 +target-version = ['py37'] +include = '\.pyi?$' +extend-exclude = ''' +^/( + | packages + | tools +)/ +''' +force-exclude = 'lib/galaxy/util/jstree.py' + [tool.poetry] name = "galaxy" version = "22.01.dev0" @@ -90,10 +102,12 @@ Whoosh = "*" zipstream-new = "*" [tool.poetry.dev-dependencies] +black = "^22.1.0" cwltest = "2.2.20210901154959" darker = "*" fluent-logger = "*" httpx = "*" +isort = "^5.10.1" lxml = "!=4.2.2" markdown-it-reporter = "*" NoseHTML = "*" diff --git a/setup.cfg b/setup.cfg index 41a9a481e9e..1ae7ccabadc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,16 +5,13 @@ universal = 1 # These are exceptions allowed by Galaxy style guidelines: # B008 Do not perform function calls in argument defaults (for FastAPI Depends and Body) # E128 continuation line under-indented for visual indent +# E203 is whitespace before ':'; we follow black's formatting here. See https://black.readthedocs.io/en/stable/faq.html#why-are-flake8-s-e203-and-w503-violated # E402 module level import not at top of file # TODO, we would like to improve this. # E501 is line length # W503 is line breaks before binary operators, which has been reversed in PEP 8. # D** are docstring linting - which we mostly ignore except D302. (Hopefully we will solve more over time). -ignore = B008,E128,E501,E402,W503,D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D204,D205,D206,D207,D208,D209,D210,D211,D300,D301,D400,D401,D402,D403,D412,D413 +ignore = B008,E128,E203,E402,E501,W503,D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D204,D205,D206,D207,D208,D209,D210,D211,D300,D301,D400,D401,D402,D403,D412,D413 exclude = lib/galaxy/util/jstree.py -# For flake8-import-order -# https://github.com/PyCQA/flake8-import-order/blob/master/tests/test_cases/complete_smarkets.py -import-order-style = smarkets -application-import-names = galaxy,galaxy_test,tool_shed [mypy] show_error_codes = True diff --git a/test/unit/app/jobs/test_job_wrapper.py b/test/unit/app/jobs/test_job_wrapper.py index 4566c8f8ba0..0ab3b6374a5 100644 --- a/test/unit/app/jobs/test_job_wrapper.py +++ b/test/unit/app/jobs/test_job_wrapper.py @@ -1,19 +1,22 @@ import abc import os from contextlib import contextmanager -from typing import Dict, Type +from typing import ( + Dict, + Type, +) from unittest import TestCase from galaxy.app_unittest_utils.tools_support import UsesApp from galaxy.jobs import ( JobWrapper, - TaskWrapper + TaskWrapper, ) from galaxy.model import ( Base, Job, Task, - User + User, ) from galaxy.util.bunch import Bunch @@ -24,7 +27,6 @@ TEST_COMMAND = "" class BaseWrapperTestCase(UsesApp): - def setUp(self): self.setup_app() job = Job() @@ -72,13 +74,11 @@ class BaseWrapperTestCase(UsesApp): class JobWrapperTestCase(BaseWrapperTestCase, TestCase): - def _wrapper(self): return JobWrapper(self.job, self.queue) # type: ignore[arg-type] class TaskWrapperTestCase(BaseWrapperTestCase, TestCase): - def setUp(self): super().setUp() self.task = Task(self.job, self.working_directory, "prepare_bwa_job.sh") @@ -90,7 +90,6 @@ class TaskWrapperTestCase(BaseWrapperTestCase, TestCase): class MockEvaluator: - def __init__(self, app, tool, job, local_working_directory): self.app = app self.tool = tool @@ -109,14 +108,12 @@ class MockEvaluator: class MockJobQueue: - def __init__(self, app): self.app = app self.dispatcher = MockJobDispatcher(app) class MockJobDispatcher: - def __init__(self, app): pass @@ -125,7 +122,6 @@ class MockJobDispatcher: class MockContext: - def __init__(self, model_objects): self.expunged_all = False self.flushed = False @@ -146,7 +142,6 @@ class MockContext: class MockQuery: - def __init__(self, class_objects): self.class_objects = class_objects @@ -158,26 +153,24 @@ class MockQuery: class MockTool: - def __init__(self, app): self.version_string_cmd = TEST_VERSION_COMMAND self.tool_dir = "/path/to/tools" self.dependencies = [] self.requires_galaxy_python_environment = False - self.id = 'mock_id' + self.id = "mock_id" self.home_target = None self.tmp_target = None - self.tool_source = Bunch(to_string=lambda: '') + self.tool_source = Bunch(to_string=lambda: "") def get_job_destination(self, params): - return Bunch(runner='local', id='local', params={}) + return Bunch(runner="local", id="local", params={}) def build_dependency_shell_commands(self, job_directory): return TEST_DEPENDENCIES_COMMANDS class MockToolbox: - def __init__(self, test_tool): self.test_tool = test_tool @@ -191,7 +184,6 @@ class MockToolbox: class MockObjectStore: - def __init__(self, working_directory): self.working_directory = working_directory os.makedirs(working_directory) diff --git a/test/unit/app/tools/test_evaluation.py b/test/unit/app/tools/test_evaluation.py index 6f2a709cc32..3e7db191c87 100644 --- a/test/unit/app/tools/test_evaluation.py +++ b/test/unit/app/tools/test_evaluation.py @@ -15,17 +15,18 @@ from galaxy.model import ( ) from galaxy.tool_util.parser.output_objects import ToolOutput from galaxy.tools.evaluation import ToolEvaluator + # For MockTool from galaxy.tools.parameters import params_from_strings from galaxy.tools.parameters.basic import ( DataToolParameter, IntegerToolParameter, - SelectToolParameter + SelectToolParameter, ) from galaxy.tools.parameters.grouping import ( Conditional, ConditionalWhen, - Repeat + Repeat, ) from galaxy.util import XML from galaxy.util.bunch import Bunch @@ -37,7 +38,6 @@ TEST_GALAXY_URL = "http://mycool.galaxyproject.org:8456" class ToolEvaluatorTestCase(TestCase, UsesApp): - def setUp(self): self.setup_app() self.tool = MockTool(self.app) @@ -54,14 +54,18 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self._setup_test_bwa_job() self._set_compute_environment() command_line = self.evaluator.build()[0] - self.assertEqual(command_line, "bwa --thresh=4 --in=/galaxy/files/dataset_1.dat --out=/galaxy/files/dataset_2.dat") + self.assertEqual( + command_line, "bwa --thresh=4 --in=/galaxy/files/dataset_1.dat --out=/galaxy/files/dataset_2.dat" + ) def test_repeat_evaluation(self): repeat = Repeat() repeat.name = "r" repeat.inputs = {"thresh": self.tool.test_thresh_param()} self.tool.set_params({"r": repeat}) - self.job.parameters = [JobParameter(name="r", value='''[{"thresh": 4, "__index__": 0},{"thresh": 5, "__index__": 1}]''')] + self.job.parameters = [ + JobParameter(name="r", value="""[{"thresh": 4, "__index__": 0},{"thresh": 5, "__index__": 1}]""") + ] self.tool._command_line = "prog1 #for $r_i in $r # $r_i.thresh#end for#" self._set_compute_environment() command_line = self.evaluator.build()[0] @@ -80,7 +84,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self.assertEqual(command_line, "prog1 '%s'" % self.app.security.encode_id(42)) def test_conditional_evaluation(self): - select_xml = XML('''''') + select_xml = XML("""""") parameter = SelectToolParameter(self.tool, select_xml) conditional = Conditional() @@ -91,7 +95,9 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): when.value = "true" conditional.cases = [when] self.tool.set_params({"c": conditional}) - self.job.parameters = [JobParameter(name="c", value='''{"thresh": 4, "always_true": "true", "__current_case__": 0}''')] + self.job.parameters = [ + JobParameter(name="c", value="""{"thresh": 4, "always_true": "true", "__current_case__": 0}""") + ] self.tool._command_line = "prog1 --thresh=${c.thresh} --test_param=${c.always_true}" self._set_compute_environment() command_line = self.evaluator.build()[0] @@ -100,9 +106,9 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): def test_evaluation_of_optional_datasets(self): # Make sure optional dataset don't cause evaluation to break and # evaluate in cheetah templates as 'None'. - select_xml = XML('''''') + select_xml = XML("""""") parameter = DataToolParameter(self.tool, select_xml) - self.job.parameters = [JobParameter(name="input1", value='null')] + self.job.parameters = [JobParameter(name="input1", value="null")] self.tool.set_params({"input1": parameter}) self.tool._command_line = "prog1 --opt_input='${input1}'" self._set_compute_environment() @@ -125,8 +131,8 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): job_path_1 = "%s/dataset_1.dat" % self.test_directory job_path_2 = "%s/dataset_2.dat" % self.test_directory self._set_compute_environment( - input_paths=[DatasetPath(1, '/galaxy/files/dataset_1.dat', false_path=job_path_1)], - output_paths=[DatasetPath(2, '/galaxy/files/dataset_2.dat', false_path=job_path_2)], + input_paths=[DatasetPath(1, "/galaxy/files/dataset_1.dat", false_path=job_path_1)], + output_paths=[DatasetPath(2, "/galaxy/files/dataset_2.dat", false_path=job_path_2)], ) command_line = self.evaluator.build()[0] self.assertEqual(command_line, f"bwa --thresh=4 --in={job_path_1} --out={job_path_2}") @@ -154,11 +160,13 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self.__test_arbitrary_path_rewriting() def __test_arbitrary_path_rewriting(self): - self.job.parameters = [JobParameter(name="index_path", value="\"/old/path/human\"")] - xml = XML(''' + self.job.parameters = [JobParameter(name="index_path", value='"/old/path/human"')] + xml = XML( + """ - ''') + """ + ) parameter = SelectToolParameter(self.tool, xml) def get_field_by_name_for_value(name, value, trans, other_values): @@ -170,9 +178,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): return [["", "/old/path/human", ""]] parameter.options = Bunch(get_field_by_name_for_value=get_field_by_name_for_value, get_options=get_options) - self.tool.set_params({ - "index_path": parameter - }) + self.tool.set_params({"index_path": parameter}) self.tool._command_line = "prog1 $index_path.fields.path" self._set_compute_environment(unstructured_path_rewrites={"/old": "/new"}) command_line = self.evaluator.build()[0] @@ -214,40 +220,32 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): assert "exec_before_job" in self.tool.hooks_called def _setup_test_bwa_job(self): - def hda(id, name, path): hda = HistoryDatasetAssociation(name=name, metadata=dict()) hda.dataset = Dataset(id=id, external_filename=path) return hda - id, name, path = 111, 'input1', '/galaxy/files/dataset_1.dat' + id, name, path = 111, "input1", "/galaxy/files/dataset_1.dat" self.job.input_datasets = [JobToInputDatasetAssociation(name=name, dataset=hda(id, name, path))] - id, name, path = 112, 'output1', '/galaxy/files/dataset_2.dat' + id, name, path = 112, "output1", "/galaxy/files/dataset_2.dat" self.job.output_datasets = [JobToOutputDatasetAssociation(name=name, dataset=hda(id, name, path))] class MockHistoryDatasetAssociation(HistoryDatasetAssociation): - def __init__(self, **kwds): self._metadata = dict() super().__init__(**kwds) class ComputeEnvironment(SimpleComputeEnvironment): - def __init__( - self, - new_file_path, - working_directory, - input_paths=None, - output_paths=None, - unstructured_path_rewrites=None + self, new_file_path, working_directory, input_paths=None, output_paths=None, unstructured_path_rewrites=None ): if input_paths is None: - input_paths = ['/galaxy/files/dataset_1.dat'] + input_paths = ["/galaxy/files/dataset_1.dat"] if output_paths is None: - output_paths = ['/galaxy/files/dataset_2.dat'] + output_paths = ["/galaxy/files/dataset_2.dat"] self._new_file_path = new_file_path self._working_directory = working_directory self._input_paths = input_paths @@ -300,10 +298,9 @@ class ComputeEnvironment(SimpleComputeEnvironment): class MockTool: - def __init__(self, app): self.profile = 16.01 - self.python_template_version = '2.7' + self.python_template_version = "2.7" self.app = app self.hooks_called = [] self.environment_variables = [] diff --git a/test/unit/data/model/mapping/test_model_mapping.py b/test/unit/data/model/mapping/test_model_mapping.py index bbe66d80d1c..09bb6837131 100644 --- a/test/unit/data/model/mapping/test_model_mapping.py +++ b/test/unit/data/model/mapping/test_model_mapping.py @@ -60,11 +60,20 @@ class TestPlanet(BaseTest): # BaseTest is a base class; we need it to get the t See other model tests in this module for examples of more complex setups. """ -from datetime import datetime, timedelta -from uuid import UUID, uuid4 +from datetime import ( + datetime, + timedelta, +) +from uuid import ( + UUID, + uuid4, +) import pytest -from sqlalchemy import func, select +from sqlalchemy import ( + func, + select, +) from galaxy import model from .common import ( @@ -82,7 +91,6 @@ from .common import ( class BaseTest(AbstractBaseTest): - def get_model(self): return model @@ -189,9 +197,7 @@ class TestCleanupEventImplicitlyConvertedDatasetAssociationAssociation(BaseTest) def test_table(self, cls_): assert cls_.__tablename__ == "cleanup_event_icda_association" - def test_columns( - self, session, cls_, cleanup_event, implicitly_converted_dataset_association - ): + def test_columns(self, session, cls_, cleanup_event, implicitly_converted_dataset_association): create_time = datetime.now() obj = cls_( create_time=create_time, @@ -251,9 +257,7 @@ class TestCleanupEventLibraryDatasetDatasetAssociationAssociation(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == "cleanup_event_ldda_association" - def test_columns( - self, session, cls_, cleanup_event, library_dataset_dataset_association - ): + def test_columns(self, session, cls_, cleanup_event, library_dataset_dataset_association): create_time = datetime.now() obj = cls_( create_time=create_time, @@ -311,9 +315,7 @@ class TestCleanupEventMetadataFileAssociation(BaseTest): class TestCustosAuthnzToken(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == "custos_authnz_token" - assert has_unique_constraint( - cls_.__table__, ("user_id", "external_user_id", "provider") - ) + assert has_unique_constraint(cls_.__table__, ("user_id", "external_user_id", "provider")) assert has_unique_constraint(cls_.__table__, ("external_user_id", "provider")) def test_columns(self, session, cls_, user): @@ -529,12 +531,8 @@ class TestDataset(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.job.id == job.id - assert collection_consists_of_objects( - stored_obj.actions, dataset_permission - ) - assert collection_consists_of_objects( - stored_obj.active_history_associations, history_dataset_association - ) + assert collection_consists_of_objects(stored_obj.actions, dataset_permission) + assert collection_consists_of_objects(stored_obj.active_history_associations, history_dataset_association) assert stored_obj.purged_history_associations[0].id == hda.id assert collection_consists_of_objects( stored_obj.active_library_associations, @@ -542,12 +540,8 @@ class TestDataset(BaseTest): ) assert collection_consists_of_objects(stored_obj.hashes, dataset_hash) assert collection_consists_of_objects(stored_obj.sources, dataset_source) - assert collection_consists_of_objects( - stored_obj.library_associations, library_dataset_dataset_association - ) - assert collection_consists_of_objects( - stored_obj.history_associations, hda, history_dataset_association - ) + assert collection_consists_of_objects(stored_obj.library_associations, library_dataset_dataset_association) + assert collection_consists_of_objects(stored_obj.history_associations, hda, history_dataset_association) delete_from_database(session, hda) @@ -594,9 +588,7 @@ class TestDatasetCollection(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert collection_consists_of_objects( - stored_obj.elements, dataset_collection_element - ) + assert collection_consists_of_objects(stored_obj.elements, dataset_collection_element) class TestDatasetCollectionElement(BaseTest): @@ -613,9 +605,7 @@ class TestDatasetCollectionElement(BaseTest): dataset_collection_factory, ): element_index, element_identifier = 1, "a" - obj = cls_( - element=history_dataset_association - ) # using hda is sufficient for this test + obj = cls_(element=history_dataset_association) # using hda is sufficient for this test obj.element_index = element_index obj.element_identifier = element_identifier obj.hda = history_dataset_association @@ -647,17 +637,13 @@ class TestDatasetCollectionElement(BaseTest): library_dataset_dataset_association, dataset_collection_factory, ): - obj = cls_( - element=history_dataset_association - ) # using hda is sufficient for this test + obj = cls_(element=history_dataset_association) # using hda is sufficient for this test obj.hda = history_dataset_association obj.ldda = library_dataset_dataset_association obj.child_collection = dataset_collection parent_collection = dataset_collection_factory() - obj.collection = ( - parent_collection # same as parent_collection.elements.append(obj) - ) + obj.collection = parent_collection # same as parent_collection.elements.append(obj) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -755,9 +741,7 @@ class TestDatasetSource(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.dataset.id == dataset.id - assert collection_consists_of_objects( - stored_obj.hashes, dataset_source_hash - ) + assert collection_consists_of_objects(stored_obj.hashes, dataset_source_hash) class TestDatasetSourceHash(BaseTest): @@ -984,9 +968,7 @@ class TestExtendedMetadata(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert collection_consists_of_objects( - stored_obj.children, extended_metadata_index - ) + assert collection_consists_of_objects(stored_obj.children, extended_metadata_index) class TestExtendedMetadataIndex(BaseTest): @@ -1128,9 +1110,7 @@ class TestGalaxySession(BaseTest): disk_usage = 9 last_action = update_time + timedelta(hours=1) - obj = cls_( - user=user, current_history=history, prev_session_id=galaxy_session.id - ) + obj = cls_(user=user, current_history=history, prev_session_id=galaxy_session.id) obj.create_time = create_time obj.update_time = update_time @@ -1158,9 +1138,7 @@ class TestGalaxySession(BaseTest): assert stored_obj.disk_usage == disk_usage assert stored_obj.last_action == last_action - def test_relationships( - self, session, cls_, user, history, galaxy_session_history_association - ): + def test_relationships(self, session, cls_, user, history, galaxy_session_history_association): obj = cls_(user=user, current_history=history) obj.session_key = get_unique_value() obj.histories.append(galaxy_session_history_association) @@ -1169,9 +1147,7 @@ class TestGalaxySession(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.user.id == user.id assert stored_obj.current_history.id == history.id - assert collection_consists_of_objects( - stored_obj.histories, galaxy_session_history_association - ) + assert collection_consists_of_objects(stored_obj.histories, galaxy_session_history_association) class TestGalaxySessionToHistoryAssociation(BaseTest): @@ -1281,15 +1257,9 @@ class TestGroup(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert collection_consists_of_objects( - stored_obj.quotas, group_quota_association - ) - assert collection_consists_of_objects( - stored_obj.roles, group_role_association - ) - assert collection_consists_of_objects( - stored_obj.users, user_group_association - ) + assert collection_consists_of_objects(stored_obj.quotas, group_quota_association) + assert collection_consists_of_objects(stored_obj.roles, group_role_association) + assert collection_consists_of_objects(stored_obj.users, user_group_association) class TestGroupQuotaAssociation(BaseTest): @@ -1426,58 +1396,32 @@ class TestHistory(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.user.id == user.id - assert collection_consists_of_objects( - stored_obj.datasets, history_dataset_association - ) - assert collection_consists_of_objects( - stored_obj.exports, job_export_history_archive - ) - assert collection_consists_of_objects( - stored_obj.active_datasets, history_dataset_association - ) + assert collection_consists_of_objects(stored_obj.datasets, history_dataset_association) + assert collection_consists_of_objects(stored_obj.exports, job_export_history_archive) + assert collection_consists_of_objects(stored_obj.active_datasets, history_dataset_association) assert collection_consists_of_objects( stored_obj.active_dataset_collections, history_dataset_collection_association, ) - assert collection_consists_of_objects( - stored_obj.visible_datasets, history_dataset_association - ) + assert collection_consists_of_objects(stored_obj.visible_datasets, history_dataset_association) assert collection_consists_of_objects( stored_obj.visible_dataset_collections, history_dataset_collection_association, ) - assert collection_consists_of_objects( - stored_obj.tags, history_tag_association - ) - assert collection_consists_of_objects( - stored_obj.annotations, history_annotation_association - ) - assert collection_consists_of_objects( - stored_obj.ratings, history_rating_association - ) + assert collection_consists_of_objects(stored_obj.tags, history_tag_association) + assert collection_consists_of_objects(stored_obj.annotations, history_annotation_association) + assert collection_consists_of_objects(stored_obj.ratings, history_rating_association) # This doesn't test the average amount, just the mapping. assert stored_obj.average_rating == history_rating_association.rating assert stored_obj.users_shared_with_count == 1 - assert collection_consists_of_objects( - stored_obj.users_shared_with, history_user_share_association - ) - assert collection_consists_of_objects( - stored_obj.default_permissions, default_history_permissions - ) - assert collection_consists_of_objects( - stored_obj.galaxy_sessions, galaxy_session_history_association - ) - assert collection_consists_of_objects( - stored_obj.workflow_invocations, workflow_invocation - ) + assert collection_consists_of_objects(stored_obj.users_shared_with, history_user_share_association) + assert collection_consists_of_objects(stored_obj.default_permissions, default_history_permissions) + assert collection_consists_of_objects(stored_obj.galaxy_sessions, galaxy_session_history_association) + assert collection_consists_of_objects(stored_obj.workflow_invocations, workflow_invocation) assert collection_consists_of_objects(stored_obj.jobs, job) - def test_average_rating( - self, session, history, user, history_rating_association_factory - ): - _run_average_rating_test( - session, history, user, history_rating_association_factory - ) + def test_average_rating(self, session, history, user, history_rating_association_factory): + _run_average_rating_test(session, history, user, history_rating_association_factory) class TestHistoryAnnotationAssociation(BaseTest): @@ -1555,9 +1499,7 @@ class TestHistoryDatasetAssociation(BaseTest): obj.update_time = update_time obj.state = state obj.copied_from_history_dataset_association = copied_from_hda - obj.copied_from_library_dataset_dataset_association = ( - library_dataset_dataset_association - ) + obj.copied_from_library_dataset_dataset_association = library_dataset_dataset_association obj.name = name obj.info = info obj.blurb = blurb @@ -1585,13 +1527,9 @@ class TestHistoryDatasetAssociation(BaseTest): assert stored_obj.create_time == create_time assert stored_obj.update_time == update_time assert stored_obj.state == state + assert stored_obj.copied_from_history_dataset_association_id == copied_from_hda.id assert ( - stored_obj.copied_from_history_dataset_association_id - == copied_from_hda.id - ) - assert ( - stored_obj.copied_from_library_dataset_dataset_association_id - == library_dataset_dataset_association.id + stored_obj.copied_from_library_dataset_dataset_association_id == library_dataset_dataset_association.id ) assert stored_obj.name == name assert stored_obj.info == info @@ -1610,10 +1548,7 @@ class TestHistoryDatasetAssociation(BaseTest): assert stored_obj.purged == purged assert stored_obj.validated_state == validated_state assert stored_obj.validated_state_message == validated_state_message - assert ( - stored_obj.hidden_beneath_collection_instance_id - == history_dataset_collection_association.id - ) + assert stored_obj.hidden_beneath_collection_instance_id == history_dataset_collection_association.id delete_from_database(session, [copied_from_hda, parent]) @@ -1672,57 +1607,34 @@ class TestHistoryDatasetAssociation(BaseTest): assert stored_obj.id == obj_id assert stored_obj.history.id == history.id assert stored_obj.dataset.id == dataset.id - assert ( - stored_obj.copied_from_history_dataset_association.id - == copied_from_hda.id - ) - assert ( - stored_obj.copied_from_library_dataset_dataset_association.id - == copied_from_ldda.id - ) + assert stored_obj.copied_from_history_dataset_association.id == copied_from_hda.id + assert stored_obj.copied_from_library_dataset_dataset_association.id == copied_from_ldda.id assert stored_obj.extended_metadata.id == extended_metadata.id - assert ( - stored_obj.hidden_beneath_collection_instance.id - == history_dataset_collection_association.id - ) + assert stored_obj.hidden_beneath_collection_instance.id == history_dataset_collection_association.id assert collection_consists_of_objects( stored_obj.copied_to_library_dataset_dataset_associations, copied_to_ldda, ) - assert collection_consists_of_objects( - stored_obj.copied_to_history_dataset_associations, copied_to_hda - ) - assert collection_consists_of_objects( - stored_obj.tags, history_dataset_association_tag_association - ) + assert collection_consists_of_objects(stored_obj.copied_to_history_dataset_associations, copied_to_hda) + assert collection_consists_of_objects(stored_obj.tags, history_dataset_association_tag_association) assert collection_consists_of_objects( stored_obj.annotations, history_dataset_association_annotation_association, ) - assert collection_consists_of_objects( - stored_obj.ratings, history_dataset_association_rating_association - ) - assert collection_consists_of_objects( - stored_obj.dependent_jobs, job_to_input_dataset_association - ) + assert collection_consists_of_objects(stored_obj.ratings, history_dataset_association_rating_association) + assert collection_consists_of_objects(stored_obj.dependent_jobs, job_to_input_dataset_association) assert collection_consists_of_objects( stored_obj.creating_job_associations, job_to_output_dataset_association ) - assert collection_consists_of_objects( - stored_obj.implicitly_converted_datasets, icda - ) - assert collection_consists_of_objects( - stored_obj.implicitly_converted_parent_datasets, icpda - ) + assert collection_consists_of_objects(stored_obj.implicitly_converted_datasets, icda) + assert collection_consists_of_objects(stored_obj.implicitly_converted_parent_datasets, icpda) delete_from_database(session, persisted) class TestHistoryDatasetAssociationAnnotationAssociation(BaseTest): def test_table(self, cls_): - assert ( - cls_.__tablename__ == "history_dataset_association_annotation_association" - ) + assert cls_.__tablename__ == "history_dataset_association_annotation_association" assert has_index(cls_.__table__, ("annotation",)) def test_columns(self, session, cls_, history_dataset_association, user): @@ -1735,10 +1647,7 @@ class TestHistoryDatasetAssociationAnnotationAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert ( - stored_obj.history_dataset_association_id - == history_dataset_association.id - ) + assert stored_obj.history_dataset_association_id == history_dataset_association.id assert stored_obj.user_id == user.id assert stored_obj.annotation == annotation @@ -1755,9 +1664,7 @@ class TestHistoryDatasetAssociationAnnotationAssociation(BaseTest): class TestHistoryDatasetAssociationDisplayAtAuthorization(BaseTest): def test_table(self, cls_): - assert ( - cls_.__tablename__ == "history_dataset_association_display_at_authorization" - ) + assert cls_.__tablename__ == "history_dataset_association_display_at_authorization" def test_columns(self, session, cls_, history_dataset_association, user): create_time = datetime.now() @@ -1770,10 +1677,7 @@ class TestHistoryDatasetAssociationDisplayAtAuthorization(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert ( - stored_obj.history_dataset_association_id - == history_dataset_association.id - ) + assert stored_obj.history_dataset_association_id == history_dataset_association.id assert stored_obj.user_id == user.id assert stored_obj.site == site @@ -1782,10 +1686,7 @@ class TestHistoryDatasetAssociationDisplayAtAuthorization(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.history_dataset_association.id - == history_dataset_association.id - ) + assert stored_obj.history_dataset_association.id == history_dataset_association.id assert stored_obj.user.id == user.id @@ -1793,9 +1694,7 @@ class TestHistoryDatasetAssociationHistory(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == "history_dataset_association_history" - def test_columns( - self, session, cls_, history_dataset_association, extended_metadata - ): + def test_columns(self, session, cls_, history_dataset_association, extended_metadata): name, update_time, version, extension, metadata = ( "a", datetime.now(), @@ -1817,10 +1716,7 @@ class TestHistoryDatasetAssociationHistory(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert ( - stored_obj.history_dataset_association_id - == history_dataset_association.id - ) + assert stored_obj.history_dataset_association_id == history_dataset_association.id assert stored_obj.name == name assert stored_obj.update_time == update_time assert stored_obj.version == version @@ -1848,10 +1744,7 @@ class TestHistoryDatasetAssociationRatingAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.history_dataset_association.id - == history_dataset_association.id - ) + assert stored_obj.history_dataset_association.id == history_dataset_association.id assert stored_obj.user.id == user.id @@ -1875,10 +1768,7 @@ class TestHistoryDatasetAssociationSubset(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert ( - stored_obj.history_dataset_association_id - == history_dataset_association.id - ) + assert stored_obj.history_dataset_association_id == history_dataset_association.id assert stored_obj.history_dataset_association_subset_id == hda_subset.id assert stored_obj.location == location @@ -1922,10 +1812,7 @@ class TestHistoryDatasetAssociationTagAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert ( - stored_obj.history_dataset_association_id - == history_dataset_association.id - ) + assert stored_obj.history_dataset_association_id == history_dataset_association.id assert stored_obj.tag_id == tag.id assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname @@ -1938,10 +1825,7 @@ class TestHistoryDatasetAssociationTagAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.history_dataset_association.id - == history_dataset_association.id - ) + assert stored_obj.history_dataset_association.id == history_dataset_association.id assert stored_obj.tag.id == tag.id assert stored_obj.user.id == user.id @@ -1975,9 +1859,7 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): obj.hid = hid obj.visible = visible obj.deleted = deleted - obj.copied_from_history_dataset_collection_association = ( - history_dataset_collection_association - ) + obj.copied_from_history_dataset_collection_association = history_dataset_collection_association obj.implicit_output_name = implicit_output_name obj.job = job obj.implicit_collection_jobs = implicit_collection_jobs @@ -2024,15 +1906,11 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): obj = cls_() obj.collection = dataset_collection obj.history = history - obj.copied_from_history_dataset_collection_association = ( - history_dataset_collection_association - ) + obj.copied_from_history_dataset_collection_association = history_dataset_collection_association obj.copied_to_history_dataset_collection_association.append(copied_to_hdca) obj.job = job obj.implicit_collection_jobs = implicit_collection_jobs - obj.implicit_input_collections.append( - implicitly_created_dataset_collection_input - ) + obj.implicit_input_collections.append(implicitly_created_dataset_collection_input) obj.tags.append(history_dataset_collection_tag_association) obj.annotations.append(history_dataset_collection_annotation_association) obj.ratings.append(history_dataset_collection_rating_association) @@ -2055,16 +1933,12 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): stored_obj.implicit_input_collections, implicitly_created_dataset_collection_input, ) - assert collection_consists_of_objects( - stored_obj.tags, history_dataset_collection_tag_association - ) + assert collection_consists_of_objects(stored_obj.tags, history_dataset_collection_tag_association) assert collection_consists_of_objects( stored_obj.annotations, history_dataset_collection_annotation_association, ) - assert collection_consists_of_objects( - stored_obj.ratings, history_dataset_collection_rating_association - ) + assert collection_consists_of_objects(stored_obj.ratings, history_dataset_collection_rating_association) # stored_obj.job_state_summary is a view: can't test in this setup. delete_from_database(session, copied_to_hdca) @@ -2084,26 +1958,18 @@ class TestHistoryDatasetCollectionAssociationAnnotationAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert ( - stored_obj.history_dataset_collection_id - == history_dataset_collection_association.id - ) + assert stored_obj.history_dataset_collection_id == history_dataset_collection_association.id assert stored_obj.user_id == user.id assert stored_obj.annotation == annotation - def test_relationships( - self, session, cls_, history_dataset_collection_association, user - ): + def test_relationships(self, session, cls_, history_dataset_collection_association, user): obj = cls_() obj.user = user obj.history_dataset_collection = history_dataset_collection_association with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.history_dataset_collection.id - == history_dataset_collection_association.id - ) + assert stored_obj.history_dataset_collection.id == history_dataset_collection_association.id assert stored_obj.user.id == user.id @@ -2118,24 +1984,16 @@ class TestHistoryDatasetCollectionRatingAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert ( - stored_obj.history_dataset_collection_id - == history_dataset_collection_association.id - ) + assert stored_obj.history_dataset_collection_id == history_dataset_collection_association.id assert stored_obj.user.id == user.id assert stored_obj.rating == rating - def test_relationships( - self, session, cls_, history_dataset_collection_association, user - ): + def test_relationships(self, session, cls_, history_dataset_collection_association, user): obj = cls_(user, history_dataset_collection_association, 1) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.dataset_collection.id - == history_dataset_collection_association.id - ) + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id assert stored_obj.user.id == user.id @@ -2143,9 +2001,7 @@ class TestHistoryDatasetCollectionTagAssociation(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == "history_dataset_collection_tag_association" - def test_columns( - self, session, cls_, history_dataset_collection_association, tag, user - ): + def test_columns(self, session, cls_, history_dataset_collection_association, tag, user): user_tname, value, user_value = "a", "b", "c" obj = cls_( user=user, @@ -2159,28 +2015,20 @@ class TestHistoryDatasetCollectionTagAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert ( - stored_obj.history_dataset_collection_id - == history_dataset_collection_association.id - ) + assert stored_obj.history_dataset_collection_id == history_dataset_collection_association.id assert stored_obj.tag_id == tag.id assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname assert stored_obj.value == value assert stored_obj.user_value == user_value - def test_relationships( - self, session, cls_, history_dataset_collection_association, tag, user - ): + def test_relationships(self, session, cls_, history_dataset_collection_association, tag, user): obj = cls_(user=user, tag=tag) obj.dataset_collection = history_dataset_collection_association with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.dataset_collection.id - == history_dataset_collection_association.id - ) + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id assert stored_obj.tag.id == tag.id assert stored_obj.user.id == user.id @@ -2291,9 +2139,7 @@ class TestImplicitCollectionJobs(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert collection_consists_of_objects( - stored_obj.jobs, implicit_collection_jobs_job_association - ) + assert collection_consists_of_objects(stored_obj.jobs, implicit_collection_jobs_job_association) class TestImplicitCollectionJobsJobAssociation(BaseTest): @@ -2412,10 +2258,7 @@ class TestImplicitlyCreatedDatasetCollectionInput(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert ( - stored_obj.input_dataset_collection_id - == history_dataset_collection_association.id - ) + assert stored_obj.input_dataset_collection_id == history_dataset_collection_association.id assert stored_obj.dataset_collection_id == hdca2.id assert stored_obj.name == name @@ -2431,10 +2274,7 @@ class TestImplicitlyCreatedDatasetCollectionInput(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.input_dataset_collection.id - == history_dataset_collection_association.id - ) + assert stored_obj.input_dataset_collection.id == history_dataset_collection_association.id class TestInteractiveToolEntryPoint(BaseTest): @@ -2649,18 +2489,10 @@ class TestJob(BaseTest): obj.user = user obj.parameters.append(job_parameter) obj.input_datasets.append(job_to_input_dataset_association) - obj.input_dataset_collections.append( - job_to_input_dataset_collection_association - ) - obj.input_dataset_collection_elements.append( - job_to_input_dataset_collection_element_association - ) - obj.output_dataset_collection_instances.append( - job_to_output_dataset_collection_association - ) - obj.output_dataset_collections.append( - job_to_implicit_output_dataset_collection_association - ) + obj.input_dataset_collections.append(job_to_input_dataset_collection_association) + obj.input_dataset_collection_elements.append(job_to_input_dataset_collection_element_association) + obj.output_dataset_collection_instances.append(job_to_output_dataset_collection_association) + obj.output_dataset_collections.append(job_to_implicit_output_dataset_collection_association) obj.post_job_actions.append(post_job_action_association) obj.input_library_datasets.append(job_to_input_library_dataset_association) obj.output_library_datasets.append(job_to_output_library_dataset_association) @@ -2671,14 +2503,10 @@ class TestJob(BaseTest): obj.text_metrics.append(job_metric_text) obj.numeric_metrics.append(job_metric_numeric) obj.interactivetool_entry_points.append(interactive_tool_entry_point) - obj.implicit_collection_jobs_association = ( - implicit_collection_jobs_job_association - ) + obj.implicit_collection_jobs_association = implicit_collection_jobs_job_association obj.container = job_container_association obj.data_manager_association = data_manager_job_association - obj.history_dataset_collection_associations.append( - history_dataset_collection_association - ) + obj.history_dataset_collection_associations.append(history_dataset_collection_association) obj.workflow_invocation_step = workflow_invocation_step with dbcleanup(session, obj) as obj_id: @@ -2688,9 +2516,7 @@ class TestJob(BaseTest): assert stored_obj.session_id == galaxy_session.id assert stored_obj.user_id == user.id assert collection_consists_of_objects(stored_obj.parameters, job_parameter) - assert collection_consists_of_objects( - stored_obj.input_datasets, job_to_input_dataset_association - ) + assert collection_consists_of_objects(stored_obj.input_datasets, job_to_input_dataset_association) assert collection_consists_of_objects( stored_obj.input_dataset_collections, job_to_input_dataset_collection_association, @@ -2707,9 +2533,7 @@ class TestJob(BaseTest): stored_obj.output_dataset_collections, job_to_implicit_output_dataset_collection_association, ) - assert collection_consists_of_objects( - stored_obj.post_job_actions, post_job_action_association - ) + assert collection_consists_of_objects(stored_obj.post_job_actions, post_job_action_association) assert collection_consists_of_objects( stored_obj.input_library_datasets, job_to_input_library_dataset_association, @@ -2718,27 +2542,14 @@ class TestJob(BaseTest): stored_obj.output_library_datasets, job_to_output_library_dataset_association, ) - assert collection_consists_of_objects( - stored_obj.external_output_metadata, job_external_output_metadata - ) + assert collection_consists_of_objects(stored_obj.external_output_metadata, job_external_output_metadata) assert collection_consists_of_objects(stored_obj.tasks, task) - assert collection_consists_of_objects( - stored_obj.output_datasets, job_to_output_dataset_association - ) + assert collection_consists_of_objects(stored_obj.output_datasets, job_to_output_dataset_association) assert job_state_history in stored_obj.state_history # sufficient for test - assert collection_consists_of_objects( - stored_obj.text_metrics, job_metric_text - ) - assert collection_consists_of_objects( - stored_obj.numeric_metrics, job_metric_numeric - ) - assert collection_consists_of_objects( - stored_obj.interactivetool_entry_points, interactive_tool_entry_point - ) - assert ( - stored_obj.implicit_collection_jobs_association - == implicit_collection_jobs_job_association - ) + assert collection_consists_of_objects(stored_obj.text_metrics, job_metric_text) + assert collection_consists_of_objects(stored_obj.numeric_metrics, job_metric_numeric) + assert collection_consists_of_objects(stored_obj.interactivetool_entry_points, interactive_tool_entry_point) + assert stored_obj.implicit_collection_jobs_association == implicit_collection_jobs_job_association assert stored_obj.container == job_container_association assert stored_obj.data_manager_association == data_manager_job_association assert collection_consists_of_objects( @@ -2858,10 +2669,7 @@ class TestJobExternalOutputMetadata(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id assert stored_obj.job_id == job.id - assert ( - stored_obj.history_dataset_association_id - == history_dataset_association.id - ) + assert stored_obj.history_dataset_association_id == history_dataset_association.id assert stored_obj.is_valid == is_valid assert stored_obj.filename_in == filename_in assert stored_obj.filename_out == filename_out @@ -2878,10 +2686,7 @@ class TestJobExternalOutputMetadata(BaseTest): obj = cls_(job, library_dataset_dataset_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.library_dataset_dataset_association_id - == library_dataset_dataset_association.id - ) + assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id def test_relationships( self, @@ -3071,25 +2876,17 @@ class TestJobToInputDatasetCollectionAssociation(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id assert stored_obj.job_id == job.id - assert ( - stored_obj.dataset_collection_id - == history_dataset_collection_association.id - ) + assert stored_obj.dataset_collection_id == history_dataset_collection_association.id assert stored_obj.name == name - def test_relationships( - self, session, cls_, history_dataset_collection_association, job - ): + def test_relationships(self, session, cls_, history_dataset_collection_association, job): obj = cls_(None, history_dataset_collection_association) obj.job = job with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.job.id == job.id - assert ( - stored_obj.dataset_collection.id - == history_dataset_collection_association.id - ) + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id class TestJobToInputDatasetCollectionElementAssociation(BaseTest): @@ -3105,10 +2902,7 @@ class TestJobToInputDatasetCollectionElementAssociation(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id assert stored_obj.job_id == job.id - assert ( - stored_obj.dataset_collection_element_id - == dataset_collection_element.id - ) + assert stored_obj.dataset_collection_element_id == dataset_collection_element.id assert stored_obj.name == name def test_relationships(self, session, cls_, dataset_collection_element, job): @@ -3118,10 +2912,7 @@ class TestJobToInputDatasetCollectionElementAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.job.id == job.id - assert ( - stored_obj.dataset_collection_element.id - == dataset_collection_element.id - ) + assert stored_obj.dataset_collection_element.id == dataset_collection_element.id class TestJobToInputLibraryDatasetAssociation(BaseTest): @@ -3140,9 +2931,7 @@ class TestJobToInputLibraryDatasetAssociation(BaseTest): assert stored_obj.ldda_id == library_dataset_dataset_association.id assert stored_obj.name == name - def test_relationships( - self, session, cls_, library_dataset_dataset_association, job - ): + def test_relationships(self, session, cls_, library_dataset_dataset_association, job): obj = cls_(None, library_dataset_dataset_association) obj.job = job @@ -3191,25 +2980,17 @@ class TestJobToOutputDatasetCollectionAssociation(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id assert stored_obj.job_id == job.id - assert ( - stored_obj.dataset_collection_id - == history_dataset_collection_association.id - ) + assert stored_obj.dataset_collection_id == history_dataset_collection_association.id assert stored_obj.name == name - def test_relationships( - self, session, cls_, history_dataset_collection_association, job - ): + def test_relationships(self, session, cls_, history_dataset_collection_association, job): obj = cls_(None, history_dataset_collection_association) obj.job = job with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.job.id == job.id - assert ( - stored_obj.dataset_collection_instance.id - == history_dataset_collection_association.id - ) + assert stored_obj.dataset_collection_instance.id == history_dataset_collection_association.id class TestJobToOutputLibraryDatasetAssociation(BaseTest): @@ -3228,9 +3009,7 @@ class TestJobToOutputLibraryDatasetAssociation(BaseTest): assert stored_obj.ldda_id == library_dataset_dataset_association.id assert stored_obj.name == name - def test_relationships( - self, session, cls_, library_dataset_dataset_association, job - ): + def test_relationships(self, session, cls_, library_dataset_dataset_association, job): obj = cls_(None, library_dataset_dataset_association) obj.job = job @@ -3279,9 +3058,7 @@ class TestLibrary(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.root_folder.id == library_folder.id - assert collection_consists_of_objects( - stored_obj.actions, library_permission - ) + assert collection_consists_of_objects(stored_obj.actions, library_permission) class TestLibraryDataset(BaseTest): @@ -3340,15 +3117,10 @@ class TestLibraryDataset(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.library_dataset_dataset_association.id - == library_dataset_dataset_association.id - ) + assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id assert stored_obj.folder.id == library_folder.id assert stored_obj.expired_datasets[0].id == ldda.id - assert collection_consists_of_objects( - stored_obj.actions, library_dataset_permission - ) + assert collection_consists_of_objects(stored_obj.actions, library_dataset_permission) delete_from_database(session, ldda) @@ -3367,26 +3139,18 @@ class TestLibraryDatasetCollectionAnnotationAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert ( - stored_obj.library_dataset_collection_id - == library_dataset_collection_association.id - ) + assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id assert stored_obj.user_id == user.id assert stored_obj.annotation == annotation - def test_relationships( - self, session, cls_, library_dataset_collection_association, user - ): + def test_relationships(self, session, cls_, library_dataset_collection_association, user): obj = cls_() obj.user = user obj.dataset_collection = library_dataset_collection_association with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.dataset_collection.id - == library_dataset_collection_association.id - ) + assert stored_obj.dataset_collection.id == library_dataset_collection_association.id assert stored_obj.user.id == user.id @@ -3432,16 +3196,12 @@ class TestLibraryDatasetCollectionAssociation(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.collection.id == dataset_collection.id assert stored_obj.folder.id == library_folder.id - assert collection_consists_of_objects( - stored_obj.tags, library_dataset_collection_tag_association - ) + assert collection_consists_of_objects(stored_obj.tags, library_dataset_collection_tag_association) assert collection_consists_of_objects( stored_obj.annotations, library_dataset_collection_annotation_association, ) - assert collection_consists_of_objects( - stored_obj.ratings, library_dataset_collection_rating_association - ) + assert collection_consists_of_objects(stored_obj.ratings, library_dataset_collection_rating_association) class TestLibraryDatasetCollectionRatingAssociation(BaseTest): @@ -3455,24 +3215,16 @@ class TestLibraryDatasetCollectionRatingAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert ( - stored_obj.library_dataset_collection_id - == library_dataset_collection_association.id - ) + assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id assert stored_obj.user_id == user.id assert stored_obj.rating == rating - def test_relationships( - self, session, cls_, library_dataset_collection_association, user - ): + def test_relationships(self, session, cls_, library_dataset_collection_association, user): obj = cls_(user, library_dataset_collection_association, 1) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.dataset_collection.id - == library_dataset_collection_association.id - ) + assert stored_obj.dataset_collection.id == library_dataset_collection_association.id assert stored_obj.user.id == user.id @@ -3480,9 +3232,7 @@ class TestLibraryDatasetCollectionTagAssociation(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == "library_dataset_collection_tag_association" - def test_columns( - self, session, cls_, library_dataset_collection_association, tag, user - ): + def test_columns(self, session, cls_, library_dataset_collection_association, tag, user): user_tname, value, user_value = "a", "b", "c" obj = cls_(user=user, tag=tag, user_tname=user_tname, value=value) obj.user_value = user_value @@ -3491,28 +3241,20 @@ class TestLibraryDatasetCollectionTagAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert ( - stored_obj.library_dataset_collection_id - == library_dataset_collection_association.id - ) + assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id assert stored_obj.tag_id == tag.id assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname assert stored_obj.value == value assert stored_obj.user_value == user_value - def test_relationships( - self, session, cls_, library_dataset_collection_association, tag, user - ): + def test_relationships(self, session, cls_, library_dataset_collection_association, tag, user): obj = cls_(user=user, tag=tag) obj.dataset_collection = library_dataset_collection_association with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.dataset_collection.id - == library_dataset_collection_association.id - ) + assert stored_obj.dataset_collection.id == library_dataset_collection_association.id assert stored_obj.tag.id == tag.id assert stored_obj.user.id == user.id @@ -3583,17 +3325,9 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): assert stored_obj.library_dataset_id == library_dataset.id assert stored_obj.dataset_id == dataset.id assert stored_obj.create_time == create_time - assert ( - stored_obj.update_time >= create_time - ) # this is sufficient for testing - assert ( - stored_obj.copied_from_history_dataset_association_id - == history_dataset_association.id - ) - assert ( - stored_obj.copied_from_library_dataset_dataset_association_id - == copied_from_ldda.id - ) + assert stored_obj.update_time >= create_time # this is sufficient for testing + assert stored_obj.copied_from_history_dataset_association_id == history_dataset_association.id + assert stored_obj.copied_from_library_dataset_dataset_association_id == copied_from_ldda.id assert stored_obj.state == state assert stored_obj.name == name assert stored_obj.info == info @@ -3669,38 +3403,20 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.library_dataset.id == library_dataset.id assert stored_obj.dataset.id == dataset.id - assert ( - stored_obj.copied_from_history_dataset_association.id - == history_dataset_association.id - ) - assert ( - stored_obj.copied_from_library_dataset_dataset_association.id - == copied_from_ldda.id - ) + assert stored_obj.copied_from_history_dataset_association.id == history_dataset_association.id + assert stored_obj.copied_from_library_dataset_dataset_association.id == copied_from_ldda.id assert stored_obj.extended_metadata.id == extended_metadata.id assert stored_obj.user.id == user.id - assert collection_consists_of_objects( - stored_obj.tags, library_dataset_dataset_association_tag_association - ) - assert collection_consists_of_objects( - stored_obj.actions, library_dataset_dataset_association_permission - ) - assert collection_consists_of_objects( - stored_obj.copied_to_history_dataset_associations, copied_to_hda - ) + assert collection_consists_of_objects(stored_obj.tags, library_dataset_dataset_association_tag_association) + assert collection_consists_of_objects(stored_obj.actions, library_dataset_dataset_association_permission) + assert collection_consists_of_objects(stored_obj.copied_to_history_dataset_associations, copied_to_hda) assert collection_consists_of_objects( stored_obj.copied_to_library_dataset_dataset_associations, copied_to_ldda, ) - assert collection_consists_of_objects( - stored_obj.implicitly_converted_datasets, icda - ) - assert collection_consists_of_objects( - stored_obj.implicitly_converted_parent_datasets, icpda - ) - assert collection_consists_of_objects( - stored_obj.dependent_jobs, job_to_input_library_dataset_association - ) + assert collection_consists_of_objects(stored_obj.implicitly_converted_datasets, icda) + assert collection_consists_of_objects(stored_obj.implicitly_converted_parent_datasets, icpda) + assert collection_consists_of_objects(stored_obj.dependent_jobs, job_to_input_library_dataset_association) assert collection_consists_of_objects( stored_obj.creating_job_associations, job_to_output_library_dataset_association, @@ -3727,35 +3443,23 @@ class TestLibraryDatasetDatasetAssociationPermissions(BaseTest): assert stored_obj.create_time == create_time assert stored_obj.update_time == update_time assert stored_obj.action == action - assert ( - stored_obj.library_dataset_dataset_association_id - == library_dataset_dataset_association.id - ) + assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id assert stored_obj.role_id == role.id - def test_relationships( - self, session, cls_, library_dataset_dataset_association, role - ): + def test_relationships(self, session, cls_, library_dataset_dataset_association, role): obj = cls_(None, library_dataset_dataset_association, role) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.library_dataset_dataset_association.id - == library_dataset_dataset_association.id - ) + assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id assert stored_obj.role.id == role.id class TestLibraryDatasetDatasetAssociationTagAssociation(BaseTest): def test_table(self, cls_): - assert ( - cls_.__tablename__ == "library_dataset_dataset_association_tag_association" - ) + assert cls_.__tablename__ == "library_dataset_dataset_association_tag_association" - def test_columns( - self, session, cls_, library_dataset_dataset_association, tag, user - ): + def test_columns(self, session, cls_, library_dataset_dataset_association, tag, user): user_tname, value, user_value = "a", "b", "c" obj = cls_(user=user, tag=tag, user_tname=user_tname, value=value) obj.user_value = user_value @@ -3764,28 +3468,20 @@ class TestLibraryDatasetDatasetAssociationTagAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert ( - stored_obj.library_dataset_dataset_association_id - == library_dataset_dataset_association.id - ) + assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id assert stored_obj.tag_id == tag.id assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname assert stored_obj.value == value assert stored_obj.user_value == user_value - def test_relationships( - self, session, cls_, library_dataset_dataset_association, tag, user - ): + def test_relationships(self, session, cls_, library_dataset_dataset_association, tag, user): obj = cls_(user=user, tag=tag) obj.library_dataset_dataset_association = library_dataset_dataset_association with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.library_dataset_dataset_association.id - == library_dataset_dataset_association.id - ) + assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id assert stored_obj.tag.id == tag.id assert stored_obj.user.id == user.id @@ -3809,10 +3505,7 @@ class TestLibraryDatasetDatasetInfoAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert ( - stored_obj.library_dataset_dataset_association_id - == library_dataset_dataset_association.id - ) + assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id assert stored_obj.form_definition_id == form_definition.id assert stored_obj.form_values_id == form_values.id assert stored_obj.deleted == deleted @@ -3829,10 +3522,7 @@ class TestLibraryDatasetDatasetInfoAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert ( - stored_obj.library_dataset_dataset_association.id - == library_dataset_dataset_association.id - ) + assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id assert stored_obj.template.id == form_definition.id assert stored_obj.info.id == form_values.id @@ -3937,9 +3627,7 @@ class TestLibraryFolder(BaseTest): assert collection_consists_of_objects(stored_obj.folders, folder1) assert collection_consists_of_objects(stored_obj.active_folders, folder1) assert collection_consists_of_objects(stored_obj.library_root, library) - assert collection_consists_of_objects( - stored_obj.actions, library_folder_permission - ) + assert collection_consists_of_objects(stored_obj.actions, library_folder_permission) # use identity equality instread of object equality. assert stored_obj.datasets[0].id == library_dataset.id assert stored_obj.active_datasets[0].id == library_dataset.id @@ -3965,9 +3653,7 @@ class TestLibraryFolderInfoAssociation(BaseTest): assert stored_obj.inheritable == inheritable assert stored_obj.deleted == deleted - def test_relationships( - self, session, cls_, library_folder, form_definition, form_values - ): + def test_relationships(self, session, cls_, library_folder, form_definition, form_values): obj = cls_(library_folder, form_definition, form_values) with dbcleanup(session, obj) as obj_id: @@ -4122,9 +3808,7 @@ class TestMetadataFile(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.history_dataset.id == history_dataset_association.id - assert ( - stored_obj.library_dataset.id == library_dataset_dataset_association.id - ) + assert stored_obj.library_dataset.id == library_dataset_dataset_association.id class TestPSAAssociation(BaseTest): @@ -4269,15 +3953,9 @@ class TestPage(BaseTest): assert collection_consists_of_objects(stored_obj.revisions, page_revision) assert stored_obj.latest_revision.id == page_revision.id assert collection_consists_of_objects(stored_obj.tags, page_tag_association) - assert collection_consists_of_objects( - stored_obj.annotations, page_annotation_association - ) - assert collection_consists_of_objects( - stored_obj.ratings, page_rating_association - ) - assert collection_consists_of_objects( - stored_obj.users_shared_with, page_user_share_association - ) + assert collection_consists_of_objects(stored_obj.annotations, page_annotation_association) + assert collection_consists_of_objects(stored_obj.ratings, page_rating_association) + assert collection_consists_of_objects(stored_obj.users_shared_with, page_user_share_association) # This doesn't test the average amount, just the mapping. assert stored_obj.average_rating == page_rating_association.rating @@ -4362,9 +4040,7 @@ class TestPageRevision(BaseTest): assert stored_obj.page_id == page.id assert stored_obj.title == title assert stored_obj.content == content - assert ( - stored_obj.content_format == model.PageRevision.DEFAULT_CONTENT_FORMAT - ) + assert stored_obj.content_format == model.PageRevision.DEFAULT_CONTENT_FORMAT def test_relationships(self, session, cls_, page): obj = cls_() @@ -4545,15 +4221,9 @@ class TestQuota(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert collection_consists_of_objects( - stored_obj.default, default_quota_association - ) - assert collection_consists_of_objects( - stored_obj.groups, group_quota_association - ) - assert collection_consists_of_objects( - stored_obj.users, user_quota_association - ) + assert collection_consists_of_objects(stored_obj.default, default_quota_association) + assert collection_consists_of_objects(stored_obj.groups, group_quota_association) + assert collection_consists_of_objects(stored_obj.users, user_quota_association) class TestRole(BaseTest): @@ -4599,15 +4269,9 @@ class TestRole(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert collection_consists_of_objects( - stored_obj.dataset_actions, dataset_permission - ) - assert collection_consists_of_objects( - stored_obj.groups, group_role_association - ) - assert collection_consists_of_objects( - stored_obj.users, user_role_association - ) + assert collection_consists_of_objects(stored_obj.dataset_actions, dataset_permission) + assert collection_consists_of_objects(stored_obj.groups, group_role_association) + assert collection_consists_of_objects(stored_obj.users, user_role_association) class TestStoredWorkflow(BaseTest): @@ -4689,32 +4353,18 @@ class TestStoredWorkflow(BaseTest): assert stored_obj.user.id == user.id assert stored_obj.latest_workflow.id == workflow.id assert stored_obj.workflows[0].id == wf.id - assert collection_consists_of_objects( - stored_obj.annotations, stored_workflow_annotation_association - ) - assert collection_consists_of_objects( - stored_obj.ratings, stored_workflow_rating_association - ) + assert collection_consists_of_objects(stored_obj.annotations, stored_workflow_annotation_association) + assert collection_consists_of_objects(stored_obj.ratings, stored_workflow_rating_association) # This doesn't test the average amount, just the mapping. - assert ( - stored_obj.average_rating == stored_workflow_rating_association.rating - ) - assert collection_consists_of_objects( - stored_obj.tags, stored_workflow_tag_association, tag_assoc2 - ) + assert stored_obj.average_rating == stored_workflow_rating_association.rating + assert collection_consists_of_objects(stored_obj.tags, stored_workflow_tag_association, tag_assoc2) assert collection_consists_of_objects(stored_obj.owner_tags, tag_assoc2) - assert collection_consists_of_objects( - stored_obj.users_shared_with, stored_workflow_user_share_association - ) + assert collection_consists_of_objects(stored_obj.users_shared_with, stored_workflow_user_share_association) delete_from_database(session, [wf, tag_assoc2]) - def test_average_rating( - self, session, stored_workflow, user, stored_workflow_rating_association_factory - ): - _run_average_rating_test( - session, stored_workflow, user, stored_workflow_rating_association_factory - ) + def test_average_rating(self, session, stored_workflow, user, stored_workflow_rating_association_factory): + _run_average_rating_test(session, stored_workflow, user, stored_workflow_rating_association_factory) class TestStoredWorkflowAnnotationAssociation(BaseTest): @@ -4965,9 +4615,7 @@ class TestTask(BaseTest): assert stored_obj.task_runner_name == task_runner_name assert stored_obj.task_runner_external_id == task_runner_external_id - def test_relationships( - self, session, cls_, job, task_metric_numeric, task_metric_text - ): + def test_relationships(self, session, cls_, job, task_metric_numeric, task_metric_text): obj = cls_(job, None, None) obj.numeric_metrics.append(task_metric_numeric) obj.text_metrics.append(task_metric_text) @@ -4975,12 +4623,8 @@ class TestTask(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.job.id == job.id - assert collection_consists_of_objects( - stored_obj.numeric_metrics, task_metric_numeric - ) - assert collection_consists_of_objects( - stored_obj.text_metrics, task_metric_text - ) + assert collection_consists_of_objects(stored_obj.numeric_metrics, task_metric_numeric) + assert collection_consists_of_objects(stored_obj.text_metrics, task_metric_text) class TestTaskMetricNumeric(BaseTest): @@ -5150,7 +4794,7 @@ class TestUser(BaseTest): obj.roles.append(private_user_role) - _non_private_role = role_factory(name='a') + _non_private_role = role_factory(name="a") cleanup.append(_non_private_role) non_private_user_role = user_role_association_factory(obj, _non_private_role) @@ -5176,45 +4820,21 @@ class TestUser(BaseTest): assert stored_obj.values.id == form_values.id assert collection_consists_of_objects(stored_obj.addresses, user_address) assert collection_consists_of_objects(stored_obj.cloudauthz, cloud_authz) - assert collection_consists_of_objects( - stored_obj.custos_auth, custos_authnz_token - ) - assert collection_consists_of_objects( - stored_obj.default_permissions, default_user_permissions - ) - assert collection_consists_of_objects( - stored_obj.groups, user_group_association - ) - assert collection_consists_of_objects( - stored_obj.histories, history1, history2 - ) + assert collection_consists_of_objects(stored_obj.custos_auth, custos_authnz_token) + assert collection_consists_of_objects(stored_obj.default_permissions, default_user_permissions) + assert collection_consists_of_objects(stored_obj.groups, user_group_association) + assert collection_consists_of_objects(stored_obj.histories, history1, history2) assert collection_consists_of_objects(stored_obj.active_histories, history1) - assert collection_consists_of_objects( - stored_obj.galaxy_sessions, galaxy_session - ) - assert collection_consists_of_objects( - stored_obj.quotas, user_quota_association - ) - assert collection_consists_of_objects( - stored_obj.social_auth, user_authnz_token - ) - assert collection_consists_of_objects( - stored_obj.stored_workflow_menu_entries, swme - ) + assert collection_consists_of_objects(stored_obj.galaxy_sessions, galaxy_session) + assert collection_consists_of_objects(stored_obj.quotas, user_quota_association) + assert collection_consists_of_objects(stored_obj.social_auth, user_authnz_token) + assert collection_consists_of_objects(stored_obj.stored_workflow_menu_entries, swme) assert user_preference in stored_obj._preferences.values() assert collection_consists_of_objects(stored_obj.api_keys, api_keys) - assert collection_consists_of_objects( - stored_obj.data_manager_histories, data_manager_history_association - ) - assert collection_consists_of_objects( - stored_obj.roles, private_user_role, non_private_user_role - ) - assert collection_consists_of_objects( - stored_obj.non_private_roles, non_private_user_role - ) - assert collection_consists_of_objects( - stored_obj.stored_workflows, stored_workflow - ) + assert collection_consists_of_objects(stored_obj.data_manager_histories, data_manager_history_association) + assert collection_consists_of_objects(stored_obj.roles, private_user_role, non_private_user_role) + assert collection_consists_of_objects(stored_obj.non_private_roles, non_private_user_role) + assert collection_consists_of_objects(stored_obj.stored_workflows, stored_workflow) delete_from_database(session, cleanup) @@ -5453,9 +5073,9 @@ class TestVault(BaseTest): def test_columns(self, session, cls_): create_time = update_time = datetime.now() - key = '/some/path' - parent_key = '/some' - value = 'helloworld' + key = "/some/path" + parent_key = "/some" + value = "helloworld" obj = cls_(create_time=create_time, update_time=update_time, key=key, parent_key=parent_key, value=value) with dbcleanup(session, obj, where_clause=cls_.key == key): @@ -5541,29 +5161,17 @@ class TestVisualization(BaseTest): assert stored_obj.user.id == user.id assert stored_obj.latest_revision.id == visualization_revision.id assert collection_consists_of_objects(stored_obj.revisions, revision2) - assert collection_consists_of_objects( - stored_obj.tags, visualization_tag_association - ) - assert collection_consists_of_objects( - stored_obj.annotations, visualization_annotation_association - ) - assert collection_consists_of_objects( - stored_obj.ratings, visualization_rating_association - ) + assert collection_consists_of_objects(stored_obj.tags, visualization_tag_association) + assert collection_consists_of_objects(stored_obj.annotations, visualization_annotation_association) + assert collection_consists_of_objects(stored_obj.ratings, visualization_rating_association) # This doesn't test the average amount, just the mapping. assert stored_obj.average_rating == visualization_rating_association.rating - assert collection_consists_of_objects( - stored_obj.users_shared_with, visualization_user_share_association - ) + assert collection_consists_of_objects(stored_obj.users_shared_with, visualization_user_share_association) delete_from_database(session, revision2) - def test_average_rating( - self, session, visualization, user, visualization_rating_association_factory - ): - _run_average_rating_test( - session, visualization, user, visualization_rating_association_factory - ) + def test_average_rating(self, session, visualization, user, visualization_rating_association_factory): + _run_average_rating_test(session, visualization, user, visualization_rating_association_factory) class TestVisualizationAnnotationAssociation(BaseTest): @@ -5778,9 +5386,7 @@ class TestWorkflow(BaseTest): assert stored_obj.license == license assert stored_obj.uuid == uuid - def test_relationships( - self, session, cls_, stored_workflow, workflow, workflow_step_factory - ): + def test_relationships(self, session, cls_, stored_workflow, workflow, workflow_step_factory): obj = cls_() obj.stored_workflow = stored_workflow obj.parent_workflow_id = workflow.id @@ -5851,9 +5457,7 @@ class TestWorkflowInvocation(BaseTest): workflow_request_to_input_dataset_collection_association, workflow_invocation_output_value, ): - subworkflow_invocation_assoc = ( - workflow_invocation_to_subworkflow_invocation_association_factory() - ) + subworkflow_invocation_assoc = workflow_invocation_to_subworkflow_invocation_association_factory() obj = cls_() obj.workflow = workflow @@ -5862,14 +5466,10 @@ class TestWorkflowInvocation(BaseTest): obj.step_states.append(workflow_request_step_state) obj.input_step_parameters.append(workflow_request_input_step_parameter) obj.input_datasets.append(workflow_request_to_input_dataset_association) - obj.input_dataset_collections.append( - workflow_request_to_input_dataset_collection_association - ) + obj.input_dataset_collections.append(workflow_request_to_input_dataset_collection_association) obj.subworkflow_invocations.append(subworkflow_invocation_assoc) obj.steps.append(workflow_invocation_step) - obj.output_dataset_collections.append( - workflow_invocation_output_dataset_collection_association - ) + obj.output_dataset_collections.append(workflow_invocation_output_dataset_collection_association) obj.output_datasets.append(workflow_invocation_output_dataset_association) obj.output_values.append(workflow_invocation_output_value) @@ -5878,12 +5478,8 @@ class TestWorkflowInvocation(BaseTest): assert stored_obj.workflow.id == workflow.id assert stored_obj.history.id == history.id - assert collection_consists_of_objects( - stored_obj.input_parameters, workflow_request_input_parameter - ) - assert collection_consists_of_objects( - stored_obj.step_states, workflow_request_step_state - ) + assert collection_consists_of_objects(stored_obj.input_parameters, workflow_request_input_parameter) + assert collection_consists_of_objects(stored_obj.step_states, workflow_request_step_state) assert collection_consists_of_objects( stored_obj.input_step_parameters, workflow_request_input_step_parameter ) @@ -5894,12 +5490,8 @@ class TestWorkflowInvocation(BaseTest): stored_obj.input_dataset_collections, workflow_request_to_input_dataset_collection_association, ) - assert collection_consists_of_objects( - stored_obj.subworkflow_invocations, subworkflow_invocation_assoc - ) - assert collection_consists_of_objects( - stored_obj.steps, workflow_invocation_step - ) + assert collection_consists_of_objects(stored_obj.subworkflow_invocations, subworkflow_invocation_assoc) + assert collection_consists_of_objects(stored_obj.steps, workflow_invocation_step) assert collection_consists_of_objects( stored_obj.output_dataset_collections, workflow_invocation_output_dataset_collection_association, @@ -5908,9 +5500,7 @@ class TestWorkflowInvocation(BaseTest): stored_obj.output_datasets, workflow_invocation_output_dataset_association, ) - assert collection_consists_of_objects( - stored_obj.output_values, workflow_invocation_output_value - ) + assert collection_consists_of_objects(stored_obj.output_values, workflow_invocation_output_value) delete_from_database(session, subworkflow_invocation_assoc) @@ -5967,10 +5557,7 @@ class TestWorkflowInvocationOutputDatasetAssociation(BaseTest): class TestWorkflowInvocationOutputDatasetCollectionAssociation(BaseTest): def test_table(self, cls_): - assert ( - cls_.__tablename__ - == "workflow_invocation_output_dataset_collection_association" - ) + assert cls_.__tablename__ == "workflow_invocation_output_dataset_collection_association" def test_columns( self, @@ -5992,10 +5579,7 @@ class TestWorkflowInvocationOutputDatasetCollectionAssociation(BaseTest): assert stored_obj.id == obj_id assert stored_obj.workflow_invocation_id == workflow_invocation.id assert stored_obj.workflow_step_id == workflow_step.id - assert ( - stored_obj.dataset_collection_id - == history_dataset_collection_association.id - ) + assert stored_obj.dataset_collection_id == history_dataset_collection_association.id assert stored_obj.workflow_output_id == workflow_output.id def test_relationships( @@ -6017,10 +5601,7 @@ class TestWorkflowInvocationOutputDatasetCollectionAssociation(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.workflow_invocation.id == workflow_invocation.id assert stored_obj.workflow_step.id == workflow_step.id - assert ( - stored_obj.dataset_collection.id - == history_dataset_collection_association.id - ) + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id assert stored_obj.workflow_output.id == workflow_output.id @@ -6075,9 +5656,7 @@ class TestWorkflowInvocationOutputValue(BaseTest): assert stored_obj.workflow_invocation.id == workflow_invocation.id assert stored_obj.workflow_step.id == workflow_step.id assert stored_obj.workflow_output.id == workflow_output.id - assert ( - stored_obj.workflow_invocation_step[0].id == workflow_invocation_step.id - ) + assert stored_obj.workflow_invocation_step[0].id == workflow_invocation_step.id delete_from_database(session, [workflow_invocation_step]) @@ -6100,9 +5679,7 @@ class TestWorkflowInvocationStep(BaseTest): state, action = "a", "b" session.add(job) # must be bound to a session for lazy load of attributes - session.add( - implicit_collection_jobs - ) # must be bound to a session for lazy load of attributes + session.add(implicit_collection_jobs) # must be bound to a session for lazy load of attributes obj = cls_() obj.create_time = create_time @@ -6139,9 +5716,7 @@ class TestWorkflowInvocationStep(BaseTest): workflow_invocation_output_value, ): session.add(job) # must be bound to a session for lazy load of attributes - session.add( - implicit_collection_jobs - ) # must be bound to a session for lazy load of attributes + session.add(implicit_collection_jobs) # must be bound to a session for lazy load of attributes # setup workflow_invocation_output_value to test the output_value attribute output_value = workflow_invocation_output_value @@ -6154,9 +5729,7 @@ class TestWorkflowInvocationStep(BaseTest): obj.workflow_step = workflow_step obj.job = job obj.implicit_collection_jobs = implicit_collection_jobs - obj.output_dataset_collections.append( - workflow_invocation_step_output_dataset_collection_association - ) + obj.output_dataset_collections.append(workflow_invocation_step_output_dataset_collection_association) obj.output_datasets.append(workflow_invocation_step_output_dataset_association) with dbcleanup(session, obj) as obj_id: @@ -6185,9 +5758,7 @@ class TestWorkflowInvocationStep(BaseTest): ): # use defaults to create 2 workflows workflow_invocation1 = workflow_invocation_factory() - workflow_invocation2 = ( - workflow_invocation_factory() - ) # this is the subworkflow invocation + workflow_invocation2 = workflow_invocation_factory() # this is the subworkflow invocation # store to retrieve object ids persist(session, workflow_invocation1) @@ -6217,13 +5788,9 @@ class TestWorkflowInvocationStep(BaseTest): class TestWorkflowInvocationStepOutputDatasetAssociation(BaseTest): def test_table(self, cls_): - assert ( - cls_.__tablename__ == "workflow_invocation_step_output_dataset_association" - ) + assert cls_.__tablename__ == "workflow_invocation_step_output_dataset_association" - def test_columns( - self, session, cls_, workflow_invocation_step, history_dataset_association - ): + def test_columns(self, session, cls_, workflow_invocation_step, history_dataset_association): output_name = "a" obj = cls_() obj.workflow_invocation_step = workflow_invocation_step @@ -6237,9 +5804,7 @@ class TestWorkflowInvocationStepOutputDatasetAssociation(BaseTest): assert stored_obj.dataset_id == history_dataset_association.id assert stored_obj.output_name == output_name - def test_relationships( - self, session, cls_, workflow_invocation_step, history_dataset_association - ): + def test_relationships(self, session, cls_, workflow_invocation_step, history_dataset_association): obj = cls_() obj.workflow_invocation_step = workflow_invocation_step obj.dataset = history_dataset_association @@ -6252,10 +5817,7 @@ class TestWorkflowInvocationStepOutputDatasetAssociation(BaseTest): class TestWorkflowInvocationStepOutputDatasetCollectionAssociation(BaseTest): def test_table(self, cls_): - assert ( - cls_.__tablename__ - == "workflow_invocation_step_output_dataset_collection_association" - ) + assert cls_.__tablename__ == "workflow_invocation_step_output_dataset_collection_association" def test_columns( self, @@ -6277,10 +5839,7 @@ class TestWorkflowInvocationStepOutputDatasetCollectionAssociation(BaseTest): assert stored_obj.id == obj_id assert stored_obj.workflow_invocation_step_id == workflow_invocation_step.id assert stored_obj.workflow_step_id == workflow_step.id - assert ( - stored_obj.dataset_collection_id - == history_dataset_collection_association.id - ) + assert stored_obj.dataset_collection_id == history_dataset_collection_association.id assert stored_obj.output_name == output_name def test_relationships( @@ -6297,18 +5856,12 @@ class TestWorkflowInvocationStepOutputDatasetCollectionAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.workflow_invocation_step.id == workflow_invocation_step.id - assert ( - stored_obj.dataset_collection.id - == history_dataset_collection_association.id - ) + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id class TestWorkflowInvocationToSubworkflowInvocationAssociation(BaseTest): def test_table(self, cls_): - assert ( - cls_.__tablename__ - == "workflow_invocation_to_subworkflow_invocation_association" - ) + assert cls_.__tablename__ == "workflow_invocation_to_subworkflow_invocation_association" def test_columns( self, @@ -6349,9 +5902,7 @@ class TestWorkflowInvocationToSubworkflowInvocationAssociation(BaseTest): persist(session, parent_workflow_invocation) obj = cls_() - obj.subworkflow_invocation = ( - workflow_invocation # We need only 1 instance, so we use the fixture - ) + obj.subworkflow_invocation = workflow_invocation # We need only 1 instance, so we use the fixture obj.workflow_step = workflow_step obj.parent_workflow_invocation = parent_workflow_invocation @@ -6359,10 +5910,7 @@ class TestWorkflowInvocationToSubworkflowInvocationAssociation(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.subworkflow_invocation.id == workflow_invocation.id assert stored_obj.workflow_step.id == workflow_step.id - assert ( - stored_obj.parent_workflow_invocation.id - == parent_workflow_invocation.id - ) + assert stored_obj.parent_workflow_invocation.id == parent_workflow_invocation.id delete_from_database(session, parent_workflow_invocation) @@ -6548,10 +6096,7 @@ class TestWorkflowRequestToInputDatasetCollectionAssociation(BaseTest): assert stored_obj.id == obj_id assert stored_obj.workflow_invocation_id == workflow_invocation.id assert stored_obj.workflow_step_id == workflow_step.id - assert ( - stored_obj.dataset_collection_id - == history_dataset_collection_association.id - ) + assert stored_obj.dataset_collection_id == history_dataset_collection_association.id def test_relationships( self, @@ -6570,10 +6115,7 @@ class TestWorkflowRequestToInputDatasetCollectionAssociation(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.workflow_invocation.id == workflow_invocation.id assert stored_obj.workflow_step.id == workflow_step.id - assert ( - stored_obj.dataset_collection.id - == history_dataset_collection_association.id - ) + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id class TestWorkflowStep(BaseTest): @@ -6662,9 +6204,7 @@ class TestWorkflowStep(BaseTest): assert stored_obj.workflow.id == workflow.id assert stored_obj.subworkflow.id == subworkflow.id assert stored_obj.dynamic_tool.id == dynamic_tool.id - assert collection_consists_of_objects( - stored_obj.output_connections, workflow_step_connection_out - ) + assert collection_consists_of_objects(stored_obj.output_connections, workflow_step_connection_out) persisted = [ subworkflow, @@ -6804,18 +6344,14 @@ class TestWorkflowStepInput(BaseTest): assert stored_obj.default_value_set == default_value_set assert stored_obj.runtime_value == runtime_value - def test_relationships( - self, session, cls_, workflow_step, workflow_step_connection - ): + def test_relationships(self, session, cls_, workflow_step, workflow_step_connection): obj = cls_(workflow_step) obj.connections.append(workflow_step_connection) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.workflow_step_id == workflow_step.id - assert collection_consists_of_objects( - stored_obj.connections, workflow_step_connection - ) + assert collection_consists_of_objects(stored_obj.connections, workflow_step_connection) class TestWorkflowStepTagAssociation(BaseTest): @@ -6873,9 +6409,8 @@ def ensure_database_is_empty(session): https://docs.pytest.org/en/6.2.x/fixture.html#fixture-instantiation-order """ # Created indirectrly (via db trigger and at Job instantiation): can't cleanup up automatically - exclude = ['HistoryAudit', 'JobStateHistory'] - models = (cls_ for cls_ in model.__dict__.values() - if hasattr(cls_, '__mapper__') and cls_.__name__ not in exclude) + exclude = ["HistoryAudit", "JobStateHistory"] + models = (cls_ for cls_ in model.__dict__.values() if hasattr(cls_, "__mapper__") and cls_.__name__ not in exclude) # For each mapped class, check that the database table to which it is mapped is empty for m in models: stmt = select(func.count()).select_from(m) @@ -6885,13 +6420,15 @@ def ensure_database_is_empty(session): # Misc. helper fixtures. -@pytest.fixture(scope='module') + +@pytest.fixture(scope="module") def init_model(engine): model.mapper_registry.metadata.create_all(engine) # Fixtures yielding persisted instances of models, deleted from the database on test exit. + @pytest.fixture def api_keys(session): instance = model.APIKeys(key=get_unique_value()) @@ -6941,12 +6478,8 @@ def dataset_collection(session): @pytest.fixture -def dataset_collection_element( - session, dataset_collection, history_dataset_association -): - instance = model.DatasetCollectionElement( - collection=dataset_collection, element=history_dataset_association - ) +def dataset_collection_element(session, dataset_collection, history_dataset_association): + instance = model.DatasetCollectionElement(collection=dataset_collection, element=history_dataset_association) yield from dbcleanup_wrapper(session, instance) @@ -7013,9 +6546,7 @@ def extended_metadata_index(session, extended_metadata): @pytest.fixture def form_definition(session, form_definition_current): - instance = model.FormDefinition( - name="a", form_definition_current=form_definition_current - ) + instance = model.FormDefinition(name="a", form_definition_current=form_definition_current) yield from dbcleanup_wrapper(session, instance) @@ -7115,9 +6646,7 @@ def history_dataset_collection_rating_association( user, history_dataset_collection_association, ): - instance = model.HistoryDatasetCollectionRatingAssociation( - user, history_dataset_collection_association - ) + instance = model.HistoryDatasetCollectionRatingAssociation(user, history_dataset_collection_association) yield from dbcleanup_wrapper(session, instance) @@ -7159,9 +6688,7 @@ def implicit_collection_jobs_job_association(session): @pytest.fixture -def implicitly_converted_dataset_association( - session, history_dataset_association -): +def implicitly_converted_dataset_association(session, history_dataset_association): instance = model.ImplicitlyConvertedDatasetAssociation( dataset=history_dataset_association, parent=history_dataset_association, # using the same dataset; should work here. @@ -7170,12 +6697,8 @@ def implicitly_converted_dataset_association( @pytest.fixture -def implicitly_created_dataset_collection_input( - session, history_dataset_collection_association -): - instance = model.ImplicitlyCreatedDatasetCollectionInput( - None, history_dataset_collection_association - ) +def implicitly_created_dataset_collection_input(session, history_dataset_collection_association): + instance = model.ImplicitlyCreatedDatasetCollectionInput(None, history_dataset_collection_association) yield from dbcleanup_wrapper(session, instance) @@ -7234,12 +6757,8 @@ def job_state_history(session, job): @pytest.fixture -def job_to_implicit_output_dataset_collection_association( - session, dataset_collection -): - instance = model.JobToImplicitOutputDatasetCollectionAssociation( - None, dataset_collection - ) +def job_to_implicit_output_dataset_collection_association(session, dataset_collection): + instance = model.JobToImplicitOutputDatasetCollectionAssociation(None, dataset_collection) yield from dbcleanup_wrapper(session, instance) @@ -7250,32 +6769,20 @@ def job_to_input_dataset_association(session, history_dataset_association): @pytest.fixture -def job_to_input_dataset_collection_association( - session, history_dataset_collection_association -): - instance = model.JobToInputDatasetCollectionAssociation( - None, history_dataset_collection_association - ) +def job_to_input_dataset_collection_association(session, history_dataset_collection_association): + instance = model.JobToInputDatasetCollectionAssociation(None, history_dataset_collection_association) yield from dbcleanup_wrapper(session, instance) @pytest.fixture -def job_to_input_dataset_collection_element_association( - session, dataset_collection_element -): - instance = model.JobToInputDatasetCollectionElementAssociation( - None, dataset_collection_element - ) +def job_to_input_dataset_collection_element_association(session, dataset_collection_element): + instance = model.JobToInputDatasetCollectionElementAssociation(None, dataset_collection_element) yield from dbcleanup_wrapper(session, instance) @pytest.fixture -def job_to_input_library_dataset_association( - session, library_dataset_dataset_association -): - instance = model.JobToInputLibraryDatasetAssociation( - None, library_dataset_dataset_association - ) +def job_to_input_library_dataset_association(session, library_dataset_dataset_association): + instance = model.JobToInputLibraryDatasetAssociation(None, library_dataset_dataset_association) yield from dbcleanup_wrapper(session, instance) @@ -7286,22 +6793,14 @@ def job_to_output_dataset_association(session, history_dataset_association): @pytest.fixture -def job_to_output_dataset_collection_association( - session, history_dataset_collection_association -): - instance = model.JobToOutputDatasetCollectionAssociation( - None, history_dataset_collection_association - ) +def job_to_output_dataset_collection_association(session, history_dataset_collection_association): + instance = model.JobToOutputDatasetCollectionAssociation(None, history_dataset_collection_association) yield from dbcleanup_wrapper(session, instance) @pytest.fixture -def job_to_output_library_dataset_association( - session, library_dataset_dataset_association -): - instance = model.JobToOutputLibraryDatasetAssociation( - None, library_dataset_dataset_association - ) +def job_to_output_library_dataset_association(session, library_dataset_dataset_association): + instance = model.JobToOutputLibraryDatasetAssociation(None, library_dataset_dataset_association) yield from dbcleanup_wrapper(session, instance) @@ -7348,12 +6847,8 @@ def library_dataset_dataset_association(session): @pytest.fixture -def library_dataset_dataset_association_permission( - session, library_dataset_dataset_association, role -): - instance = model.LibraryDatasetDatasetAssociationPermissions( - "a", library_dataset_dataset_association, role - ) +def library_dataset_dataset_association_permission(session, library_dataset_dataset_association, role): + instance = model.LibraryDatasetDatasetAssociationPermissions("a", library_dataset_dataset_association, role) yield from dbcleanup_wrapper(session, instance) @@ -7720,6 +7215,7 @@ def workflow_step_tag_association(session): # how to construct the model it is testing, so instead of constructing an object directly, # a test calls a factory function, passed to it as a fixture. + @pytest.fixture def dataset_collection_factory(): def make_instance(*args, **kwds): @@ -7766,9 +7262,7 @@ def history_rating_association_factory(): @pytest.fixture -def implicitly_converted_dataset_association_factory( - history_dataset_association -): +def implicitly_converted_dataset_association_factory(history_dataset_association): def make_instance(*args, **kwds): instance = model.ImplicitlyConvertedDatasetAssociation( dataset=history_dataset_association, diff --git a/test/unit/webapps/test_request_scoped_sqlalchemy_sessions.py b/test/unit/webapps/test_request_scoped_sqlalchemy_sessions.py index 6c619059a74..d7eb3ad7b90 100644 --- a/test/unit/webapps/test_request_scoped_sqlalchemy_sessions.py +++ b/test/unit/webapps/test_request_scoped_sqlalchemy_sessions.py @@ -9,7 +9,6 @@ import pytest from fastapi import FastAPI from fastapi.param_functions import Depends from httpx import AsyncClient -pytest.importorskip("starlette_context") from starlette_context import context as request_context from galaxy.app_unittest_utils.galaxy_mock import MockApp @@ -37,7 +36,7 @@ async def _get_app(): GX_APP = MockApp() GX_APP.stop = False app = GX_APP - request_id = request_context.data['X-Request-ID'] + request_id = request_context.data["X-Request-ID"] app.model.set_request_id(request_id) try: yield app @@ -59,17 +58,17 @@ async def read_main(app=Depends(get_app)): return {"msg": "Hello World"} -@app.get('/internal_server_error') +@app.get("/internal_server_error") def error(app=Depends(get_app)): assert app.model.scoped_registry.registry == {} app.model.session() assert len(app.model.scoped_registry.registry) == 1 request_id = app.model.request_scopefunc() assert is_valid_uuid(request_id) - raise UnexpectedException('Oh noes!') + raise UnexpectedException("Oh noes!") -@app.get('/sync_wait') +@app.get("/sync_wait") def sync_wait(app=Depends(get_app)): app.model.session() time.sleep(0.2) @@ -78,7 +77,7 @@ def sync_wait(app=Depends(get_app)): return request_id -@app.get('/async_wait') +@app.get("/async_wait") async def async_wait(app=Depends(get_app)): app.model.session() await asyncio.sleep(0.2) @@ -119,7 +118,7 @@ async def test_request_scoped_sa_session_exception(): async def test_request_scoped_sa_session_concurrent_requests_sync(): add_request_id_middleware(app) async with AsyncClient(app=app, base_url="http://test") as client: - awaitables = (client.get('/sync_wait') for _ in range(10)) + awaitables = (client.get("/sync_wait") for _ in range(10)) result = await asyncio.gather(*awaitables) uuids = [] for r in result: @@ -134,7 +133,7 @@ async def test_request_scoped_sa_session_concurrent_requests_sync(): async def test_request_scoped_sa_session_concurrent_requests_async(): add_request_id_middleware(app) async with AsyncClient(app=app, base_url="http://test") as client: - awaitables = (client.get('/async_wait') for _ in range(10)) + awaitables = (client.get("/async_wait") for _ in range(10)) result = await asyncio.gather(*awaitables) uuids = [] for r in result: @@ -153,7 +152,7 @@ async def test_request_scoped_sa_session_concurrent_requests_and_background_thre with concurrent.futures.ThreadPoolExecutor() as pool: background_pool = loop.run_in_executor(pool, target) async with AsyncClient(app=app, base_url="http://test") as client: - awaitables = (client.get('/async_wait') for _ in range(10)) + awaitables = (client.get("/async_wait") for _ in range(10)) result = await asyncio.gather(*awaitables) uuids = [] for r in result: