opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release_20.01' into dev

Browse Source
This commit is contained in:
mvdbeek
2020-03-05 12:52:43 +01:00
parent 476b194819 7f2686c5c7
commit e954807275
12 changed files with 176 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/galaxy/scripts/components/admin/DataManager/DataManagerTable.vue
+22 -18
View File
@@ -13,22 +13,20 @@
<b-row>
<b-col>
<b-card id="data-table-card" flush>
<b-container v-slot:header>
<b-row align-v="center">
<b-col cols="auto">
<b-button
@click="reload()"
v-b-tooltip.hover
:title="'Reload ' + this.dataTable['name'] + ' tool data table'"
>
<span class="fa fa-refresh" />
</b-button>
</b-col>
<b-col>
<b>{{ this.dataTable["name"] }}</b>
</b-col>
</b-row>
</b-container>
<template v-slot:header>
<b-container>
<b-row align-v="center">
<b-col cols="auto">
<b-button @click="reload()" v-b-tooltip.hover :title="buttonLabel">
<span class="fa fa-refresh" />
</b-button>
</b-col>
<b-col>
<b>{{ dataTableName }}</b>
</b-col>
</b-row>
</b-container>
</template>
<b-table
:fields="fields(this.dataTable['columns'])"
:items="dataTable['data']"
@@ -68,6 +66,12 @@ export default {
};
},
computed: {
dataTableName() {
return this.dataTable && this.dataTable.name ? this.dataTable.name : "null";
},
buttonLabel() {
return `Reload ${this.dataTableName} tool data table`;
},
breadcrumbItems() {
return [
{
@@ -75,7 +79,7 @@ export default {
to: "/"
},
{
text: this.dataTable["name"]
text: this.dataTableName
}
];
}
@@ -88,7 +92,7 @@ export default {
},
reload() {
axios
.get(`${getAppRoot()}data_manager/reload_tool_data_tables?table_name=${this.dataTable["name"]}`)
.get(`${getAppRoot()}data_manager/reload_tool_data_tables?table_name=${this.dataTableName}`)
.then(response => {
if (response.data.dataTable) {
this.dataTable = response.data.dataTable;
client/galaxy/scripts/layout/menu.js
-1
View File
@@ -267,7 +267,6 @@ const Collection = Backbone.Collection.extend({
cls: "loggedout-only",
tooltip: _l("Login"),
url: "login",
target: "galaxy_main",
noscratchbook: true
};
}
doc/source/admin/galaxy_options.rst
-13
View File
@@ -711,19 +711,6 @@
:Type: int
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``manage_dependency_relationships``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:Description:
Enable use of an in-memory registry with bi-directional
relationships between repositories (i.e., in addition to lists of
dependencies for a repository, keep an in-memory registry of
dependent items for each repository.
:Default: ``false``
:Type: bool
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``tool_data_table_config_path``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
lib/galaxy/actions/admin.py
+1 -1
View File
@@ -45,7 +45,7 @@ class AdminActions(object):
# If this is a default quota, create the DefaultQuotaAssociation
if params.default != 'no':
self.app.quota_agent.set_default_quota(params.default, quota)
message = "Default quota '%s' has been created."
message = "Default quota '%s' has been created." % quota.name
else:
# Create the UserQuotaAssociations
in_users = [self.sa_session.query(self.app.model.User).get(decode_id(x) if decode_id else x) for x in util.listify(params.in_users)]
lib/galaxy/app.py
-2
View File
@@ -91,9 +91,7 @@ class UniverseApplication(config.ConfiguresGalaxyMixin):
check_migrate_tools = self.config.check_migrate_tools
self._configure_models(check_migrate_databases=self.config.check_migrate_databases, check_migrate_tools=check_migrate_tools, config_file=config_file)
# Manage installed tool shed repositories.
self.installed_repository_manager = InstalledRepositoryManager(self)
self._configure_datatypes_registry(self.installed_repository_manager)
galaxy.model.set_datatypes_registry(self.datatypes_registry)
lib/galaxy/config/sample/galaxy.yml.sample
-6
View File
@@ -451,12 +451,6 @@ galaxy:
# between 1 and 24.
#hours_between_check: 12
# Enable use of an in-memory registry with bi-directional
# relationships between repositories (i.e., in addition to lists of
# dependencies for a repository, keep an in-memory registry of
# dependent items for each repository.
#manage_dependency_relationships: false
# XML config file that contains data table entries for the
# ToolDataTableManager. This file is manually # maintained by the
# Galaxy administrator (.sample used if default does not exist).
lib/galaxy/objectstore/__init__.py
+3 -1
View File
@@ -82,7 +82,7 @@ class ObjectStore(object):
000/obj.id)
"""
def __init__(self, config, config_dict={}, **kwargs):
def __init__(self, config, config_dict=None, **kwargs):
"""
:type config: object
:param config: An object, most likely populated from
@@ -95,6 +95,8 @@ class ObjectStore(object):
parent directory those directories will be created.
* new_file_path -- Used to set the 'temp' extra_dir.
"""
if config_dict is None:
config_dict = {}
self.running = True
self.config = config
self.check_old_style = config.object_store_check_old_style
lib/galaxy/objectstore/s3.py
+37 -29
View File
@@ -125,7 +125,8 @@ class CloudConfigMixin(object):
'cache': {
'size': self.cache_size,
'path': self.staging_path,
}
},
'enable_cache_monitor': False,
}
@@ -138,7 +139,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
store_type = 's3'
def __init__(self, config, config_dict):
super(S3ObjectStore, self).__init__(config)
super(S3ObjectStore, self).__init__(config, config_dict)
self.transfer_progress = 0
@@ -146,6 +147,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
bucket_dict = config_dict['bucket']
connection_dict = config_dict.get('connection', {})
cache_dict = config_dict['cache']
self.enable_cache_monitor = config_dict.get('enable_cache_monitor', True)
self.access_key = auth_dict.get('access_key')
self.secret_key = auth_dict.get('secret_key')
@@ -167,9 +169,6 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
(e['type'], e['path']) for e in config_dict.get('extra_dirs', []))
self.extra_dirs.update(extra_dirs)
log.debug("Object cache dir: %s", self.staging_path)
log.debug(" job work dir: %s", self.extra_dirs['job_work'])
self._initialize()
def _initialize(self):
@@ -187,9 +186,17 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
'conn_path': self.conn_path}
self._configure_connection()
self.bucket = self._get_bucket(self.bucket)
self._bucket = self._get_bucket(self.bucket)
self.start_cache_monitor()
# Test if 'axel' is available for parallel download and pull the key into cache
if which('axel'):
self.use_axel = True
else:
self.use_axel = False
def start_cache_monitor(self):
# Clean cache only if value is set in galaxy.ini
if self.cache_size != -1:
if self.cache_size != -1 and self.enable_cache_monitor:
# Convert GBs to bytes for comparison
self.cache_size = self.cache_size * 1073741824
# Helper for interruptable sleep
@@ -197,11 +204,6 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
self.cache_monitor_thread = threading.Thread(target=self.__cache_monitor)
self.cache_monitor_thread.start()
log.info("Cache cleaner manager started")
# Test if 'axel' is available for parallel download and pull the key into cache
if which('axel'):
self.use_axel = True
else:
self.use_axel = False
def _configure_connection(self):
log.debug("Configuring S3 Connection")
@@ -325,7 +327,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
# alt_name can contain parent directory references, but S3 will not
# follow them, so if they are valid we normalize them out
alt_name = os.path.normpath(alt_name)
rel_path = os.path.join(*directory_hash_id(obj.id))
rel_path = os.path.join(*directory_hash_id(self._get_object_id(obj)))
if extra_dir is not None:
if extra_dir_at_root:
rel_path = os.path.join(extra_dir, rel_path)
@@ -334,7 +336,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
# for JOB_WORK directory
if obj_dir:
rel_path = os.path.join(rel_path, str(obj.id))
rel_path = os.path.join(rel_path, str(self._get_object_id(obj)))
if base_dir:
base = self.extra_dirs.get(base_dir)
return os.path.join(base, rel_path)
@@ -343,7 +345,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
rel_path = '%s/' % rel_path
if not dir_only:
rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % obj.id)
rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % self._get_object_id(obj))
return rel_path
def _get_cache_path(self, rel_path):
@@ -354,7 +356,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
def _get_size_in_s3(self, rel_path):
try:
key = self.bucket.get_key(rel_path)
key = self._bucket.get_key(rel_path)
if key:
return key.size
except S3ResponseError:
@@ -367,19 +369,17 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
# A hackish way of testing if the rel_path is a folder vs a file
is_dir = rel_path[-1] == '/'
if is_dir:
keyresult = self.bucket.get_all_keys(prefix=rel_path)
keyresult = self._bucket.get_all_keys(prefix=rel_path)
if len(keyresult) > 0:
exists = True
else:
exists = False
else:
key = Key(self.bucket, rel_path)
key = Key(self._bucket, rel_path)
exists = key.exists()
except S3ResponseError:
log.exception("Trouble checking existence of S3 key '%s'", rel_path)
return False
if rel_path[0] == '/':
raise
return exists
def _in_cache(self, rel_path):
@@ -427,7 +427,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
def _download(self, rel_path):
try:
log.debug("Pulling key '%s' into cache to %s", rel_path, self._get_cache_path(rel_path))
key = self.bucket.get_key(rel_path)
key = self._bucket.get_key(rel_path)
# Test if cache is large enough to hold the new file
if self.cache_size > 0 and key.size > self.cache_size:
log.critical("File %s is larger (%s) than the cache size (%s). Cannot download.",
@@ -446,7 +446,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
key.get_contents_to_filename(self._get_cache_path(rel_path), cb=self._transfer_cb, num_cb=10)
return True
except S3ResponseError:
log.exception("Problem downloading key '%s' from S3 bucket '%s'", rel_path, self.bucket.name)
log.exception("Problem downloading key '%s' from S3 bucket '%s'", rel_path, self._bucket.name)
return False
def _push_to_os(self, rel_path, source_file=None, from_string=None):
@@ -460,7 +460,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
try:
source_file = source_file if source_file else self._get_cache_path(rel_path)
if os.path.exists(source_file):
key = Key(self.bucket, rel_path)
key = Key(self._bucket, rel_path)
if os.path.getsize(source_file) == 0 and key.exists():
log.debug("Wanted to push file '%s' to S3 key '%s' but its size is 0; skipping.", source_file, rel_path)
return True
@@ -478,7 +478,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
cb=self._transfer_cb,
num_cb=10)
else:
multipart_upload(self.s3server, self.bucket, key.name, source_file, mb_size)
multipart_upload(self.s3server, self._bucket, key.name, source_file, mb_size)
end_time = datetime.now()
log.debug("Pushed cache file '%s' to key '%s' (%s bytes transfered in %s sec)",
source_file, rel_path, os.path.getsize(source_file), end_time - start_time)
@@ -547,7 +547,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
alt_name = kwargs.get('alt_name', None)
# Construct hashed path
rel_path = os.path.join(*directory_hash_id(obj.id))
rel_path = os.path.join(*directory_hash_id(self._get_object_id(obj)))
# Optionally append extra_dir
if extra_dir is not None:
@@ -568,7 +568,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
# self._push_to_os(s3_dir, from_string='')
# If instructed, create the dataset in cache & in S3
if not dir_only:
rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % obj.id)
rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % self._get_object_id(obj))
open(os.path.join(self.staging_path, rel_path), 'w').close()
self._push_to_os(rel_path, from_string='')
@@ -609,7 +609,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
# but requires iterating through each individual key in S3 and deleing it.
if entire_dir and extra_dir:
shutil.rmtree(self._get_cache_path(rel_path))
results = self.bucket.get_all_keys(prefix=rel_path)
results = self._bucket.get_all_keys(prefix=rel_path)
for key in results:
log.debug("Deleting key %s", key.name)
key.delete()
@@ -619,7 +619,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
os.unlink(self._get_cache_path(rel_path))
# Delete from S3 as well
if self._key_exists(rel_path):
key = Key(self.bucket, rel_path)
key = Key(self._bucket, rel_path)
log.debug("Deleting key %s", key.name)
key.delete()
return True
@@ -707,7 +707,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
if self.exists(obj, **kwargs):
rel_path = self._construct_path(obj, **kwargs)
try:
key = Key(self.bucket, rel_path)
key = Key(self._bucket, rel_path)
return key.generate_url(expires_in=86400) # 24hrs
except S3ResponseError:
log.exception("Trouble generating URL for dataset '%s'", rel_path)
@@ -716,6 +716,14 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
def get_store_usage_percent(self):
return 0.0
def shutdown(self):
self.running = False
thread = getattr(self, 'cache_monitor_thread', None)
if thread:
log.debug("Shutting down thread")
self.sleeper.wake()
thread.join(5)
class SwiftObjectStore(S3ObjectStore):
"""
lib/galaxy/tool_shed/galaxy_install/install_manager.py
-8
View File
@@ -215,10 +215,6 @@ class InstallToolDependencyManager(object):
if tool_dependency and tool_dependency.status in [self.install_model.ToolDependency.installation_status.INSTALLED,
self.install_model.ToolDependency.installation_status.ERROR]:
installed_packages.append(tool_dependency)
if self.app.config.manage_dependency_relationships:
# Add the tool_dependency to the in-memory dictionaries in the installed_repository_manager.
self.app.installed_repository_manager.handle_tool_dependency_install(tool_shed_repository,
tool_dependency)
return installed_packages
def install_via_fabric(self, tool_shed_repository, tool_dependency, install_dir, package_name=None, custom_fabfile_path=None,
@@ -946,10 +942,6 @@ class InstallRepositoryManager(object):
basic_util.remove_dir(work_dir)
self.update_tool_shed_repository_status(tool_shed_repository,
self.install_model.ToolShedRepository.installation_status.INSTALLED)
if self.app.config.manage_dependency_relationships:
# Add the installed repository and any tool dependencies to the in-memory dictionaries
# in the installed_repository_manager.
self.app.installed_repository_manager.handle_repository_install(tool_shed_repository)
else:
# An error occurred while cloning the repository, so reset everything necessary to enable another attempt.
repository_util.set_repository_attributes(self.app,
lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py
-91
View File
@@ -79,9 +79,6 @@ class InstalledRepositoryManager(object):
# whose values are a list of tuples defining tool_dependency objects (whose status is 'Installed') that require the key
# at runtime. The value defines the entire tool dependency tree.
self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies = {}
if app.config.manage_dependency_relationships:
# Load defined dependency relationships for installed tool shed repositories and their contents.
self.load_dependency_relationships()
def activate_repository(self, repository):
"""Activate an installed tool shed repository that has been marked as deactivated."""
@@ -651,90 +648,6 @@ class InstalledRepositoryManager(object):
deleted_tool_dependency_names.append(original_dependency_val_dict['name'])
return updated_tool_dependency_names, deleted_tool_dependency_names
def handle_repository_install(self, repository):
"""Load the dependency relationships for a repository that was just installed or reinstalled."""
# Populate self.repository_dependencies_of_installed_repositories.
self.add_entry_to_repository_dependencies_of_installed_repositories(repository)
# Populate self.installed_repository_dependencies_of_installed_repositories.
self.add_entry_to_installed_repository_dependencies_of_installed_repositories(repository)
# Populate self.tool_dependencies_of_installed_repositories.
self.add_entry_to_tool_dependencies_of_installed_repositories(repository)
# Populate self.installed_tool_dependencies_of_installed_repositories.
self.add_entry_to_installed_tool_dependencies_of_installed_repositories(repository)
for tool_dependency in repository.tool_dependencies:
# Populate self.runtime_tool_dependencies_of_installed_tool_dependencies.
self.add_entry_to_runtime_tool_dependencies_of_installed_tool_dependencies(tool_dependency)
# Populate self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies.
self.add_entry_to_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies(tool_dependency)
def handle_repository_uninstall(self, repository):
"""Remove the dependency relationships for a repository that was just uninstalled."""
for tool_dependency in repository.tool_dependencies:
tool_dependency_tup = self.get_tool_dependency_tuple_for_installed_repository_manager(tool_dependency)
# Remove this tool_dependency from all values in
# self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies
altered_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies = {}
for (td_tup, installed_runtime_dependent_tool_dependency_tups) in self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies.items():
if tool_dependency_tup in installed_runtime_dependent_tool_dependency_tups:
# Remove the tool_dependency from the list.
installed_runtime_dependent_tool_dependency_tups.remove(tool_dependency_tup)
# Add the possibly altered list to the altered dictionary.
altered_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies[td_tup] = \
installed_runtime_dependent_tool_dependency_tups
self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies = \
altered_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies
# Remove the entry for this tool_dependency from self.runtime_tool_dependencies_of_installed_tool_dependencies.
self.remove_entry_from_runtime_tool_dependencies_of_installed_tool_dependencies(tool_dependency)
# Remove the entry for this tool_dependency from
# self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies.
self.remove_entry_from_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies(tool_dependency)
# Remove this repository's entry from self.installed_tool_dependencies_of_installed_repositories.
self.remove_entry_from_installed_tool_dependencies_of_installed_repositories(repository)
# Remove this repository's entry from self.tool_dependencies_of_installed_repositories
self.remove_entry_from_tool_dependencies_of_installed_repositories(repository)
# Remove this repository's entry from self.installed_repository_dependencies_of_installed_repositories.
self.remove_entry_from_installed_repository_dependencies_of_installed_repositories(repository)
# Remove this repository's entry from self.repository_dependencies_of_installed_repositories.
self.remove_entry_from_repository_dependencies_of_installed_repositories(repository)
def handle_tool_dependency_install(self, repository, tool_dependency):
"""Load the dependency relationships for a tool dependency that was just installed independently of its containing repository."""
# The received repository must have a status of 'Installed'. The value of tool_dependency.status will either be
# 'Installed' or 'Error', but we only need to change the in-memory dictionaries if it is 'Installed'.
if tool_dependency.is_installed:
# Populate self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies.
self.add_entry_to_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies(tool_dependency)
# Populate self.installed_tool_dependencies_of_installed_repositories.
repository_tup = self.get_repository_tuple_for_installed_repository_manager(repository)
tool_dependency_tup = self.get_tool_dependency_tuple_for_installed_repository_manager(tool_dependency)
if repository_tup in self.installed_tool_dependencies_of_installed_repositories:
self.installed_tool_dependencies_of_installed_repositories[repository_tup].append(tool_dependency_tup)
else:
self.installed_tool_dependencies_of_installed_repositories[repository_tup] = [tool_dependency_tup]
def load_dependency_relationships(self):
"""Load relationships for all installed repositories and tool dependencies into in-memnory dictionaries."""
# Get the list of installed tool shed repositories.
for repository in self.context.query(self.app.install_model.ToolShedRepository) \
.filter(self.app.install_model.ToolShedRepository.table.c.status ==
self.app.install_model.ToolShedRepository.installation_status.INSTALLED):
# Populate self.repository_dependencies_of_installed_repositories.
self.add_entry_to_repository_dependencies_of_installed_repositories(repository)
# Populate self.installed_repository_dependencies_of_installed_repositories.
self.add_entry_to_installed_repository_dependencies_of_installed_repositories(repository)
# Populate self.tool_dependencies_of_installed_repositories.
self.add_entry_to_tool_dependencies_of_installed_repositories(repository)
# Populate self.installed_tool_dependencies_of_installed_repositories.
self.add_entry_to_installed_tool_dependencies_of_installed_repositories(repository)
# Get the list of installed tool dependencies.
for tool_dependency in self.context.query(self.app.install_model.ToolDependency) \
.filter(self.app.install_model.ToolDependency.table.c.status ==
self.app.install_model.ToolDependency.installation_status.INSTALLED):
# Populate self.runtime_tool_dependencies_of_installed_tool_dependencies.
self.add_entry_to_runtime_tool_dependencies_of_installed_tool_dependencies(tool_dependency)
# Populate self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies.
self.add_entry_to_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies(tool_dependency)
def load_proprietary_datatypes(self):
cdl = custom_datatype_manager.CustomDatatypeLoader(self.app)
for tool_shed_repository in self.context.query(self.install_model.ToolShedRepository) \
@@ -814,10 +727,6 @@ class InstalledRepositoryManager(object):
if remove_from_disk:
repository.status = self.app.install_model.ToolShedRepository.installation_status.UNINSTALLED
repository.error_message = None
if self.app.config.manage_dependency_relationships:
# Remove the uninstalled repository and any tool dependencies from the in-memory dictionaries in the
# installed_repository_manager.
self.handle_repository_uninstall(repository)
else:
repository.status = self.app.install_model.ToolShedRepository.installation_status.DEACTIVATED
self.app.install_model.context.current.add(repository)
lib/galaxy/webapps/galaxy/config_schema.yml
-9
View File
@@ -537,15 +537,6 @@ mapping:
server process should be able to check for repository updates. The
setting for hours_between_check should be an integer between 1 and 24.
manage_dependency_relationships:
type: bool
default: false
required: false
desc: |
Enable use of an in-memory registry with bi-directional relationships between
repositories (i.e., in addition to lists of dependencies for a repository,
keep an in-memory registry of dependent items for each repository.
tool_data_table_config_path:
type: str
default: config/tool_data_table_conf.xml
test/integration/objectstore/test_swift_objectstore.py
+113
View File
@@ -0,0 +1,113 @@
import os
import string
import subprocess
from galaxy_test.driver import integration_util
OBJECT_STORE_HOST = os.environ.get('GALAXY_INTEGRATION_OBJECT_STORE_HOST', '127.0.0.1')
OBJECT_STORE_PORT = int(os.environ.get('GALAXY_INTEGRATION_OBJECT_STORE_PORT', 9000))
OBJECT_STORE_ACCESS_KEY = os.environ.get('GALAXY_INTEGRATION_OBJECT_STORE_ACCESS_KEY', 'minioadmin')
OBJECT_STORE_SECRET_KEY = os.environ.get('GALAXY_INTEGRATION_OBJECT_STORE_SECRET_KEY', 'minioadmin')
OBJECT_STORE_CONFIG = string.Template("""
<object_store type="hierarchical" id="primary">
<backends>
<object_store id="swifty" type="swift" weight="1" order="0">
<auth access_key="${access_key}" secret_key="${secret_key}" />
<bucket name="galaxy" use_reduced_redundancy="False" max_chunk_size="250"/>
<connection host="${host}" port="${port}" is_secure="False" conn_path="" multipart="True"/>
<cache path="${temp_directory}/object_store_cache" size="1000" />
<extra_dir type="job_work" path="${temp_directory}/job_working_directory_swift"/>
<extra_dir type="temp" path="${temp_directory}/tmp_swift"/>
</object_store>
</backends>
</object_store>
""")
TEST_TOOL_IDS = [
"multi_output",
"multi_output_configured",
"multi_output_assign_primary",
"multi_output_recurse",
"tool_provided_metadata_1",
"tool_provided_metadata_2",
"tool_provided_metadata_3",
"tool_provided_metadata_4",
"tool_provided_metadata_5",
"tool_provided_metadata_6",
"tool_provided_metadata_7",
"tool_provided_metadata_8",
"tool_provided_metadata_9",
"tool_provided_metadata_10",
"tool_provided_metadata_11",
"tool_provided_metadata_12",
"composite_output",
"composite_output_tests",
"metadata",
"metadata_bam",
"output_format",
"output_auto_format",
]
def start_minio(container_name):
minio_start_args = [
'docker',
'run',
'-p',
'{port}:9000'.format(port=OBJECT_STORE_PORT),
'-d',
'--name',
container_name,
# '--rm',
'minio/minio:latest',
'server',
'/tmp/data']
subprocess.check_call(minio_start_args)
def stop_minio(container_name):
subprocess.check_call(['docker', 'rm', '-f', container_name])
@integration_util.skip_unless_docker()
class SwiftObjectStoreIntegrationTestCase(integration_util.IntegrationTestCase):
@classmethod
def setUpClass(cls):
cls.container_name = "%s_container" % cls.__name__
start_minio(cls.container_name)
super(SwiftObjectStoreIntegrationTestCase, cls).setUpClass()
@classmethod
def tearDownClass(cls):
stop_minio(cls.container_name)
super(SwiftObjectStoreIntegrationTestCase, cls).tearDownClass()
@classmethod
def handle_galaxy_config_kwds(cls, config):
temp_directory = cls._test_driver.mkdtemp()
cls.object_stores_parent = temp_directory
config_path = os.path.join(temp_directory, "object_store_conf.xml")
config["object_store_store_by"] = "uuid"
# This doesn't quite work yet, fails with extra_files_path
# config["metadata_strategy"] = "extended"
config["outpus_to_working_dir"] = True
config["retry_metadata_internally"] = False
with open(config_path, "w") as f:
f.write(
OBJECT_STORE_CONFIG.safe_substitute(
{
"temp_directory": temp_directory,
"host": OBJECT_STORE_HOST,
"port": OBJECT_STORE_PORT,
"access_key": OBJECT_STORE_ACCESS_KEY,
"secret_key": OBJECT_STORE_SECRET_KEY,
}
)
)
config["object_store_config_file"] = config_path
instance = integration_util.integration_module_instance(SwiftObjectStoreIntegrationTestCase)
test_tools = integration_util.integration_tool_runner(TEST_TOOL_IDS)