mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-10-30 18:13:23 +08:00
Merge pull request #5119 from mvdbeek/dev_toolshed_install_improvements_2
Improve tool validation and metadata generation
This commit is contained in:
John Chilton
@@ -143,7 +143,7 @@ class ToolShedRepository(object):
|
||||
if tool_id in tool_ids:
|
||||
self.shed_config_filename = name
|
||||
return shed_tool_conf_dict
|
||||
if self.includes_datatypes:
|
||||
if self.includes_datatypes or self.includes_data_managers:
|
||||
# We need to search by file paths here, which is less desirable.
|
||||
tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url(self.tool_shed)
|
||||
for shed_tool_conf_dict in app.toolbox.dynamic_confs(include_migrated_tool_conf=True):
|
||||
|
||||
@@ -940,9 +940,10 @@ class Tool(object, Dictifiable):
|
||||
for citation_elem in citations_elem:
|
||||
if citation_elem.tag != "citation":
|
||||
pass
|
||||
citation = self.app.citations_manager.parse_citation(citation_elem, self.tool_dir)
|
||||
if citation:
|
||||
citations.append(citation)
|
||||
if hasattr(self.app, 'citations_manager'):
|
||||
citation = self.app.citations_manager.parse_citation(citation_elem, self.tool_dir)
|
||||
if citation:
|
||||
citations.append(citation)
|
||||
return citations
|
||||
|
||||
def parse_input_elem(self, page_source, enctypes, context=None):
|
||||
@@ -1378,7 +1379,7 @@ class Tool(object, Dictifiable):
|
||||
for input_param in self.input_params:
|
||||
if isinstance(input_param, SelectToolParameter) and input_param.is_dynamic:
|
||||
options = input_param.options
|
||||
if options and options.missing_index_file and input_param not in params:
|
||||
if options and options.tool_data_table and options.tool_data_table.missing_index_file and input_param not in params:
|
||||
params.append(input_param)
|
||||
return params
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ import requests
|
||||
from galaxy import util
|
||||
from galaxy.util.dictifiable import Dictifiable
|
||||
from galaxy.util.odict import odict
|
||||
from galaxy.util.renamed_temporary_file import RenamedTemporaryFile
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -186,7 +187,7 @@ class ToolDataTableManager(object):
|
||||
# add new elems
|
||||
out_elems.extend(new_elems)
|
||||
out_path_is_new = not os.path.exists(full_path)
|
||||
with open(full_path, 'wb') as out:
|
||||
with RenamedTemporaryFile(full_path) as out:
|
||||
out.write('<?xml version="1.0"?>\n<tables>\n')
|
||||
for elem in out_elems:
|
||||
out.write(util.xml_to_string(elem, pretty=True))
|
||||
|
||||
@@ -34,6 +34,7 @@ from ..parameters import (
|
||||
history_query
|
||||
)
|
||||
from ..parser import get_input_source as ensure_input_source
|
||||
from ..repositories import ValidationContext
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -1435,7 +1436,7 @@ class BaseDataToolParameter(ToolParameter):
|
||||
super(BaseDataToolParameter, self).__init__(tool, input_source)
|
||||
self.refresh_on_change = True
|
||||
|
||||
def _datatypes_registery(self, trans, tool):
|
||||
def _datatypes_registry(self, trans, tool):
|
||||
# Find datatypes_registry
|
||||
if tool is None:
|
||||
if trans:
|
||||
@@ -1448,22 +1449,26 @@ class BaseDataToolParameter(ToolParameter):
|
||||
datatypes_registry = galaxy.datatypes.registry.Registry()
|
||||
datatypes_registry.load_datatypes()
|
||||
else:
|
||||
datatypes_registry = tool.app.datatypes_registry
|
||||
if isinstance(tool.app, ValidationContext):
|
||||
datatypes_registry = {}
|
||||
else:
|
||||
datatypes_registry = tool.app.datatypes_registry
|
||||
return datatypes_registry
|
||||
|
||||
def _parse_formats(self, trans, tool, input_source):
|
||||
datatypes_registry = self._datatypes_registery(trans, tool)
|
||||
|
||||
datatypes_registry = self._datatypes_registry(trans, tool)
|
||||
formats = []
|
||||
# Build list of classes for supported data formats
|
||||
self.extensions = input_source.get('format', 'data').split(",")
|
||||
normalized_extensions = [extension.strip().lower() for extension in self.extensions]
|
||||
formats = []
|
||||
for extension in normalized_extensions:
|
||||
datatype = datatypes_registry.get_datatype_by_extension(extension)
|
||||
if datatype is not None:
|
||||
formats.append(datatype)
|
||||
else:
|
||||
log.warning("Datatype class not found for extension '%s', which is used in the 'format' attribute of parameter '%s'" % (extension, self.name))
|
||||
if datatypes_registry:
|
||||
# Skip during validation since no datatypes are loaded
|
||||
for extension in normalized_extensions:
|
||||
datatype = datatypes_registry.get_datatype_by_extension(extension)
|
||||
if datatype is not None:
|
||||
formats.append(datatype)
|
||||
else:
|
||||
log.warning("Datatype class not found for extension '%s', which is used in the 'format' attribute of parameter '%s'" % (extension, self.name))
|
||||
self.formats = formats
|
||||
|
||||
def _parse_options(self, input_source):
|
||||
@@ -1777,9 +1782,9 @@ class DataToolParameter(BaseDataToolParameter):
|
||||
# create dictionary and fill default parameters
|
||||
d = super(DataToolParameter, self).to_dict(trans)
|
||||
extensions = self.extensions
|
||||
datatypes_registery = self._datatypes_registery(trans, self.tool)
|
||||
all_edam_formats = datatypes_registery.edam_formats if hasattr(datatypes_registery, 'edam_formats') else {}
|
||||
all_edam_data = datatypes_registery.edam_data if hasattr(datatypes_registery, 'edam_formats') else {}
|
||||
datatypes_registry = self._datatypes_registry(trans, self.tool)
|
||||
all_edam_formats = datatypes_registry.edam_formats if hasattr(datatypes_registry, 'edam_formats') else {}
|
||||
all_edam_data = datatypes_registry.edam_data if hasattr(datatypes_registry, 'edam_formats') else {}
|
||||
edam_formats = [all_edam_formats.get(ext, None) for ext in extensions]
|
||||
edam_data = [all_edam_data.get(ext, None) for ext in extensions]
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Provides a subset of app for verifying tools."""
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
|
||||
from galaxy.datatypes.registry import Registry
|
||||
from galaxy.tools.data import ToolDataTableManager
|
||||
from galaxy.util.bunch import Bunch
|
||||
from galaxy.util.dbkeys import GenomeBuilds
|
||||
|
||||
|
||||
class ValidationContext(object):
|
||||
"""Minimal App object for tool validation."""
|
||||
|
||||
def __init__(self, app_name,
|
||||
security,
|
||||
model,
|
||||
tool_data_path,
|
||||
shed_tool_data_path,
|
||||
tool_data_tables=None,
|
||||
registry=None,
|
||||
hgweb_config_manager=None):
|
||||
self.name = app_name
|
||||
self.security = security
|
||||
self.model = model
|
||||
self.config = Bunch()
|
||||
self.config.tool_data_path = tool_data_path
|
||||
self.config.shed_tool_data_path = shed_tool_data_path
|
||||
_, self.config.tool_data_table_config = tempfile.mkstemp()
|
||||
_, self.config.shed_tool_data_table_config = tempfile.mkstemp()
|
||||
self.tool_data_tables = tool_data_tables
|
||||
self.datatypes_registry = registry or Registry()
|
||||
self.hgweb_config_manager = hgweb_config_manager
|
||||
_, self.config.len_file_path = tempfile.mkstemp()
|
||||
_, self.config.builds_file_path = tempfile.mkstemp()
|
||||
self.genome_builds = GenomeBuilds(self)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
cleanup_paths = {self.config.builds_file_path,
|
||||
self.config.len_file_path,
|
||||
self.config.tool_data_table_config,
|
||||
self.config.shed_tool_data_table_config}
|
||||
for path in cleanup_paths:
|
||||
try:
|
||||
os.remove(path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
@contextmanager
|
||||
def from_app(app, work_dir=None):
|
||||
cleanup = False
|
||||
if not work_dir:
|
||||
work_dir = tempfile.mkdtemp()
|
||||
cleanup = True
|
||||
tool_data_tables = ToolDataTableManager(work_dir)
|
||||
with ValidationContext(app_name=app.name,
|
||||
security=app.security,
|
||||
model=app.model,
|
||||
tool_data_path=work_dir,
|
||||
shed_tool_data_path=work_dir,
|
||||
tool_data_tables=tool_data_tables,
|
||||
hgweb_config_manager=getattr(app, 'hgweb_config_manager', None)
|
||||
) as app:
|
||||
yield app
|
||||
if cleanup:
|
||||
shutil.rmtree(work_dir, ignore_errors=True)
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Safely write file to temporary file and then move file into place."""
|
||||
# Copied from https://stackoverflow.com/a/12007885.
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
|
||||
class RenamedTemporaryFile(object):
|
||||
"""
|
||||
A temporary file object which will be renamed to the specified
|
||||
path on exit.
|
||||
"""
|
||||
def __init__(self, final_path, **kwargs):
|
||||
tmpfile_dir = kwargs.pop('dir', None)
|
||||
|
||||
# Put temporary file in the same directory as the location for the
|
||||
# final file so that an atomic move into place can occur.
|
||||
|
||||
if tmpfile_dir is None:
|
||||
tmpfile_dir = os.path.dirname(final_path)
|
||||
|
||||
self.tmpfile = tempfile.NamedTemporaryFile(dir=tmpfile_dir, **kwargs)
|
||||
self.final_path = final_path
|
||||
|
||||
def __getattr__(self, attr):
|
||||
"""
|
||||
Delegate attribute access to the underlying temporary file object.
|
||||
"""
|
||||
return getattr(self.tmpfile, attr)
|
||||
|
||||
def __enter__(self):
|
||||
self.tmpfile.__enter__()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if exc_type is None:
|
||||
self.tmpfile.delete = False
|
||||
result = self.tmpfile.__exit__(exc_type, exc_val, exc_tb)
|
||||
os.rename(self.tmpfile.name, self.final_path)
|
||||
else:
|
||||
result = self.tmpfile.__exit__(exc_type, exc_val, exc_tb)
|
||||
|
||||
return result
|
||||
@@ -12,6 +12,7 @@ from galaxy.exceptions import (
|
||||
RequestParameterInvalidException
|
||||
)
|
||||
from galaxy.tools.parameters import params_to_strings
|
||||
from galaxy.tools.repositories import ValidationContext
|
||||
from galaxy.web import _future_expose_api_raw_anonymous_and_sessionless as expose_api_raw_anonymous_and_sessionless
|
||||
from galaxy.web.base.controller import BaseAPIController
|
||||
from galaxy.webapps.tool_shed.search.tool_search import ToolSearch
|
||||
@@ -163,10 +164,11 @@ class ToolsController(BaseAPIController):
|
||||
trans.response.status = 404
|
||||
return {'status': 'error', 'message': message}
|
||||
|
||||
tv = tool_validator.ToolValidator(trans.app)
|
||||
repository, tool, message = tv.load_tool_from_changeset_revision(tsr_id,
|
||||
changeset,
|
||||
found_tool.tool_config)
|
||||
with ValidationContext.from_app(trans.app) as validation_context:
|
||||
tv = tool_validator.ToolValidator(validation_context)
|
||||
repository, tool, message = tv.load_tool_from_changeset_revision(tsr_id,
|
||||
changeset,
|
||||
found_tool.tool_config)
|
||||
if message:
|
||||
status = 'error'
|
||||
return dict(message=message, status=status)
|
||||
|
||||
@@ -18,6 +18,7 @@ from galaxy import (
|
||||
util,
|
||||
web
|
||||
)
|
||||
from galaxy.tools.repositories import ValidationContext
|
||||
from galaxy.web.base.controller import BaseUIController
|
||||
from galaxy.web.form_builder import CheckboxField
|
||||
from galaxy.web.framework.helpers import grids
|
||||
@@ -864,10 +865,11 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
|
||||
message = escape(kwd.get('message', ''))
|
||||
status = kwd.get('status', 'done')
|
||||
render_repository_actions_for = kwd.get('render_repository_actions_for', 'tool_shed')
|
||||
tv = tool_validator.ToolValidator(trans.app)
|
||||
repository, tool, message = tv.load_tool_from_changeset_revision(repository_id,
|
||||
changeset_revision,
|
||||
tool_config)
|
||||
with ValidationContext.from_app(trans.app) as validation_context:
|
||||
tv = tool_validator.ToolValidator(validation_context)
|
||||
repository, tool, message = tv.load_tool_from_changeset_revision(repository_id,
|
||||
changeset_revision,
|
||||
tool_config)
|
||||
if message:
|
||||
status = 'error'
|
||||
tool_state = tool_util.new_state(trans, tool, invalid=False)
|
||||
@@ -1757,17 +1759,19 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
|
||||
def load_invalid_tool(self, trans, repository_id, tool_config, changeset_revision, **kwd):
|
||||
message = escape(kwd.get('message', ''))
|
||||
render_repository_actions_for = kwd.get('render_repository_actions_for', 'tool_shed')
|
||||
tv = tool_validator.ToolValidator(trans.app)
|
||||
repository, tool, error_message = tv.load_tool_from_changeset_revision(repository_id,
|
||||
changeset_revision,
|
||||
tool_config)
|
||||
tool_state = tool_util.new_state(trans, tool, invalid=True)
|
||||
invalid_file_tups = []
|
||||
if tool:
|
||||
invalid_file_tups = tv.check_tool_input_params(repository.repo_path(trans.app),
|
||||
tool_config,
|
||||
tool,
|
||||
[])
|
||||
|
||||
with ValidationContext.from_app(trans.app) as validation_context:
|
||||
tv = tool_validator.ToolValidator(validation_context)
|
||||
repository, tool, error_message = tv.load_tool_from_changeset_revision(repository_id,
|
||||
changeset_revision,
|
||||
tool_config)
|
||||
tool_state = tool_util.new_state(trans, tool, invalid=True)
|
||||
invalid_file_tups = []
|
||||
if tool:
|
||||
invalid_file_tups = tv.check_tool_input_params(repository.repo_path(trans.app),
|
||||
tool_config,
|
||||
tool,
|
||||
[])
|
||||
if invalid_file_tups:
|
||||
message = tool_util.generate_message_for_invalid_tools(trans.app,
|
||||
invalid_file_tups,
|
||||
@@ -2884,7 +2888,6 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
|
||||
tool_lineage = []
|
||||
tool = None
|
||||
guid = None
|
||||
original_tool_data_path = trans.app.config.tool_data_path
|
||||
revision_label = hg_util.get_revision_label(trans.app, repository, changeset_revision, include_date=False)
|
||||
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(trans.app, repository_id, changeset_revision)
|
||||
if repository_metadata:
|
||||
@@ -2892,43 +2895,43 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
|
||||
metadata = repository_metadata.metadata
|
||||
if metadata:
|
||||
if 'tools' in metadata:
|
||||
tv = tool_validator.ToolValidator(trans.app)
|
||||
for tool_metadata_dict in metadata['tools']:
|
||||
if tool_metadata_dict['id'] == tool_id:
|
||||
work_dir = tempfile.mkdtemp()
|
||||
relative_path_to_tool_config = tool_metadata_dict['tool_config']
|
||||
guid = tool_metadata_dict['guid']
|
||||
full_path_to_tool_config = os.path.abspath(relative_path_to_tool_config)
|
||||
full_path_to_dir, tool_config_filename = os.path.split(full_path_to_tool_config)
|
||||
can_use_disk_file = tv.can_use_tool_config_disk_file(repository,
|
||||
repo,
|
||||
full_path_to_tool_config,
|
||||
changeset_revision)
|
||||
if can_use_disk_file:
|
||||
trans.app.config.tool_data_path = work_dir
|
||||
tool, valid, message, sample_files = \
|
||||
tv.handle_sample_files_and_load_tool_from_disk(repo_files_dir,
|
||||
repository_id,
|
||||
full_path_to_tool_config,
|
||||
work_dir)
|
||||
if message:
|
||||
status = 'error'
|
||||
else:
|
||||
tool, message, sample_files = \
|
||||
tv.handle_sample_files_and_load_tool_from_tmp_config(repo,
|
||||
repository_id,
|
||||
changeset_revision,
|
||||
tool_config_filename,
|
||||
work_dir)
|
||||
if message:
|
||||
status = 'error'
|
||||
basic_util.remove_dir(work_dir)
|
||||
break
|
||||
if guid:
|
||||
tvm = tool_version_manager.ToolVersionManager(trans.app)
|
||||
tool_lineage = tvm.get_version_lineage_for_tool(repository_id,
|
||||
repository_metadata,
|
||||
guid)
|
||||
with ValidationContext.from_app(trans.app) as validation_context:
|
||||
tv = tool_validator.ToolValidator(validation_context)
|
||||
for tool_metadata_dict in metadata['tools']:
|
||||
if tool_metadata_dict['id'] == tool_id:
|
||||
work_dir = tempfile.mkdtemp()
|
||||
relative_path_to_tool_config = tool_metadata_dict['tool_config']
|
||||
guid = tool_metadata_dict['guid']
|
||||
full_path_to_tool_config = os.path.abspath(relative_path_to_tool_config)
|
||||
full_path_to_dir, tool_config_filename = os.path.split(full_path_to_tool_config)
|
||||
can_use_disk_file = tv.can_use_tool_config_disk_file(repository,
|
||||
repo,
|
||||
full_path_to_tool_config,
|
||||
changeset_revision)
|
||||
if can_use_disk_file:
|
||||
tool, valid, message, sample_files = \
|
||||
tv.handle_sample_files_and_load_tool_from_disk(repo_files_dir,
|
||||
repository_id,
|
||||
full_path_to_tool_config,
|
||||
work_dir)
|
||||
if message:
|
||||
status = 'error'
|
||||
else:
|
||||
tool, message, sample_files = \
|
||||
tv.handle_sample_files_and_load_tool_from_tmp_config(repo,
|
||||
repository_id,
|
||||
changeset_revision,
|
||||
tool_config_filename,
|
||||
work_dir)
|
||||
if message:
|
||||
status = 'error'
|
||||
basic_util.remove_dir(work_dir)
|
||||
break
|
||||
if guid:
|
||||
tvm = tool_version_manager.ToolVersionManager(trans.app)
|
||||
tool_lineage = tvm.get_version_lineage_for_tool(repository_id,
|
||||
repository_metadata,
|
||||
guid)
|
||||
else:
|
||||
repository_metadata_id = None
|
||||
metadata = None
|
||||
@@ -2937,7 +2940,6 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
|
||||
selected_value=changeset_revision,
|
||||
add_id_to_name=False,
|
||||
downloadable=False)
|
||||
trans.app.config.tool_data_path = original_tool_data_path
|
||||
return trans.fill_template("/webapps/tool_shed/repository/view_tool_metadata.mako",
|
||||
render_repository_actions_for=render_repository_actions_for,
|
||||
repository=repository,
|
||||
|
||||
@@ -17,7 +17,7 @@ from tool_shed.dependencies import attribute_handlers
|
||||
from tool_shed.galaxy_install import dependency_display
|
||||
from tool_shed.metadata import repository_metadata_manager
|
||||
from tool_shed.repository_types import util as rt_util
|
||||
from tool_shed.tools import data_table_manager
|
||||
from tool_shed.tools.data_table_manager import ShedToolDataTableManager
|
||||
from tool_shed.util import (
|
||||
basic_util,
|
||||
commit_util,
|
||||
@@ -99,7 +99,7 @@ class UploadController(BaseUIController):
|
||||
if uploaded_file or uploaded_directory:
|
||||
rdah = attribute_handlers.RepositoryDependencyAttributeHandler(trans.app, unpopulate=False)
|
||||
tdah = attribute_handlers.ToolDependencyAttributeHandler(trans.app, unpopulate=False)
|
||||
tdtm = data_table_manager.ToolDataTableManager(trans.app)
|
||||
stdtm = ShedToolDataTableManager(trans.app)
|
||||
ok = True
|
||||
isgzip = False
|
||||
isbz2 = False
|
||||
@@ -223,7 +223,7 @@ class UploadController(BaseUIController):
|
||||
# Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded
|
||||
# by parsing the file and adding new entries to the in-memory trans.app.tool_data_tables
|
||||
# dictionary.
|
||||
error, error_message = tdtm.handle_sample_tool_data_table_conf_file(full_path, persist=False)
|
||||
error, error_message = stdtm.handle_sample_tool_data_table_conf_file(full_path, persist=False)
|
||||
if error:
|
||||
message = '%s<br/>%s' % (message, error_message)
|
||||
# See if the content of the change set was valid.
|
||||
@@ -312,7 +312,7 @@ class UploadController(BaseUIController):
|
||||
message += invalid_repository_dependencies_message
|
||||
status = 'error'
|
||||
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
|
||||
tdtm.reset_tool_data_tables()
|
||||
stdtm.reset_tool_data_tables()
|
||||
if uploaded_directory:
|
||||
basic_util.remove_dir(uploaded_directory)
|
||||
trans.response.send_redirect(web.url_for(controller='repository',
|
||||
@@ -326,7 +326,7 @@ class UploadController(BaseUIController):
|
||||
basic_util.remove_dir(uploaded_directory)
|
||||
status = 'error'
|
||||
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
|
||||
tdtm.reset_tool_data_tables()
|
||||
stdtm.reset_tool_data_tables()
|
||||
return trans.fill_template('/webapps/tool_shed/repository/upload.mako',
|
||||
repository=repository,
|
||||
changeset_revision=tip,
|
||||
|
||||
@@ -19,7 +19,7 @@ from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment impor
|
||||
from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager
|
||||
from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager
|
||||
from tool_shed.galaxy_install.tools import data_manager, tool_panel_manager
|
||||
from tool_shed.tools import data_table_manager
|
||||
from tool_shed.tools.data_table_manager import ShedToolDataTableManager
|
||||
from tool_shed.util import basic_util, encoding_util, hg_util, repository_util
|
||||
from tool_shed.util import shed_util_common as suc, tool_dependency_util
|
||||
from tool_shed.util import tool_util, xml_util
|
||||
@@ -499,7 +499,7 @@ class InstallRepositoryManager(object):
|
||||
reinstalling an uninstalled repository.
|
||||
"""
|
||||
shed_config_dict = self.app.toolbox.get_shed_config_dict_by_filename(shed_tool_conf)
|
||||
tdtm = data_table_manager.ToolDataTableManager(self.app)
|
||||
stdtm = ShedToolDataTableManager(self.app)
|
||||
irmm = InstalledRepositoryMetadataManager(app=self.app,
|
||||
tpm=self.tpm,
|
||||
repository=tool_shed_repository,
|
||||
@@ -527,9 +527,9 @@ class InstallRepositoryManager(object):
|
||||
set_status=True)
|
||||
if 'sample_files' in irmm_metadata_dict:
|
||||
sample_files = irmm_metadata_dict.get('sample_files', [])
|
||||
tool_index_sample_files = tdtm.get_tool_index_sample_files(sample_files)
|
||||
tool_index_sample_files = stdtm.get_tool_index_sample_files(sample_files)
|
||||
tool_data_table_conf_filename, tool_data_table_elems = \
|
||||
tdtm.install_tool_data_tables(tool_shed_repository, tool_index_sample_files)
|
||||
stdtm.install_tool_data_tables(tool_shed_repository, tool_index_sample_files)
|
||||
if tool_data_table_elems:
|
||||
self.app.tool_data_tables.add_new_entries_from_config_file(tool_data_table_conf_filename,
|
||||
None,
|
||||
@@ -538,13 +538,13 @@ class InstallRepositoryManager(object):
|
||||
if 'tools' in irmm_metadata_dict:
|
||||
tool_panel_dict = self.tpm.generate_tool_panel_dict_for_new_install(irmm_metadata_dict['tools'], tool_section)
|
||||
sample_files = irmm_metadata_dict.get('sample_files', [])
|
||||
tool_index_sample_files = tdtm.get_tool_index_sample_files(sample_files)
|
||||
tool_index_sample_files = stdtm.get_tool_index_sample_files(sample_files)
|
||||
tool_util.copy_sample_files(self.app, tool_index_sample_files, tool_path=tool_path)
|
||||
sample_files_copied = [str(s) for s in tool_index_sample_files]
|
||||
repository_tools_tups = irmm.get_repository_tools_tups()
|
||||
if repository_tools_tups:
|
||||
# Handle missing data table entries for tool parameters that are dynamically generated select lists.
|
||||
repository_tools_tups = tdtm.handle_missing_data_table_entry(relative_install_dir,
|
||||
repository_tools_tups = stdtm.handle_missing_data_table_entry(relative_install_dir,
|
||||
tool_path,
|
||||
repository_tools_tups)
|
||||
# Handle missing index files for tool parameters that are dynamically generated select lists.
|
||||
|
||||
@@ -9,6 +9,7 @@ from galaxy import util
|
||||
from galaxy.tools.data_manager.manager import DataManager
|
||||
from galaxy.tools.loader_directory import looks_like_a_tool
|
||||
from galaxy.tools.parser.interface import TestCollectionDef
|
||||
from galaxy.tools.repositories import ValidationContext
|
||||
from galaxy.web import url_for
|
||||
from tool_shed.repository_types import util as rt_util
|
||||
from tool_shed.tools import tool_validator
|
||||
@@ -304,7 +305,6 @@ class MetadataGenerator(object):
|
||||
tool_data_table_conf.xml.sample file, in which case the entries should ultimately be
|
||||
persisted to the file referred to by self.app.config.shed_tool_data_table_config.
|
||||
"""
|
||||
tv = tool_validator.ToolValidator(self.app)
|
||||
if self.shed_config_dict is None:
|
||||
self.shed_config_dict = {}
|
||||
if self.updating_installed_repository:
|
||||
@@ -339,120 +339,125 @@ class MetadataGenerator(object):
|
||||
files_dir = self.relative_install_dir
|
||||
if self.shed_config_dict.get('tool_path'):
|
||||
files_dir = os.path.join(self.shed_config_dict['tool_path'], files_dir)
|
||||
# Handle proprietary datatypes, if any.
|
||||
datatypes_config = hg_util.get_config_from_disk(suc.DATATYPES_CONFIG_FILENAME, files_dir)
|
||||
if datatypes_config:
|
||||
metadata_dict = self.generate_datatypes_metadata(tv,
|
||||
files_dir,
|
||||
datatypes_config,
|
||||
metadata_dict)
|
||||
# Get the relative path to all sample files included in the repository for storage in
|
||||
# the repository's metadata.
|
||||
sample_file_metadata_paths, sample_file_copy_paths = \
|
||||
self.get_sample_files_from_disk(repository_files_dir=files_dir,
|
||||
tool_path=self.shed_config_dict.get('tool_path'),
|
||||
relative_install_dir=self.relative_install_dir)
|
||||
if sample_file_metadata_paths:
|
||||
metadata_dict['sample_files'] = sample_file_metadata_paths
|
||||
# Copy all sample files included in the repository to a single directory location so we
|
||||
# can load tools that depend on them.
|
||||
for sample_file in sample_file_copy_paths:
|
||||
tool_util.copy_sample_file(self.app, sample_file, dest_path=work_dir)
|
||||
# If the list of sample files includes a tool_data_table_conf.xml.sample file, load
|
||||
# its table elements into memory.
|
||||
relative_path, filename = os.path.split(sample_file)
|
||||
if filename == 'tool_data_table_conf.xml.sample':
|
||||
new_table_elems, error_message = \
|
||||
self.app.tool_data_tables.add_new_entries_from_config_file(config_filename=sample_file,
|
||||
tool_data_path=work_dir,
|
||||
shed_tool_data_table_config=work_dir,
|
||||
persist=False)
|
||||
if error_message:
|
||||
self.invalid_file_tups.append((filename, error_message))
|
||||
for root, dirs, files in os.walk(files_dir):
|
||||
if root.find('.hg') < 0 and root.find('hgrc') < 0:
|
||||
if '.hg' in dirs:
|
||||
dirs.remove('.hg')
|
||||
for name in files:
|
||||
# See if we have a repository dependencies defined.
|
||||
if name == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
|
||||
path_to_repository_dependencies_config = os.path.join(root, name)
|
||||
metadata_dict, error_message = \
|
||||
self.generate_repository_dependency_metadata(path_to_repository_dependencies_config,
|
||||
metadata_dict)
|
||||
if error_message:
|
||||
self.invalid_file_tups.append((name, error_message))
|
||||
# See if we have one or more READ_ME files.
|
||||
elif name.lower() in readme_file_names:
|
||||
relative_path_to_readme = self.get_relative_path_to_repository_file(root,
|
||||
name,
|
||||
self.relative_install_dir,
|
||||
work_dir,
|
||||
self.shed_config_dict)
|
||||
readme_files.append(relative_path_to_readme)
|
||||
# See if we have a tool config.
|
||||
elif looks_like_a_tool(os.path.join(root, name), invalid_names=self.NOT_TOOL_CONFIGS):
|
||||
full_path = str(os.path.abspath(os.path.join(root, name))) # why the str, seems very odd
|
||||
element_tree, error_message = xml_util.parse_xml(full_path)
|
||||
if element_tree is None:
|
||||
is_tool = False
|
||||
else:
|
||||
element_tree_root = element_tree.getroot()
|
||||
is_tool = element_tree_root.tag == 'tool'
|
||||
if is_tool:
|
||||
tool, valid, error_message = \
|
||||
tv.load_tool_from_config(self.app.security.encode_id(self.repository.id),
|
||||
full_path)
|
||||
if tool is None:
|
||||
if not valid:
|
||||
invalid_tool_configs.append(name)
|
||||
self.invalid_file_tups.append((name, error_message))
|
||||
# Create ValidationContext to load and validate tools, data tables and datatypes
|
||||
with ValidationContext.from_app(app=self.app, work_dir=work_dir) as validation_context:
|
||||
tv = tool_validator.ToolValidator(validation_context)
|
||||
# Handle proprietary datatypes, if any.
|
||||
datatypes_config = hg_util.get_config_from_disk(suc.DATATYPES_CONFIG_FILENAME, files_dir)
|
||||
if datatypes_config:
|
||||
metadata_dict = self.generate_datatypes_metadata(tv,
|
||||
files_dir,
|
||||
datatypes_config,
|
||||
metadata_dict)
|
||||
# Get the relative path to all sample files included in the repository for storage in
|
||||
# the repository's metadata.
|
||||
sample_file_metadata_paths, sample_file_copy_paths = \
|
||||
self.get_sample_files_from_disk(repository_files_dir=files_dir,
|
||||
tool_path=self.shed_config_dict.get('tool_path'),
|
||||
relative_install_dir=self.relative_install_dir)
|
||||
if sample_file_metadata_paths:
|
||||
metadata_dict['sample_files'] = sample_file_metadata_paths
|
||||
# Copy all sample files included in the repository to a single directory location so we
|
||||
# can load tools that depend on them.
|
||||
for sample_file in sample_file_copy_paths:
|
||||
tool_util.copy_sample_file(self.app, sample_file, dest_path=work_dir)
|
||||
# If the list of sample files includes a tool_data_table_conf.xml.sample file, load
|
||||
# its table elements into memory.
|
||||
relative_path, filename = os.path.split(sample_file)
|
||||
if filename == 'tool_data_table_conf.xml.sample':
|
||||
# We create a new ToolDataTableManager to avoid adding entries to the app-wide
|
||||
# tool data tables. This is only used for checking that the data table is valid.
|
||||
new_table_elems, error_message = \
|
||||
validation_context.tool_data_tables.add_new_entries_from_config_file(config_filename=sample_file,
|
||||
tool_data_path=work_dir,
|
||||
shed_tool_data_table_config=work_dir,
|
||||
persist=False)
|
||||
if error_message:
|
||||
self.invalid_file_tups.append((filename, error_message))
|
||||
for root, dirs, files in os.walk(files_dir):
|
||||
if root.find('.hg') < 0 and root.find('hgrc') < 0:
|
||||
if '.hg' in dirs:
|
||||
dirs.remove('.hg')
|
||||
for name in files:
|
||||
# See if we have a repository dependencies defined.
|
||||
if name == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
|
||||
path_to_repository_dependencies_config = os.path.join(root, name)
|
||||
metadata_dict, error_message = \
|
||||
self.generate_repository_dependency_metadata(path_to_repository_dependencies_config,
|
||||
metadata_dict)
|
||||
if error_message:
|
||||
self.invalid_file_tups.append((name, error_message))
|
||||
# See if we have one or more READ_ME files.
|
||||
elif name.lower() in readme_file_names:
|
||||
relative_path_to_readme = self.get_relative_path_to_repository_file(root,
|
||||
name,
|
||||
self.relative_install_dir,
|
||||
work_dir,
|
||||
self.shed_config_dict)
|
||||
readme_files.append(relative_path_to_readme)
|
||||
# See if we have a tool config.
|
||||
elif looks_like_a_tool(os.path.join(root, name), invalid_names=self.NOT_TOOL_CONFIGS):
|
||||
full_path = str(os.path.abspath(os.path.join(root, name))) # why the str, seems very odd
|
||||
element_tree, error_message = xml_util.parse_xml(full_path)
|
||||
if element_tree is None:
|
||||
is_tool = False
|
||||
else:
|
||||
invalid_files_and_errors_tups = \
|
||||
tv.check_tool_input_params(files_dir,
|
||||
name,
|
||||
tool,
|
||||
sample_file_copy_paths)
|
||||
can_set_metadata = True
|
||||
for tup in invalid_files_and_errors_tups:
|
||||
if name in tup:
|
||||
can_set_metadata = False
|
||||
element_tree_root = element_tree.getroot()
|
||||
is_tool = element_tree_root.tag == 'tool'
|
||||
if is_tool:
|
||||
tool, valid, error_message = \
|
||||
tv.load_tool_from_config(self.app.security.encode_id(self.repository.id),
|
||||
full_path)
|
||||
if tool is None:
|
||||
if not valid:
|
||||
invalid_tool_configs.append(name)
|
||||
break
|
||||
if can_set_metadata:
|
||||
relative_path_to_tool_config = \
|
||||
self.get_relative_path_to_repository_file(root,
|
||||
name,
|
||||
self.relative_install_dir,
|
||||
work_dir,
|
||||
self.shed_config_dict)
|
||||
metadata_dict = self.generate_tool_metadata(relative_path_to_tool_config,
|
||||
tool,
|
||||
metadata_dict)
|
||||
self.invalid_file_tups.append((name, error_message))
|
||||
else:
|
||||
invalid_files_and_errors_tups = \
|
||||
tv.check_tool_input_params(files_dir,
|
||||
name,
|
||||
tool,
|
||||
sample_file_copy_paths)
|
||||
can_set_metadata = True
|
||||
for tup in invalid_files_and_errors_tups:
|
||||
self.invalid_file_tups.append(tup)
|
||||
# Find all exported workflows.
|
||||
elif name.endswith('.ga'):
|
||||
relative_path = os.path.join(root, name)
|
||||
if os.path.getsize(os.path.abspath(relative_path)) > 0:
|
||||
fp = open(relative_path, 'rb')
|
||||
workflow_text = fp.read()
|
||||
fp.close()
|
||||
if workflow_text:
|
||||
valid_exported_galaxy_workflow = True
|
||||
try:
|
||||
exported_workflow_dict = json.loads(workflow_text)
|
||||
except Exception:
|
||||
log.exception("Skipping file %s since it does not seem to be a valid exported Galaxy workflow",
|
||||
str(relative_path))
|
||||
valid_exported_galaxy_workflow = False
|
||||
if valid_exported_galaxy_workflow and \
|
||||
'a_galaxy_workflow' in exported_workflow_dict and \
|
||||
exported_workflow_dict['a_galaxy_workflow'] == 'true':
|
||||
metadata_dict = self.generate_workflow_metadata(relative_path,
|
||||
exported_workflow_dict,
|
||||
metadata_dict)
|
||||
if name in tup:
|
||||
can_set_metadata = False
|
||||
invalid_tool_configs.append(name)
|
||||
break
|
||||
if can_set_metadata:
|
||||
relative_path_to_tool_config = \
|
||||
self.get_relative_path_to_repository_file(root,
|
||||
name,
|
||||
self.relative_install_dir,
|
||||
work_dir,
|
||||
self.shed_config_dict)
|
||||
metadata_dict = self.generate_tool_metadata(relative_path_to_tool_config,
|
||||
tool,
|
||||
metadata_dict)
|
||||
else:
|
||||
for tup in invalid_files_and_errors_tups:
|
||||
self.invalid_file_tups.append(tup)
|
||||
# Find all exported workflows.
|
||||
elif name.endswith('.ga'):
|
||||
relative_path = os.path.join(root, name)
|
||||
if os.path.getsize(os.path.abspath(relative_path)) > 0:
|
||||
fp = open(relative_path, 'rb')
|
||||
workflow_text = fp.read()
|
||||
fp.close()
|
||||
if workflow_text:
|
||||
valid_exported_galaxy_workflow = True
|
||||
try:
|
||||
exported_workflow_dict = json.loads(workflow_text)
|
||||
except Exception:
|
||||
log.exception("Skipping file %s since it does not seem to be a valid exported Galaxy workflow",
|
||||
str(relative_path))
|
||||
valid_exported_galaxy_workflow = False
|
||||
if valid_exported_galaxy_workflow and \
|
||||
'a_galaxy_workflow' in exported_workflow_dict and \
|
||||
exported_workflow_dict['a_galaxy_workflow'] == 'true':
|
||||
metadata_dict = self.generate_workflow_metadata(relative_path,
|
||||
exported_workflow_dict,
|
||||
metadata_dict)
|
||||
# Handle any data manager entries
|
||||
data_manager_config = hg_util.get_config_from_disk(suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir)
|
||||
metadata_dict = self.generate_data_manager_metadata(files_dir,
|
||||
@@ -567,7 +572,6 @@ class MetadataGenerator(object):
|
||||
Generate a repository dependencies dictionary based on valid information defined in the received
|
||||
repository_dependencies_config. This method is called from the tool shed as well as from Galaxy.
|
||||
"""
|
||||
error_message = ''
|
||||
# Make sure we're looking at a valid repository_dependencies.xml file.
|
||||
tree, error_message = xml_util.parse_xml(repository_dependencies_config)
|
||||
if tree is None:
|
||||
@@ -817,6 +821,7 @@ class MetadataGenerator(object):
|
||||
return relative_path_to_file
|
||||
|
||||
def get_sample_files_from_disk(self, repository_files_dir, tool_path=None, relative_install_dir=None):
|
||||
work_dir = ''
|
||||
if self.resetting_all_metadata_on_repository:
|
||||
# Keep track of the location where the repository is temporarily cloned so that we can strip
|
||||
# it when setting metadata.
|
||||
@@ -888,8 +893,6 @@ class MetadataGenerator(object):
|
||||
# Default to the current tool shed.
|
||||
toolshed = str(url_for('/', qualified=True)).rstrip('/')
|
||||
repository_dependency_tup[0] = toolshed
|
||||
user = None
|
||||
repository = None
|
||||
toolshed = common_util.remove_protocol_from_tool_shed_url(toolshed)
|
||||
if self.app.name == 'galaxy':
|
||||
# We're in Galaxy. We reach here when we're generating the metadata for a tool
|
||||
@@ -1087,15 +1090,6 @@ class MetadataGenerator(object):
|
||||
else:
|
||||
repository_dependencies_dict = metadata.get('invalid_repository_dependencies', None)
|
||||
for repository_dependency_tup in repository_dependency_tups:
|
||||
if is_valid:
|
||||
tool_shed, name, owner, changeset_revision, \
|
||||
prior_installation_required, \
|
||||
only_if_compiling_contained_td = repository_dependency_tup
|
||||
else:
|
||||
tool_shed, name, owner, changeset_revision, \
|
||||
prior_installation_required, \
|
||||
only_if_compiling_contained_td, error_message = \
|
||||
repository_dependency_tup
|
||||
if repository_dependencies_dict:
|
||||
repository_dependencies = repository_dependencies_dict.get('repository_dependencies', [])
|
||||
for repository_dependency_tup in repository_dependency_tups:
|
||||
|
||||
@@ -809,8 +809,6 @@ class RepositoryMetadataManager(metadata_generator.MetadataGenerator):
|
||||
# Set tool version information for all downloadable changeset revisions. Get the list of changeset
|
||||
# revisions from the changelog.
|
||||
self.reset_all_tool_versions(repo)
|
||||
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
|
||||
self.app.tool_data_tables.data_tables = {}
|
||||
|
||||
def reset_all_tool_versions(self, repo):
|
||||
"""Reset tool version lineage for those changeset revisions that include valid tools."""
|
||||
@@ -989,8 +987,6 @@ class RepositoryMetadataManager(metadata_generator.MetadataGenerator):
|
||||
self.repository,
|
||||
self.metadata_dict)
|
||||
status = 'error'
|
||||
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
|
||||
self.app.tool_data_tables.data_tables = {}
|
||||
return message, status
|
||||
|
||||
def set_repository_metadata_due_to_new_tip(self, host, content_alert_str=None, **kwd):
|
||||
|
||||
@@ -8,7 +8,7 @@ from tool_shed.util import hg_util, xml_util
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ToolDataTableManager(object):
|
||||
class ShedToolDataTableManager(object):
|
||||
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
@@ -81,9 +81,6 @@ class ToolDataTableManager(object):
|
||||
if error:
|
||||
# TODO: Do more here than logging an exception.
|
||||
log.debug(message)
|
||||
# Reload the tool into the local list of repository_tools_tups.
|
||||
repository_tool = self.app.toolbox.load_tool(os.path.join(tool_path, tup_path), guid=guid, use_cached=False)
|
||||
repository_tools_tups[index] = (tup_path, guid, repository_tool)
|
||||
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
|
||||
self.reset_tool_data_tables()
|
||||
return repository_tools_tups
|
||||
@@ -97,7 +94,6 @@ class ToolDataTableManager(object):
|
||||
file on disk.
|
||||
"""
|
||||
error = False
|
||||
message = ''
|
||||
try:
|
||||
new_table_elems, message = self.app.tool_data_tables \
|
||||
.add_new_entries_from_config_file(config_filename=filename,
|
||||
|
||||
@@ -4,11 +4,14 @@ import os
|
||||
import tempfile
|
||||
|
||||
from galaxy.tools import (
|
||||
create_tool_from_source,
|
||||
get_tool_source,
|
||||
parameters,
|
||||
Tool
|
||||
)
|
||||
from galaxy.tools.fetcher import ToolLocationFetcher
|
||||
from galaxy.tools.parameters import dynamic_options
|
||||
from tool_shed.tools import data_table_manager
|
||||
from tool_shed.tools.data_table_manager import ShedToolDataTableManager
|
||||
from tool_shed.util import (
|
||||
basic_util,
|
||||
hg_util,
|
||||
@@ -24,7 +27,7 @@ class ToolValidator(object):
|
||||
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
self.tdtm = data_table_manager.ToolDataTableManager(self.app)
|
||||
self.stdtm = ShedToolDataTableManager(self.app)
|
||||
|
||||
def can_use_tool_config_disk_file(self, repository, repo, file_path, changeset_revision):
|
||||
"""
|
||||
@@ -55,7 +58,6 @@ class ToolValidator(object):
|
||||
generated using external data files to make sure the files exist.
|
||||
"""
|
||||
invalid_files_and_errors_tups = []
|
||||
correction_msg = ''
|
||||
for input_param in tool.input_params:
|
||||
if isinstance(input_param, parameters.basic.SelectToolParameter) and input_param.is_dynamic:
|
||||
# If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml,
|
||||
@@ -67,7 +69,7 @@ class ToolValidator(object):
|
||||
sample_tool_data_table_conf = hg_util.get_config_from_disk('tool_data_table_conf.xml.sample', repo_dir)
|
||||
if sample_tool_data_table_conf:
|
||||
error, correction_msg = \
|
||||
self.tdtm.handle_sample_tool_data_table_conf_file(sample_tool_data_table_conf,
|
||||
self.stdtm.handle_sample_tool_data_table_conf_file(sample_tool_data_table_conf,
|
||||
persist=False)
|
||||
if error:
|
||||
invalid_files_and_errors_tups.append(('tool_data_table_conf.xml.sample', correction_msg))
|
||||
@@ -78,9 +80,9 @@ class ToolValidator(object):
|
||||
invalid_tup = (tool_config_name, correction_msg)
|
||||
if invalid_tup not in invalid_files_and_errors_tups:
|
||||
invalid_files_and_errors_tups.append(invalid_tup)
|
||||
if options.index_file or options.missing_index_file:
|
||||
if options.index_file or options.tool_data_table and options.tool_data_table.missing_index_file:
|
||||
# Make sure the repository contains the required xxx.loc.sample file.
|
||||
index_file = options.index_file or options.missing_index_file
|
||||
index_file = options.index_file or options.tool_data_table.missing_index_file
|
||||
index_file_name = basic_util.strip_path(index_file)
|
||||
sample_found = False
|
||||
for sample_file in sample_files:
|
||||
@@ -200,11 +202,12 @@ class ToolValidator(object):
|
||||
"""
|
||||
message = ''
|
||||
sample_files = self.copy_disk_sample_files_to_dir(repo_files_dir, work_dir)
|
||||
tool_data_table_config = None
|
||||
if sample_files:
|
||||
if 'tool_data_table_conf.xml.sample' in sample_files:
|
||||
# Load entries into the tool_data_tables if the tool requires them.
|
||||
tool_data_table_config = os.path.join(work_dir, 'tool_data_table_conf.xml')
|
||||
error, message = self.tdtm.handle_sample_tool_data_table_conf_file(tool_data_table_config,
|
||||
error, message = self.stdtm.handle_sample_tool_data_table_conf_file(tool_data_table_config,
|
||||
persist=False)
|
||||
tool, valid, message2 = self.load_tool_from_config(repository_id, tool_config_filepath)
|
||||
message = self.concat_messages(message, message2)
|
||||
@@ -219,12 +222,11 @@ class ToolValidator(object):
|
||||
# intended to help handle sample files that are in the manifest, but have been deleted from disk.
|
||||
sample_files, deleted_sample_files = self.get_list_of_copied_sample_files(repo, ctx, dir=work_dir)
|
||||
if sample_files:
|
||||
self.app.config.tool_data_path = work_dir
|
||||
if 'tool_data_table_conf.xml.sample' in sample_files:
|
||||
# Load entries into the tool_data_tables if the tool requires them.
|
||||
tool_data_table_config = os.path.join(work_dir, 'tool_data_table_conf.xml')
|
||||
if tool_data_table_config:
|
||||
error, message = self.tdtm.handle_sample_tool_data_table_conf_file(tool_data_table_config,
|
||||
error, message = self.stdtm.handle_sample_tool_data_table_conf_file(tool_data_table_config,
|
||||
persist=False)
|
||||
if error:
|
||||
log.debug(message)
|
||||
@@ -242,13 +244,9 @@ class ToolValidator(object):
|
||||
the received valid changeset revision and the first changeset revision in the repository,
|
||||
searching backwards.
|
||||
"""
|
||||
original_tool_data_path = self.app.config.tool_data_path
|
||||
repository = repository_util.get_repository_in_tool_shed(self.app, repository_id)
|
||||
repo_files_dir = repository.repo_path(self.app)
|
||||
repo = hg_util.get_repo_for_repository(self.app, repository=None, repo_path=repo_files_dir, create=False)
|
||||
message = ''
|
||||
tool = None
|
||||
can_use_disk_file = False
|
||||
tool_config_filepath = repository_util.get_absolute_path_to_file_in_repository(repo_files_dir, tool_config_filename)
|
||||
work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-ltfcr")
|
||||
can_use_disk_file = self.can_use_tool_config_disk_file(repository,
|
||||
@@ -256,7 +254,6 @@ class ToolValidator(object):
|
||||
tool_config_filepath,
|
||||
changeset_revision)
|
||||
if can_use_disk_file:
|
||||
self.app.config.tool_data_path = work_dir
|
||||
tool, valid, message, sample_files = \
|
||||
self.handle_sample_files_and_load_tool_from_disk(repo_files_dir,
|
||||
repository_id,
|
||||
@@ -284,14 +281,18 @@ class ToolValidator(object):
|
||||
tool_config_filename,
|
||||
work_dir)
|
||||
basic_util.remove_dir(work_dir)
|
||||
self.app.config.tool_data_path = original_tool_data_path
|
||||
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
|
||||
self.tdtm.reset_tool_data_tables()
|
||||
self.stdtm.reset_tool_data_tables()
|
||||
return repository, tool, message
|
||||
|
||||
def load_tool_from_config(self, repository_id, full_path):
|
||||
tool_source = get_tool_source(
|
||||
full_path,
|
||||
enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False),
|
||||
tool_location_fetcher=ToolLocationFetcher(),
|
||||
)
|
||||
try:
|
||||
tool = self.app.toolbox.load_tool(full_path, repository_id=repository_id, allow_code_files=False, use_cached=False)
|
||||
tool = create_tool_from_source(config_file=full_path, app=self.app, tool_source=tool_source, repository_id=repository_id)
|
||||
valid = True
|
||||
error_message = None
|
||||
except KeyError as e:
|
||||
|
||||
@@ -12,7 +12,7 @@ from sqlalchemy.sql.expression import null
|
||||
import tool_shed.repository_types.util as rt_util
|
||||
from galaxy.util import checkers
|
||||
from galaxy.util.path import safe_relpath
|
||||
from tool_shed.tools import data_table_manager
|
||||
from tool_shed.tools.data_table_manager import ShedToolDataTableManager
|
||||
from tool_shed.util import basic_util, hg_util, shed_util_common as suc
|
||||
|
||||
if sys.version_info < (3, 3):
|
||||
@@ -214,8 +214,8 @@ def handle_directory_changes(app, host, username, repository, full_path, filenam
|
||||
# Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded
|
||||
# by parsing the file and adding new entries to the in-memory app.tool_data_tables
|
||||
# dictionary.
|
||||
tdtm = data_table_manager.ToolDataTableManager(app)
|
||||
error, message = tdtm.handle_sample_tool_data_table_conf_file(filename_in_archive, persist=False)
|
||||
stdtm = ShedToolDataTableManager(app)
|
||||
error, message = stdtm.handle_sample_tool_data_table_conf_file(filename_in_archive, persist=False)
|
||||
if error:
|
||||
return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed
|
||||
hg_util.commit_changeset(repo.ui,
|
||||
|
||||
@@ -41,13 +41,13 @@ def build_tool_panel_section_select_field(app):
|
||||
|
||||
def copy_sample_file(app, filename, dest_path=None):
|
||||
"""
|
||||
Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx. The default value for dest_path
|
||||
is ~/tool-data.
|
||||
Copies a sample file at `filename` to `the dest_path`
|
||||
directory and strips the '.sample' extensions from `filename`.
|
||||
"""
|
||||
if dest_path is None:
|
||||
dest_path = os.path.abspath(app.config.tool_data_path)
|
||||
sample_file_name = basic_util.strip_path(filename)
|
||||
copied_file = sample_file_name.replace('.sample', '')
|
||||
copied_file = sample_file_name.rsplit('.sample', 1)[0]
|
||||
full_source_path = os.path.abspath(filename)
|
||||
full_destination_path = os.path.join(dest_path, sample_file_name)
|
||||
# Don't copy a file to itself - not sure how this happens, but sometimes it does...
|
||||
@@ -159,14 +159,11 @@ def handle_missing_index_file(app, tool_path, sample_files, repository_tools_tup
|
||||
for sample_file in sample_files:
|
||||
sample_file_name = basic_util.strip_path(sample_file)
|
||||
if sample_file_name == '%s.sample' % missing_file_name:
|
||||
copy_sample_file(app, sample_file)
|
||||
copy_sample_file(app, os.path.join(tool_path, sample_file))
|
||||
if options.tool_data_table and options.tool_data_table.missing_index_file:
|
||||
options.tool_data_table.handle_found_index_file(options.missing_index_file)
|
||||
sample_files_copied.append(options.missing_index_file)
|
||||
break
|
||||
# Reload the tool into the local list of repository_tools_tups.
|
||||
repository_tool = app.toolbox.load_tool(os.path.join(tool_path, tup_path), guid=guid, use_cached=False)
|
||||
repository_tools_tups[index] = (tup_path, guid, repository_tool)
|
||||
return repository_tools_tups, sample_files_copied
|
||||
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import os
|
||||
|
||||
import galaxy.tools
|
||||
import galaxy.tools.parameters
|
||||
from galaxy.tools.repositories import ValidationContext
|
||||
from galaxy.util.sanitize_html import sanitize_html
|
||||
from galaxy.workflow.modules import (
|
||||
module_types,
|
||||
@@ -36,17 +37,18 @@ class RepoToolModule(ToolModule):
|
||||
self.tool = None
|
||||
self.errors = None
|
||||
self.tool_version = None
|
||||
self.tv = tool_validator.ToolValidator(trans.app)
|
||||
if trans.webapp.name == 'tool_shed':
|
||||
# We're in the tool shed.
|
||||
for tool_dict in tools_metadata:
|
||||
if self.tool_id in [tool_dict['id'], tool_dict['guid']]:
|
||||
repository, self.tool, message = self.tv.load_tool_from_changeset_revision(repository_id,
|
||||
changeset_revision,
|
||||
tool_dict['tool_config'])
|
||||
if message and self.tool is None:
|
||||
self.errors = 'unavailable'
|
||||
break
|
||||
with ValidationContext.from_app(trans.app) as validation_context:
|
||||
tv = tool_validator.ToolValidator(validation_context)
|
||||
for tool_dict in tools_metadata:
|
||||
if self.tool_id in [tool_dict['id'], tool_dict['guid']]:
|
||||
repository, self.tool, message = tv.load_tool_from_changeset_revision(repository_id,
|
||||
changeset_revision,
|
||||
tool_dict['tool_config'])
|
||||
if message and self.tool is None:
|
||||
self.errors = 'unavailable'
|
||||
break
|
||||
else:
|
||||
# We're in Galaxy.
|
||||
self.tool = trans.app.toolbox.get_tool(self.tool_id, tool_version=self.tool_version)
|
||||
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
# bowtie2_indices.loc.sample
|
||||
# This is a *.loc.sample file distributed with Galaxy that enables tools
|
||||
# to use a directory of indexed data files. This one is for Bowtie2 and Tophat2.
|
||||
# See the wiki: http://wiki.galaxyproject.org/Admin/NGS%20Local%20Setup
|
||||
# First create these data files and save them in your own data directory structure.
|
||||
# Then, create a bowtie_indices.loc file to use those indexes with tools.
|
||||
# Copy this file, save it with the same name (minus the .sample),
|
||||
# follow the format examples, and store the result in this directory.
|
||||
# The file should include an one line entry for each index set.
|
||||
# The path points to the "basename" for the set, not a specific file.
|
||||
# It has four text columns seperated by TABS.
|
||||
#
|
||||
# <unique_build_id> <dbkey> <display_name> <file_base_path>
|
||||
#
|
||||
# So, for example, if you had hg18 indexes stored in:
|
||||
#
|
||||
# /depot/data2/galaxy/hg19/bowtie2/
|
||||
#
|
||||
# containing hg19 genome and hg19.*.bt2 files, such as:
|
||||
# -rw-rw-r-- 1 james james 914M Feb 10 18:56 hg19canon.fa
|
||||
# -rw-rw-r-- 1 james james 914M Feb 10 18:56 hg19canon.1.bt2
|
||||
# -rw-rw-r-- 1 james james 683M Feb 10 18:56 hg19canon.2.bt2
|
||||
# -rw-rw-r-- 1 james james 3.3K Feb 10 16:54 hg19canon.3.bt2
|
||||
# -rw-rw-r-- 1 james james 683M Feb 10 16:54 hg19canon.4.bt2
|
||||
# -rw-rw-r-- 1 james james 914M Feb 10 20:45 hg19canon.rev.1.bt2
|
||||
# -rw-rw-r-- 1 james james 683M Feb 10 20:45 hg19canon.rev.2.bt2
|
||||
#
|
||||
# then the bowtie2_indices.loc entry could look like this:
|
||||
#
|
||||
#hg19 hg19 Human (hg19) /depot/data2/galaxy/hg19/bowtie2/hg19canon
|
||||
#
|
||||
#More examples:
|
||||
#
|
||||
#mm10 mm10 Mouse (mm10) /depot/data2/galaxy/mm10/bowtie2/mm10
|
||||
#dm3 dm3 D. melanogaster (dm3) /depot/data2/galaxy/mm10/bowtie2/dm3
|
||||
#
|
||||
#
|
||||
@@ -0,0 +1,74 @@
|
||||
import os
|
||||
import shutil
|
||||
import tarfile
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
|
||||
from galaxy.tools.repositories import ValidationContext
|
||||
from tool_shed.tools.tool_validator import ToolValidator
|
||||
from ..unittest_utils.galaxy_mock import MockApp
|
||||
|
||||
CURRENT_DIR = os.path.dirname(__file__)
|
||||
BISMARK_TAR = os.path.join(CURRENT_DIR,
|
||||
'../../shed_functional/test_data/bismark/bismark.tar')
|
||||
BOWTIE2_INDICES = os.path.join(CURRENT_DIR,
|
||||
'../../shed_functional/test_data/bowtie2_loc_sample/bowtie2_indices.loc.sample')
|
||||
|
||||
|
||||
def test_validate_valid_tool():
|
||||
with get_tool_validator() as tv, setup_bismark() as repo_dir:
|
||||
full_path = os.path.join(repo_dir, 'bismark_methylation_extractor.xml')
|
||||
tool, valid, message = tv.load_tool_from_config(repository_id=None,
|
||||
full_path=full_path)
|
||||
assert tool.name == 'Bismark'
|
||||
assert not tool.params_with_missing_data_table_entry
|
||||
assert not tool.params_with_missing_index_file
|
||||
assert valid is True
|
||||
assert message is None
|
||||
|
||||
|
||||
def test_validate_tool_without_index():
|
||||
with get_tool_validator() as tv, setup_bismark() as repo_dir:
|
||||
full_path = os.path.join(repo_dir, 'bismark_bowtie2_wrapper.xml')
|
||||
tool, valid, message = tv.load_tool_from_config(repository_id=None,
|
||||
full_path=full_path)
|
||||
assert valid is True
|
||||
# tool is missing a data table
|
||||
assert tool.params_with_missing_data_table_entry
|
||||
# check input parameters, which will set up the missing data table (if present in repo_dir)
|
||||
invalid_files_and_errors_tups = tv.check_tool_input_params(repo_dir=os.path.dirname(full_path),
|
||||
tool_config_name=full_path,
|
||||
tool=tool,
|
||||
sample_files=[])
|
||||
# tool is invalid because loc file is not present, but data table is present now
|
||||
assert len(invalid_files_and_errors_tups) == 1
|
||||
assert invalid_files_and_errors_tups[0][1].startswith('This file refers to a file named')
|
||||
assert not tool.params_with_missing_data_table_entry
|
||||
assert tool.params_with_missing_index_file
|
||||
# copy a loc file
|
||||
shutil.copy(BOWTIE2_INDICES, repo_dir)
|
||||
tool, valid, message, sample_files = tv.handle_sample_files_and_load_tool_from_disk(repo_dir, None, full_path, repo_dir)
|
||||
invalid_files_and_errors_tups = tv.check_tool_input_params(repo_dir=repo_dir,
|
||||
tool_config_name=full_path,
|
||||
tool=tool,
|
||||
sample_files=sample_files)
|
||||
# no parameters should be missing
|
||||
assert len(invalid_files_and_errors_tups) == 0
|
||||
assert not tool.params_with_missing_data_table_entry
|
||||
assert not tool.params_with_missing_index_file
|
||||
|
||||
|
||||
@contextmanager
|
||||
def setup_bismark():
|
||||
repo_dir = tempfile.mkdtemp()
|
||||
with tarfile.open(BISMARK_TAR) as archive:
|
||||
archive.extractall(repo_dir)
|
||||
yield repo_dir
|
||||
shutil.rmtree(repo_dir, ignore_errors=True)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_tool_validator():
|
||||
app = MockApp()
|
||||
with ValidationContext.from_app(app) as validation_context:
|
||||
yield ToolValidator(validation_context)
|
||||
Reference in New Issue
Block a user