diff --git a/Makefile b/Makefile index 777f587abf9..830bc896db0 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ RELEASE_NEXT:=16.04 RELEASE_NEXT_BRANCH:=dev RELEASE_UPSTREAM:=upstream MY_UPSTREAM:=origin -CONFIG_MANAGE=$(IN_VENV) python lib/galaxy/webapps/config_manage.py +CONFIG_MANAGE=$(IN_VENV) python lib/galaxy/config/config_manage.py PROJECT_URL?=https://github.com/galaxyproject/galaxy DOCS_DIR=doc DOC_SOURCE_DIR=$(DOCS_DIR)/source diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index bf4f8b049e0..69cb8ba51cf 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -47,7 +47,7 @@ from galaxy.web_stack import ( get_stack_facts, register_postfork_function ) -from galaxy.webapps.config_manage import GALAXY_APP +from .config_manage import GALAXY_APP from ..version import VERSION_MAJOR log = logging.getLogger(__name__) diff --git a/lib/galaxy/webapps/config_manage.py b/lib/galaxy/config/config_manage.py similarity index 100% rename from lib/galaxy/webapps/config_manage.py rename to lib/galaxy/config/config_manage.py diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index e161c1f844d..98a595252b1 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -811,7 +811,7 @@ class Loom(H5): def sniff(self, filename): if super(Loom, self).sniff(filename): try: - with h5py.File(filename) as loom_file: + with h5py.File(filename, 'r') as loom_file: return bool(loom_file.attrs.get('LOOM_SPEC_VERSION', False)) except Exception: return False @@ -834,13 +834,13 @@ class Loom(H5): def set_meta(self, dataset, overwrite=True, **kwd): super(Loom, self).set_meta(dataset, overwrite=overwrite, **kwd) try: - with h5py.File(dataset.file_name) as loom_file: - dataset.metadata.title = loom_file.attrs.get('title', None) - dataset.metadata.description = loom_file.attrs.get('description', None) - dataset.metadata.url = loom_file.attrs.get('url', None) - dataset.metadata.doi = loom_file.attrs.get('doi', None) - dataset.metadata.loom_spec_version = loom_file.attrs.get('LOOM_SPEC_VERSION', None) - dataset.creation_date = loom_file.attrs.get('creation_date', None) + with h5py.File(dataset.file_name, 'r') as loom_file: + dataset.metadata.title = util.unicodify(loom_file.attrs.get('title')) + dataset.metadata.description = util.unicodify(loom_file.attrs.get('description')) + dataset.metadata.url = util.unicodify(loom_file.attrs.get('url')) + dataset.metadata.doi = util.unicodify(loom_file.attrs.get('doi')) + dataset.metadata.loom_spec_version = util.unicodify(loom_file.attrs.get('LOOM_SPEC_VERSION')) + dataset.creation_date = util.unicodify(loom_file.attrs.get('creation_date')) dataset.metadata.shape = tuple(loom_file['matrix'].shape) tmp = list(loom_file['layers'].keys()) @@ -880,7 +880,7 @@ class Anndata(H5): def sniff(self, filename): if super(Anndata, self).sniff(filename): try: - with h5py.File(filename) as f: + with h5py.File(filename, 'r') as f: return all(attr in f for attr in ['X', 'obs', 'var']) except Exception: return False @@ -1002,45 +1002,40 @@ class Biom2(H5): False """ if super(Biom2, self).sniff(filename): - try: - f = h5py.File(filename) - attributes = list(dict(f.attrs.items())) - required_fields = ['id', 'format-url', 'type', 'generated-by', 'creation-date', 'nnz', 'shape'] - return set(required_fields).issubset(attributes) - except Exception: - return False + with h5py.File(filename, 'r') as f: + required_fields = {'id', 'format-url', 'type', 'generated-by', 'creation-date', 'nnz', 'shape'} + return required_fields.issubset(f.attrs.keys()) return False def set_meta(self, dataset, overwrite=True, **kwd): super(Biom2, self).set_meta(dataset, overwrite=overwrite, **kwd) try: - f = h5py.File(dataset.file_name) - attributes = dict(f.attrs.items()) - - dataset.metadata.id = attributes['id'] - dataset.metadata.format_url = attributes['format-url'] - if 'format-version' in attributes: # biom 2.1 - dataset.metadata.format_version = '.'.join(map(str, list(attributes['format-version']))) - elif 'format' in attributes: # biom 2.0 - dataset.metadata.format = attributes['format'] - dataset.metadata.type = attributes['type'] - dataset.metadata.shape = tuple(attributes['shape']) - dataset.metadata.generated_by = attributes['generated-by'] - dataset.metadata.creation_date = attributes['creation-date'] - dataset.metadata.nnz = int(attributes['nnz']) + with h5py.File(dataset.file_name, 'r') as f: + attributes = f.attrs + dataset.metadata.id = util.unicodify(attributes['id']) + dataset.metadata.format_url = util.unicodify(attributes['format-url']) + if 'format-version' in attributes: # biom 2.1 + dataset.metadata.format_version = '.'.join(str(_) for _ in attributes['format-version']) + elif 'format' in attributes: # biom 2.0 + dataset.metadata.format = util.unicodify(attributes['format']) + dataset.metadata.type = util.unicodify(attributes['type']) + dataset.metadata.shape = tuple((int(_) for _ in attributes['shape'])) + dataset.metadata.generated_by = util.unicodify(attributes['generated-by']) + dataset.metadata.creation_date = util.unicodify(attributes['creation-date']) + dataset.metadata.nnz = int(attributes['nnz']) except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning('%s, set_meta Exception: %s', self, util.unicodify(e)) def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: lines = ['Biom2 (HDF5) file'] try: - f = h5py.File(dataset.file_name) - for k, v in dict(f.attrs).items(): - lines.append('%s: %s' % (k, v)) + with h5py.File(dataset.file_name) as f: + for k, v in f.attrs.items(): + lines.append('%s: %s' % (k, util.unicodify(v))) except Exception as e: - log.warning('%s, set_peek Exception: %s', self, e) + log.warning('%s, set_peek Exception: %s', self, util.unicodify(e)) dataset.peek = '\n'.join(lines) dataset.blurb = nice_size(dataset.get_size()) else: @@ -1084,8 +1079,8 @@ class Cool(H5): if super(Cool, self).sniff(filename): keys = ['chroms', 'bins', 'pixels', 'indexes'] with h5py.File(filename, 'r') as handle: - fmt = handle.attrs.get('format', None) - url = handle.attrs.get('format-url', None) + fmt = handle.attrs.get('format') + url = handle.attrs.get('format-url') if fmt == MAGIC or url == URL: if not all(name in handle.keys() for name in keys): return False @@ -1144,8 +1139,8 @@ class MCool(H5): return False res0 = list(handle['resolutions'].keys())[0] keys = ['chroms', 'bins', 'pixels', 'indexes'] - fmt = handle['resolutions'][res0].attrs.get('format', None) - url = handle['resolutions'][res0].attrs.get('format-url', None) + fmt = handle['resolutions'][res0].attrs.get('format') + url = handle['resolutions'][res0].attrs.get('format-url') if fmt == MAGIC or url == URL: if not all(name in handle['resolutions'][res0].keys() for name in keys): return False @@ -1953,17 +1948,14 @@ class PostgresqlArchive(CompressedArchive): if dataset and tarfile.is_tarfile(dataset.file_name): with tarfile.open(dataset.file_name, 'r') as temptar: pg_version_file = temptar.extractfile('postgresql/db/PG_VERSION') - dataset.metadata.version = pg_version_file.read().strip() + dataset.metadata.version = util.unicodify(pg_version_file.read()).strip() except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning('%s, set_meta Exception: %s', self, util.unicodify(e)) def sniff(self, filename): if filename and tarfile.is_tarfile(filename): - try: - with tarfile.open(filename, 'r') as temptar: - return 'postgresql/db/PG_VERSION' in temptar.getnames() - except Exception as e: - log.warning('%s, sniff Exception: %s', self, e) + with tarfile.open(filename, 'r') as temptar: + return 'postgresql/db/PG_VERSION' in temptar.getnames() return False def set_peek(self, dataset, is_multi_byte=False): diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index 52486966374..91de2729256 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -14,6 +14,7 @@ import json import logging import os import sys +import traceback from six.moves import cPickle from sqlalchemy.orm import clear_mappers @@ -29,8 +30,6 @@ from galaxy.util import ( logging.basicConfig() log = logging.getLogger(__name__) -galaxy.model.Job() # this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here - def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size): # This method is somewhat odd, in that we set the metadata attributes from tool, @@ -122,8 +121,8 @@ def set_metadata_portable(): set_meta(dataset, file_dict) dataset.metadata.to_JSON_dict(filename_out) # write out results of set_meta json.dump((True, 'Metadata has been set successfully'), open(filename_results_code, 'wt+')) # setting metadata has succeeded - except Exception as e: - json.dump((False, unicodify(e)), open(filename_results_code, 'wt+')) # setting metadata has failed somehow + except Exception: + json.dump((False, traceback.format_exc()), open(filename_results_code, 'wt+')) # setting metadata has failed somehow write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_provided_metadata) diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index b7bc6f5fa8b..7bc4b6b3c41 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -242,7 +242,9 @@ class MutationList(MutationObj, list): self.changed() def extend(self, values): - list.extend(self, (MutationObj.coerce(self._key, v) for v in values)) + if hasattr(self, '_key'): + values = (MutationObj.coerce(self._key, value) for value in values) + list.extend(self, values) self.changed() def pop(self, *args, **kw): diff --git a/test/base/driver_util.py b/test/base/driver_util.py index 560e29d00a4..e168e8a7791 100644 --- a/test/base/driver_util.py +++ b/test/base/driver_util.py @@ -220,6 +220,7 @@ def setup_galaxy_config( conda_auto_init=conda_auto_init, conda_auto_install=conda_auto_install, cleanup_job=cleanup_job, + retry_metadata_internally=False, data_manager_config_file=data_manager_config_file, enable_beta_tool_formats=True, expose_dataset_path=True, diff --git a/test/unit/webapps/config_manage/test_appschema.py b/test/unit/webapps/config_manage/test_appschema.py index c6f0467f4cc..13eddba2900 100644 --- a/test/unit/webapps/config_manage/test_appschema.py +++ b/test/unit/webapps/config_manage/test_appschema.py @@ -1,4 +1,4 @@ -from galaxy.webapps.config_manage import AppSchema +from galaxy.config.config_manage import AppSchema def test_get_reloadable_option_defaults(monkeypatch): diff --git a/test/unit/webapps/config_manage/test_config_manage.py b/test/unit/webapps/config_manage/test_config_manage.py index 0bb17d83208..51165e40650 100644 --- a/test/unit/webapps/config_manage/test_config_manage.py +++ b/test/unit/webapps/config_manage/test_config_manage.py @@ -5,7 +5,7 @@ import tempfile import yaml -from galaxy.webapps.config_manage import main +from galaxy.config.config_manage import main THIS_DIR = os.path.dirname(__file__)