From 7266e3cd2e4d84a89708ad7b6b4628bcb184122c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 22 Aug 2019 12:09:14 +0200 Subject: [PATCH 01/14] Don't retry metadata setting internally --- test/base/driver_util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/base/driver_util.py b/test/base/driver_util.py index 560e29d00a4..e168e8a7791 100644 --- a/test/base/driver_util.py +++ b/test/base/driver_util.py @@ -220,6 +220,7 @@ def setup_galaxy_config( conda_auto_init=conda_auto_init, conda_auto_install=conda_auto_install, cleanup_job=cleanup_job, + retry_metadata_internally=False, data_manager_config_file=data_manager_config_file, enable_beta_tool_formats=True, expose_dataset_path=True, From 62f25f9d185635262a0dddaecfd90572d4e26f1b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 22 Aug 2019 10:55:21 +0200 Subject: [PATCH 02/14] Fix missed control_worker->queue_worker rename Missed that one in 6b52f62fe4fd835ceafdff072757538b855446bf --- scripts/galaxy-main | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/galaxy-main b/scripts/galaxy-main index 1d649f2ffad..470d34502a0 100755 --- a/scripts/galaxy-main +++ b/scripts/galaxy-main @@ -108,7 +108,7 @@ def load_galaxy_app( **kwds ) app.database_heartbeat.start() - app.control_worker.bind_and_start() + app.queue_worker.bind_and_start() app.application_stack.log_startup() return app From 6db527196fc7df75eb90ff319130dc723a12f5d5 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 22 Aug 2019 14:34:56 +0200 Subject: [PATCH 03/14] Fix dataset unpickling in set_metadata Otherwise setting metadata externally fails with ``` galaxy.metadata DEBUG 2019-08-22 14:27:06,483 [p:4375,w:0,m:1] [LocalRunner.work_thread-1] setting metadata externally failed for HistoryDatasetAssociation 51: Traceback (most recent call last): File "/Users/mvandenb/src/galaxy/lib/galaxy/metadata/set_metadata.py", line 108, in set_metadata_portable dataset = cPickle.load(open(filename_in, 'rb')) # load DatasetInstance File "/Users/mvandenb/src/galaxy/lib/galaxy/model/custom_types.py", line 245, in extend list.extend(self, (MutationObj.coerce(self._key, v) for v in values)) File "/Users/mvandenb/src/galaxy/lib/galaxy/model/custom_types.py", line 245, in list.extend(self, (MutationObj.coerce(self._key, v) for v in values)) AttributeError: 'MutationList' object has no attribute '_key' ``` --- lib/galaxy/model/custom_types.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index b7bc6f5fa8b..7bc4b6b3c41 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -242,7 +242,9 @@ class MutationList(MutationObj, list): self.changed() def extend(self, values): - list.extend(self, (MutationObj.coerce(self._key, v) for v in values)) + if hasattr(self, '_key'): + values = (MutationObj.coerce(self._key, value) for value in values) + list.extend(self, values) self.changed() def pop(self, *args, **kw): From 95465ed83b1f1f3db19b4f5b5caef9b9743aa395 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 22 Aug 2019 14:38:14 +0200 Subject: [PATCH 04/14] Remove workaround that doesn't seem necessary anymore --- lib/galaxy/metadata/set_metadata.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index 52486966374..34b20cac50d 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -29,8 +29,6 @@ from galaxy.util import ( logging.basicConfig() log = logging.getLogger(__name__) -galaxy.model.Job() # this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here - def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size): # This method is somewhat odd, in that we set the metadata attributes from tool, From 61f338f6a2c1c005ac52cd871a46b25e7872409a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 22 Aug 2019 14:38:50 +0200 Subject: [PATCH 05/14] Log full tracback instead of just the exception message --- lib/galaxy/metadata/set_metadata.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index 34b20cac50d..91de2729256 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -14,6 +14,7 @@ import json import logging import os import sys +import traceback from six.moves import cPickle from sqlalchemy.orm import clear_mappers @@ -120,8 +121,8 @@ def set_metadata_portable(): set_meta(dataset, file_dict) dataset.metadata.to_JSON_dict(filename_out) # write out results of set_meta json.dump((True, 'Metadata has been set successfully'), open(filename_results_code, 'wt+')) # setting metadata has succeeded - except Exception as e: - json.dump((False, unicodify(e)), open(filename_results_code, 'wt+')) # setting metadata has failed somehow + except Exception: + json.dump((False, traceback.format_exc()), open(filename_results_code, 'wt+')) # setting metadata has failed somehow write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_provided_metadata) From 9548d8fa4c3641692b8e6d736d9c0edf1d7c47ad Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 22 Aug 2019 15:07:26 +0200 Subject: [PATCH 06/14] Fix Loom MetdatataElement setting on python 3 h5py may return bytestring if attribuets where written as bytestrings (xref https://github.com/h5py/h5py/issues/379). --- lib/galaxy/datatypes/binary.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index e161c1f844d..966f0cba391 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -835,12 +835,12 @@ class Loom(H5): super(Loom, self).set_meta(dataset, overwrite=overwrite, **kwd) try: with h5py.File(dataset.file_name) as loom_file: - dataset.metadata.title = loom_file.attrs.get('title', None) - dataset.metadata.description = loom_file.attrs.get('description', None) - dataset.metadata.url = loom_file.attrs.get('url', None) - dataset.metadata.doi = loom_file.attrs.get('doi', None) - dataset.metadata.loom_spec_version = loom_file.attrs.get('LOOM_SPEC_VERSION', None) - dataset.creation_date = loom_file.attrs.get('creation_date', None) + dataset.metadata.title = util.unicodify(loom_file.attrs.get('title', None)) + dataset.metadata.description = util.unicodify(loom_file.attrs.get('description', None)) + dataset.metadata.url = util.unicodify(loom_file.attrs.get('url', None)) + dataset.metadata.doi = util.unicodify(loom_file.attrs.get('doi', None)) + dataset.metadata.loom_spec_version = util.unicodify(loom_file.attrs.get('LOOM_SPEC_VERSION', None)) + dataset.creation_date = util.unicodify(loom_file.attrs.get('creation_date', None)) dataset.metadata.shape = tuple(loom_file['matrix'].shape) tmp = list(loom_file['layers'].keys()) From 50883944149677e2a5987048f98b4d114baaa700 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 22 Aug 2019 15:10:15 +0200 Subject: [PATCH 07/14] Fix Biom2 MetdatataElement setting on python 3 --- lib/galaxy/datatypes/binary.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 966f0cba391..ad93dbc658f 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1017,20 +1017,20 @@ class Biom2(H5): f = h5py.File(dataset.file_name) attributes = dict(f.attrs.items()) - dataset.metadata.id = attributes['id'] - dataset.metadata.format_url = attributes['format-url'] + dataset.metadata.id = util.unicodify(attributes['id']) + dataset.metadata.format_url = util.unicodify(attributes['format-url']) if 'format-version' in attributes: # biom 2.1 - dataset.metadata.format_version = '.'.join(map(str, list(attributes['format-version']))) + dataset.metadata.format_version = '.'.join((util.unicodify(_) for _ in list(attributes['format-version']))) elif 'format' in attributes: # biom 2.0 - dataset.metadata.format = attributes['format'] - dataset.metadata.type = attributes['type'] + dataset.metadata.format = util.unicodify(attributes['format']) + dataset.metadata.type = util.unicodify(attributes['type']) dataset.metadata.shape = tuple(attributes['shape']) - dataset.metadata.generated_by = attributes['generated-by'] - dataset.metadata.creation_date = attributes['creation-date'] + dataset.metadata.generated_by = util.unicodify(attributes['generated-by']) + dataset.metadata.creation_date = util.unicodify(attributes['creation-date']) dataset.metadata.nnz = int(attributes['nnz']) except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning('%s, set_meta Exception: %s', self, util.unicodify(e)) def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: @@ -1038,9 +1038,9 @@ class Biom2(H5): try: f = h5py.File(dataset.file_name) for k, v in dict(f.attrs).items(): - lines.append('%s: %s' % (k, v)) + lines.append('%s: %s' % (k, util.unicodify(v))) except Exception as e: - log.warning('%s, set_peek Exception: %s', self, e) + log.warning('%s, set_peek Exception: %s', self, util.unicodify(e)) dataset.peek = '\n'.join(lines) dataset.blurb = nice_size(dataset.get_size()) else: From 907442e62dc16139e02fff849981b10c11fac57a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 22 Aug 2019 15:10:43 +0200 Subject: [PATCH 08/14] Fix postgresql metadata setting on python 3 --- lib/galaxy/datatypes/binary.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index ad93dbc658f..83d203c2f6c 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1953,17 +1953,14 @@ class PostgresqlArchive(CompressedArchive): if dataset and tarfile.is_tarfile(dataset.file_name): with tarfile.open(dataset.file_name, 'r') as temptar: pg_version_file = temptar.extractfile('postgresql/db/PG_VERSION') - dataset.metadata.version = pg_version_file.read().strip() + dataset.metadata.version = util.unicodify(pg_version_file.read()).strip() except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning('%s, set_meta Exception: %s', self, util.unicodify(e)) def sniff(self, filename): if filename and tarfile.is_tarfile(filename): - try: - with tarfile.open(filename, 'r') as temptar: - return 'postgresql/db/PG_VERSION' in temptar.getnames() - except Exception as e: - log.warning('%s, sniff Exception: %s', self, e) + with tarfile.open(filename, 'r') as temptar: + return 'postgresql/db/PG_VERSION' in temptar.getnames() return False def set_peek(self, dataset, is_multi_byte=False): From 3dd500478df5e0db96de502fe14569cf92462d75 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 22 Aug 2019 19:54:21 +0200 Subject: [PATCH 09/14] Fix Biom2 shape MetadataElement --- lib/galaxy/datatypes/binary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 83d203c2f6c..7f9d1504a42 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1024,7 +1024,7 @@ class Biom2(H5): elif 'format' in attributes: # biom 2.0 dataset.metadata.format = util.unicodify(attributes['format']) dataset.metadata.type = util.unicodify(attributes['type']) - dataset.metadata.shape = tuple(attributes['shape']) + dataset.metadata.shape = tuple((util.unicodify(_) for _ in attributes['shape'])) dataset.metadata.generated_by = util.unicodify(attributes['generated-by']) dataset.metadata.creation_date = util.unicodify(attributes['creation-date']) dataset.metadata.nnz = int(attributes['nnz']) From fb98cf3585100ea72bffefd47d7a536a8e773401 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 22 Aug 2019 15:46:08 -0400 Subject: [PATCH 10/14] Move config_manage out of webapps/ into config/ First pass. --- Makefile | 2 +- lib/galaxy/config/__init__.py | 2 +- lib/galaxy/{webapps => config}/config_manage.py | 0 test/unit/webapps/config_manage/test_appschema.py | 2 +- test/unit/webapps/config_manage/test_config_manage.py | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename lib/galaxy/{webapps => config}/config_manage.py (100%) diff --git a/Makefile b/Makefile index 777f587abf9..830bc896db0 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ RELEASE_NEXT:=16.04 RELEASE_NEXT_BRANCH:=dev RELEASE_UPSTREAM:=upstream MY_UPSTREAM:=origin -CONFIG_MANAGE=$(IN_VENV) python lib/galaxy/webapps/config_manage.py +CONFIG_MANAGE=$(IN_VENV) python lib/galaxy/config/config_manage.py PROJECT_URL?=https://github.com/galaxyproject/galaxy DOCS_DIR=doc DOC_SOURCE_DIR=$(DOCS_DIR)/source diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 2f29ff4ec44..0c49f4b3d10 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -47,7 +47,7 @@ from galaxy.web_stack import ( get_stack_facts, register_postfork_function ) -from galaxy.webapps.config_manage import GALAXY_APP +from .config_manage import GALAXY_APP from ..version import VERSION_MAJOR log = logging.getLogger(__name__) diff --git a/lib/galaxy/webapps/config_manage.py b/lib/galaxy/config/config_manage.py similarity index 100% rename from lib/galaxy/webapps/config_manage.py rename to lib/galaxy/config/config_manage.py diff --git a/test/unit/webapps/config_manage/test_appschema.py b/test/unit/webapps/config_manage/test_appschema.py index c6f0467f4cc..13eddba2900 100644 --- a/test/unit/webapps/config_manage/test_appschema.py +++ b/test/unit/webapps/config_manage/test_appschema.py @@ -1,4 +1,4 @@ -from galaxy.webapps.config_manage import AppSchema +from galaxy.config.config_manage import AppSchema def test_get_reloadable_option_defaults(monkeypatch): diff --git a/test/unit/webapps/config_manage/test_config_manage.py b/test/unit/webapps/config_manage/test_config_manage.py index 0bb17d83208..51165e40650 100644 --- a/test/unit/webapps/config_manage/test_config_manage.py +++ b/test/unit/webapps/config_manage/test_config_manage.py @@ -5,7 +5,7 @@ import tempfile import yaml -from galaxy.webapps.config_manage import main +from galaxy.config.config_manage import main THIS_DIR = os.path.dirname(__file__) From f7caec54af1d397e10d7925cb6431220f0b46560 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 23 Aug 2019 14:41:40 +0200 Subject: [PATCH 11/14] Open h5py files in 'r' mode, drop redundant .get('x', None) --- lib/galaxy/datatypes/binary.py | 47 +++++++++++++++------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 7f9d1504a42..9be060a805b 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -811,7 +811,7 @@ class Loom(H5): def sniff(self, filename): if super(Loom, self).sniff(filename): try: - with h5py.File(filename) as loom_file: + with h5py.File(filename, 'r') as loom_file: return bool(loom_file.attrs.get('LOOM_SPEC_VERSION', False)) except Exception: return False @@ -834,13 +834,13 @@ class Loom(H5): def set_meta(self, dataset, overwrite=True, **kwd): super(Loom, self).set_meta(dataset, overwrite=overwrite, **kwd) try: - with h5py.File(dataset.file_name) as loom_file: - dataset.metadata.title = util.unicodify(loom_file.attrs.get('title', None)) - dataset.metadata.description = util.unicodify(loom_file.attrs.get('description', None)) - dataset.metadata.url = util.unicodify(loom_file.attrs.get('url', None)) - dataset.metadata.doi = util.unicodify(loom_file.attrs.get('doi', None)) - dataset.metadata.loom_spec_version = util.unicodify(loom_file.attrs.get('LOOM_SPEC_VERSION', None)) - dataset.creation_date = util.unicodify(loom_file.attrs.get('creation_date', None)) + with h5py.File(dataset.file_name, 'r') as loom_file: + dataset.metadata.title = util.unicodify(loom_file.attrs.get('title')) + dataset.metadata.description = util.unicodify(loom_file.attrs.get('description')) + dataset.metadata.url = util.unicodify(loom_file.attrs.get('url')) + dataset.metadata.doi = util.unicodify(loom_file.attrs.get('doi')) + dataset.metadata.loom_spec_version = util.unicodify(loom_file.attrs.get('LOOM_SPEC_VERSION')) + dataset.creation_date = util.unicodify(loom_file.attrs.get('creation_date')) dataset.metadata.shape = tuple(loom_file['matrix'].shape) tmp = list(loom_file['layers'].keys()) @@ -880,7 +880,7 @@ class Anndata(H5): def sniff(self, filename): if super(Anndata, self).sniff(filename): try: - with h5py.File(filename) as f: + with h5py.File(filename, 'r') as f: return all(attr in f for attr in ['X', 'obs', 'var']) except Exception: return False @@ -1002,43 +1002,36 @@ class Biom2(H5): False """ if super(Biom2, self).sniff(filename): - try: - f = h5py.File(filename) + with h5py.File(filename, 'r') as f: attributes = list(dict(f.attrs.items())) required_fields = ['id', 'format-url', 'type', 'generated-by', 'creation-date', 'nnz', 'shape'] return set(required_fields).issubset(attributes) - except Exception: - return False return False def set_meta(self, dataset, overwrite=True, **kwd): super(Biom2, self).set_meta(dataset, overwrite=overwrite, **kwd) - try: - f = h5py.File(dataset.file_name) + with h5py.File(dataset.file_name, 'r') as f: attributes = dict(f.attrs.items()) dataset.metadata.id = util.unicodify(attributes['id']) dataset.metadata.format_url = util.unicodify(attributes['format-url']) if 'format-version' in attributes: # biom 2.1 - dataset.metadata.format_version = '.'.join((util.unicodify(_) for _ in list(attributes['format-version']))) + dataset.metadata.format_version = '.'.join((str(_) for _ in list(attributes['format-version']))) elif 'format' in attributes: # biom 2.0 dataset.metadata.format = util.unicodify(attributes['format']) dataset.metadata.type = util.unicodify(attributes['type']) - dataset.metadata.shape = tuple((util.unicodify(_) for _ in attributes['shape'])) + dataset.metadata.shape = tuple((int(_) for _ in attributes['shape'])) dataset.metadata.generated_by = util.unicodify(attributes['generated-by']) dataset.metadata.creation_date = util.unicodify(attributes['creation-date']) dataset.metadata.nnz = int(attributes['nnz']) - except Exception as e: - log.warning('%s, set_meta Exception: %s', self, util.unicodify(e)) - def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: lines = ['Biom2 (HDF5) file'] try: - f = h5py.File(dataset.file_name) - for k, v in dict(f.attrs).items(): - lines.append('%s: %s' % (k, util.unicodify(v))) + with h5py.File(dataset.file_name) as f: + for k, v in dict(f.attrs).items(): + lines.append('%s: %s' % (k, util.unicodify(v))) except Exception as e: log.warning('%s, set_peek Exception: %s', self, util.unicodify(e)) dataset.peek = '\n'.join(lines) @@ -1084,8 +1077,8 @@ class Cool(H5): if super(Cool, self).sniff(filename): keys = ['chroms', 'bins', 'pixels', 'indexes'] with h5py.File(filename, 'r') as handle: - fmt = handle.attrs.get('format', None) - url = handle.attrs.get('format-url', None) + fmt = handle.attrs.get('format') + url = handle.attrs.get('format-url') if fmt == MAGIC or url == URL: if not all(name in handle.keys() for name in keys): return False @@ -1144,8 +1137,8 @@ class MCool(H5): return False res0 = list(handle['resolutions'].keys())[0] keys = ['chroms', 'bins', 'pixels', 'indexes'] - fmt = handle['resolutions'][res0].attrs.get('format', None) - url = handle['resolutions'][res0].attrs.get('format-url', None) + fmt = handle['resolutions'][res0].attrs.get('format') + url = handle['resolutions'][res0].attrs.get('format-url') if fmt == MAGIC or url == URL: if not all(name in handle['resolutions'][res0].keys() for name in keys): return False From fa3e09b689bbc51d385f89cd1fc481d7a492ce2a Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Fri, 23 Aug 2019 16:11:39 +0200 Subject: [PATCH 12/14] Update lib/galaxy/datatypes/binary.py Co-Authored-By: Nicola Soranzo --- lib/galaxy/datatypes/binary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 9be060a805b..d1a19b6b77d 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1016,7 +1016,7 @@ class Biom2(H5): dataset.metadata.id = util.unicodify(attributes['id']) dataset.metadata.format_url = util.unicodify(attributes['format-url']) if 'format-version' in attributes: # biom 2.1 - dataset.metadata.format_version = '.'.join((str(_) for _ in list(attributes['format-version']))) + dataset.metadata.format_version = '.'.join(str(_) for _ in attributes['format-version']) elif 'format' in attributes: # biom 2.0 dataset.metadata.format = util.unicodify(attributes['format']) dataset.metadata.type = util.unicodify(attributes['type']) From 14f6483378217ab51044ed3c2eed3c888313d0fd Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 23 Aug 2019 18:01:10 +0200 Subject: [PATCH 13/14] Restore try/except around Biom2 metadata setting --- lib/galaxy/datatypes/binary.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index d1a19b6b77d..cf7c344499c 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1010,20 +1010,23 @@ class Biom2(H5): def set_meta(self, dataset, overwrite=True, **kwd): super(Biom2, self).set_meta(dataset, overwrite=overwrite, **kwd) - with h5py.File(dataset.file_name, 'r') as f: - attributes = dict(f.attrs.items()) + try: + with h5py.File(dataset.file_name, 'r') as f: + attributes = dict(f.attrs.items()) - dataset.metadata.id = util.unicodify(attributes['id']) - dataset.metadata.format_url = util.unicodify(attributes['format-url']) - if 'format-version' in attributes: # biom 2.1 - dataset.metadata.format_version = '.'.join(str(_) for _ in attributes['format-version']) - elif 'format' in attributes: # biom 2.0 - dataset.metadata.format = util.unicodify(attributes['format']) - dataset.metadata.type = util.unicodify(attributes['type']) - dataset.metadata.shape = tuple((int(_) for _ in attributes['shape'])) - dataset.metadata.generated_by = util.unicodify(attributes['generated-by']) - dataset.metadata.creation_date = util.unicodify(attributes['creation-date']) - dataset.metadata.nnz = int(attributes['nnz']) + dataset.metadata.id = util.unicodify(attributes['id']) + dataset.metadata.format_url = util.unicodify(attributes['format-url']) + if 'format-version' in attributes: # biom 2.1 + dataset.metadata.format_version = '.'.join(str(_) for _ in attributes['format-version']) + elif 'format' in attributes: # biom 2.0 + dataset.metadata.format = util.unicodify(attributes['format']) + dataset.metadata.type = util.unicodify(attributes['type']) + dataset.metadata.shape = tuple((int(_) for _ in attributes['shape'])) + dataset.metadata.generated_by = util.unicodify(attributes['generated-by']) + dataset.metadata.creation_date = util.unicodify(attributes['creation-date']) + dataset.metadata.nnz = int(attributes['nnz']) + except Exception as e: + log.warning('%s, set_meta Exception: %s', self, util.unicodify(e)) def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: From 47ebdd8910270281f3743c30326faab790c0695d Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 23 Aug 2019 22:22:53 +0100 Subject: [PATCH 14/14] Remove some unnecessary casts --- lib/galaxy/datatypes/binary.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index cf7c344499c..98a595252b1 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1003,16 +1003,15 @@ class Biom2(H5): """ if super(Biom2, self).sniff(filename): with h5py.File(filename, 'r') as f: - attributes = list(dict(f.attrs.items())) - required_fields = ['id', 'format-url', 'type', 'generated-by', 'creation-date', 'nnz', 'shape'] - return set(required_fields).issubset(attributes) + required_fields = {'id', 'format-url', 'type', 'generated-by', 'creation-date', 'nnz', 'shape'} + return required_fields.issubset(f.attrs.keys()) return False def set_meta(self, dataset, overwrite=True, **kwd): super(Biom2, self).set_meta(dataset, overwrite=overwrite, **kwd) try: with h5py.File(dataset.file_name, 'r') as f: - attributes = dict(f.attrs.items()) + attributes = f.attrs dataset.metadata.id = util.unicodify(attributes['id']) dataset.metadata.format_url = util.unicodify(attributes['format-url']) @@ -1033,7 +1032,7 @@ class Biom2(H5): lines = ['Biom2 (HDF5) file'] try: with h5py.File(dataset.file_name) as f: - for k, v in dict(f.attrs).items(): + for k, v in f.attrs.items(): lines.append('%s: %s' % (k, util.unicodify(v))) except Exception as e: log.warning('%s, set_peek Exception: %s', self, util.unicodify(e))