opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' of https://github.com/galaxyproject/galaxy into data-managers-update

Browse Source
This commit is contained in:
Peter Selten
2019-08-24 11:06:30 +02:00
parent 333950ec46 28096601b2
commit d48d66644c
9 changed files with 49 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+1 -1
View File
@@ -10,7 +10,7 @@ RELEASE_NEXT:=16.04
RELEASE_NEXT_BRANCH:=dev
RELEASE_UPSTREAM:=upstream
MY_UPSTREAM:=origin
CONFIG_MANAGE=$(IN_VENV) python lib/galaxy/webapps/config_manage.py
CONFIG_MANAGE=$(IN_VENV) python lib/galaxy/config/config_manage.py
PROJECT_URL?=https://github.com/galaxyproject/galaxy
DOCS_DIR=doc
DOC_SOURCE_DIR=$(DOCS_DIR)/source
lib/galaxy/config/__init__.py
+1 -1
View File
@@ -47,7 +47,7 @@ from galaxy.web_stack import (
get_stack_facts,
register_postfork_function
)
from galaxy.webapps.config_manage import GALAXY_APP
from .config_manage import GALAXY_APP
from ..version import VERSION_MAJOR
log = logging.getLogger(__name__)
lib/galaxy/webapps/config_manage.py → lib/galaxy/config/config_manage.py
View File
lib/galaxy/datatypes/binary.py
+38 -46
View File
@@ -811,7 +811,7 @@ class Loom(H5):
def sniff(self, filename):
if super(Loom, self).sniff(filename):
try:
with h5py.File(filename) as loom_file:
with h5py.File(filename, 'r') as loom_file:
return bool(loom_file.attrs.get('LOOM_SPEC_VERSION', False))
except Exception:
return False
@@ -834,13 +834,13 @@ class Loom(H5):
def set_meta(self, dataset, overwrite=True, **kwd):
super(Loom, self).set_meta(dataset, overwrite=overwrite, **kwd)
try:
with h5py.File(dataset.file_name) as loom_file:
dataset.metadata.title = loom_file.attrs.get('title', None)
dataset.metadata.description = loom_file.attrs.get('description', None)
dataset.metadata.url = loom_file.attrs.get('url', None)
dataset.metadata.doi = loom_file.attrs.get('doi', None)
dataset.metadata.loom_spec_version = loom_file.attrs.get('LOOM_SPEC_VERSION', None)
dataset.creation_date = loom_file.attrs.get('creation_date', None)
with h5py.File(dataset.file_name, 'r') as loom_file:
dataset.metadata.title = util.unicodify(loom_file.attrs.get('title'))
dataset.metadata.description = util.unicodify(loom_file.attrs.get('description'))
dataset.metadata.url = util.unicodify(loom_file.attrs.get('url'))
dataset.metadata.doi = util.unicodify(loom_file.attrs.get('doi'))
dataset.metadata.loom_spec_version = util.unicodify(loom_file.attrs.get('LOOM_SPEC_VERSION'))
dataset.creation_date = util.unicodify(loom_file.attrs.get('creation_date'))
dataset.metadata.shape = tuple(loom_file['matrix'].shape)
tmp = list(loom_file['layers'].keys())
@@ -880,7 +880,7 @@ class Anndata(H5):
def sniff(self, filename):
if super(Anndata, self).sniff(filename):
try:
with h5py.File(filename) as f:
with h5py.File(filename, 'r') as f:
return all(attr in f for attr in ['X', 'obs', 'var'])
except Exception:
return False
@@ -1002,45 +1002,40 @@ class Biom2(H5):
False
"""
if super(Biom2, self).sniff(filename):
try:
f = h5py.File(filename)
attributes = list(dict(f.attrs.items()))
required_fields = ['id', 'format-url', 'type', 'generated-by', 'creation-date', 'nnz', 'shape']
return set(required_fields).issubset(attributes)
except Exception:
return False
with h5py.File(filename, 'r') as f:
required_fields = {'id', 'format-url', 'type', 'generated-by', 'creation-date', 'nnz', 'shape'}
return required_fields.issubset(f.attrs.keys())
return False
def set_meta(self, dataset, overwrite=True, **kwd):
super(Biom2, self).set_meta(dataset, overwrite=overwrite, **kwd)
try:
f = h5py.File(dataset.file_name)
attributes = dict(f.attrs.items())
dataset.metadata.id = attributes['id']
dataset.metadata.format_url = attributes['format-url']
if 'format-version' in attributes: # biom 2.1
dataset.metadata.format_version = '.'.join(map(str, list(attributes['format-version'])))
elif 'format' in attributes: # biom 2.0
dataset.metadata.format = attributes['format']
dataset.metadata.type = attributes['type']
dataset.metadata.shape = tuple(attributes['shape'])
dataset.metadata.generated_by = attributes['generated-by']
dataset.metadata.creation_date = attributes['creation-date']
dataset.metadata.nnz = int(attributes['nnz'])
with h5py.File(dataset.file_name, 'r') as f:
attributes = f.attrs
dataset.metadata.id = util.unicodify(attributes['id'])
dataset.metadata.format_url = util.unicodify(attributes['format-url'])
if 'format-version' in attributes: # biom 2.1
dataset.metadata.format_version = '.'.join(str(_) for _ in attributes['format-version'])
elif 'format' in attributes: # biom 2.0
dataset.metadata.format = util.unicodify(attributes['format'])
dataset.metadata.type = util.unicodify(attributes['type'])
dataset.metadata.shape = tuple((int(_) for _ in attributes['shape']))
dataset.metadata.generated_by = util.unicodify(attributes['generated-by'])
dataset.metadata.creation_date = util.unicodify(attributes['creation-date'])
dataset.metadata.nnz = int(attributes['nnz'])
except Exception as e:
log.warning('%s, set_meta Exception: %s', self, e)
log.warning('%s, set_meta Exception: %s', self, util.unicodify(e))
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
lines = ['Biom2 (HDF5) file']
try:
f = h5py.File(dataset.file_name)
for k, v in dict(f.attrs).items():
lines.append('%s: %s' % (k, v))
with h5py.File(dataset.file_name) as f:
for k, v in f.attrs.items():
lines.append('%s: %s' % (k, util.unicodify(v)))
except Exception as e:
log.warning('%s, set_peek Exception: %s', self, e)
log.warning('%s, set_peek Exception: %s', self, util.unicodify(e))
dataset.peek = '\n'.join(lines)
dataset.blurb = nice_size(dataset.get_size())
else:
@@ -1084,8 +1079,8 @@ class Cool(H5):
if super(Cool, self).sniff(filename):
keys = ['chroms', 'bins', 'pixels', 'indexes']
with h5py.File(filename, 'r') as handle:
fmt = handle.attrs.get('format', None)
url = handle.attrs.get('format-url', None)
fmt = handle.attrs.get('format')
url = handle.attrs.get('format-url')
if fmt == MAGIC or url == URL:
if not all(name in handle.keys() for name in keys):
return False
@@ -1144,8 +1139,8 @@ class MCool(H5):
return False
res0 = list(handle['resolutions'].keys())[0]
keys = ['chroms', 'bins', 'pixels', 'indexes']
fmt = handle['resolutions'][res0].attrs.get('format', None)
url = handle['resolutions'][res0].attrs.get('format-url', None)
fmt = handle['resolutions'][res0].attrs.get('format')
url = handle['resolutions'][res0].attrs.get('format-url')
if fmt == MAGIC or url == URL:
if not all(name in handle['resolutions'][res0].keys() for name in keys):
return False
@@ -1953,17 +1948,14 @@ class PostgresqlArchive(CompressedArchive):
if dataset and tarfile.is_tarfile(dataset.file_name):
with tarfile.open(dataset.file_name, 'r') as temptar:
pg_version_file = temptar.extractfile('postgresql/db/PG_VERSION')
dataset.metadata.version = pg_version_file.read().strip()
dataset.metadata.version = util.unicodify(pg_version_file.read()).strip()
except Exception as e:
log.warning('%s, set_meta Exception: %s', self, e)
log.warning('%s, set_meta Exception: %s', self, util.unicodify(e))
def sniff(self, filename):
if filename and tarfile.is_tarfile(filename):
try:
with tarfile.open(filename, 'r') as temptar:
return 'postgresql/db/PG_VERSION' in temptar.getnames()
except Exception as e:
log.warning('%s, sniff Exception: %s', self, e)
with tarfile.open(filename, 'r') as temptar:
return 'postgresql/db/PG_VERSION' in temptar.getnames()
return False
def set_peek(self, dataset, is_multi_byte=False):
lib/galaxy/metadata/set_metadata.py
+3 -4
View File
@@ -14,6 +14,7 @@ import json
import logging
import os
import sys
import traceback
from six.moves import cPickle
from sqlalchemy.orm import clear_mappers
@@ -29,8 +30,6 @@ from galaxy.util import (
logging.basicConfig()
log = logging.getLogger(__name__)
galaxy.model.Job() # this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here
def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size):
# This method is somewhat odd, in that we set the metadata attributes from tool,
@@ -122,8 +121,8 @@ def set_metadata_portable():
set_meta(dataset, file_dict)
dataset.metadata.to_JSON_dict(filename_out) # write out results of set_meta
json.dump((True, 'Metadata has been set successfully'), open(filename_results_code, 'wt+')) # setting metadata has succeeded
except Exception as e:
json.dump((False, unicodify(e)), open(filename_results_code, 'wt+')) # setting metadata has failed somehow
except Exception:
json.dump((False, traceback.format_exc()), open(filename_results_code, 'wt+')) # setting metadata has failed somehow
write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_provided_metadata)
lib/galaxy/model/custom_types.py
+3 -1
View File
@@ -242,7 +242,9 @@ class MutationList(MutationObj, list):
self.changed()
def extend(self, values):
list.extend(self, (MutationObj.coerce(self._key, v) for v in values))
if hasattr(self, '_key'):
values = (MutationObj.coerce(self._key, value) for value in values)
list.extend(self, values)
self.changed()
def pop(self, *args, **kw):
test/base/driver_util.py
+1
View File
@@ -220,6 +220,7 @@ def setup_galaxy_config(
conda_auto_init=conda_auto_init,
conda_auto_install=conda_auto_install,
cleanup_job=cleanup_job,
retry_metadata_internally=False,
data_manager_config_file=data_manager_config_file,
enable_beta_tool_formats=True,
expose_dataset_path=True,
test/unit/webapps/config_manage/test_appschema.py
+1 -1
View File
@@ -1,4 +1,4 @@
from galaxy.webapps.config_manage import AppSchema
from galaxy.config.config_manage import AppSchema
def test_get_reloadable_option_defaults(monkeypatch):
test/unit/webapps/config_manage/test_config_manage.py
+1 -1
View File
@@ -5,7 +5,7 @@ import tempfile
import yaml
from galaxy.webapps.config_manage import main
from galaxy.config.config_manage import main
THIS_DIR = os.path.dirname(__file__)