From 16b7d6a031821ca8180fb7357414853b44e6085d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 6 Jun 2019 13:57:41 -0400 Subject: [PATCH] Move set metadata. --- lib/galaxy/metadata/set_metadata.py | 219 ++++++++++++++++++ lib/galaxy_ext/metadata/_provided_metadata.py | 1 - lib/galaxy_ext/metadata/set_metadata.py | 210 +---------------- packages/job_execution/setup.py | 1 + 4 files changed, 223 insertions(+), 208 deletions(-) create mode 100644 lib/galaxy/metadata/set_metadata.py delete mode 120000 lib/galaxy_ext/metadata/_provided_metadata.py diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py new file mode 100644 index 00000000000..09fcd1f5403 --- /dev/null +++ b/lib/galaxy/metadata/set_metadata.py @@ -0,0 +1,219 @@ +""" +Execute an external process to set_meta() on a provided list of pickled datasets. + +This was formerly scripts/set_metadata.py and expects these arguments: + + %prog datatypes_conf.xml job_metadata_file metadata_in,metadata_kwds,metadata_out,metadata_results_code,output_filename_override,metadata_override... max_metadata_value_size + +Galaxy should be importable on sys.path and output_filename_override should be +set to the path of the dataset on which metadata is being set +(output_filename_override could previously be left empty and the path would be +constructed automatically). +""" +import json +import logging +import os +import sys + +from six.moves import cPickle +from sqlalchemy.orm import clear_mappers + +import galaxy.model.mapping # need to load this before we unpickle, in order to setup properties assigned by the mappers +from galaxy.model.custom_types import total_size +from galaxy.tool_util.provided_metadata import parse_tool_provided_metadata +from galaxy.util import stringify_dictionary_keys + +logging.basicConfig() +log = logging.getLogger(__name__) + +galaxy.model.Job() # this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here + + +def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size): + # This method is somewhat odd, in that we set the metadata attributes from tool, + # then call set_meta, then set metadata attributes from tool again. + # This is intentional due to interplay of overwrite kwd, the fact that some metadata + # parameters may rely on the values of others, and that we are accepting the + # values provided by the tool as Truth. + extension = dataset_instance.extension + if extension == "_sniff_": + try: + from galaxy.datatypes import sniff + extension = sniff.handle_uploaded_dataset_file(dataset_instance.dataset.external_filename, datatypes_registry) + # We need to both set the extension so it is available to set_meta + # and record it in the metadata so it can be reloaded on the server + # side and the model updated (see MetadataCollection.{from,to}_JSON_dict) + dataset_instance.extension = extension + # Set special metadata property that will reload this on server side. + setattr(dataset_instance.metadata, "__extension__", extension) + except Exception: + log.exception("Problem sniffing datatype.") + + for metadata_name, metadata_value in file_dict.get('metadata', {}).items(): + setattr(dataset_instance.metadata, metadata_name, metadata_value) + dataset_instance.datatype.set_meta(dataset_instance, **set_meta_kwds) + for metadata_name, metadata_value in file_dict.get('metadata', {}).items(): + setattr(dataset_instance.metadata, metadata_name, metadata_value) + + if max_metadata_value_size: + for k, v in list(dataset_instance.metadata.items()): + if total_size(v) > max_metadata_value_size: + log.info("Key %s too large for metadata, discarding" % k) + dataset_instance.metadata.remove_key(k) + + +def set_metadata(): + if len(sys.argv) == 1: + set_metadata_portable() + else: + set_metadata_legacy() + + +def set_metadata_portable(): + import galaxy.model + galaxy.model.metadata.MetadataTempFile.tmp_dir = tool_job_working_directory = os.path.abspath(os.getcwd()) + + metadata_params_path = os.path.join("metadata", "params.json") + try: + with open(metadata_params_path, "r") as f: + metadata_params = json.load(f) + except IOError: + raise Exception("Failed to find metadata/params.json from cwd [%s]" % tool_job_working_directory) + datatypes_config = metadata_params["datatypes_config"] + job_metadata = metadata_params["job_metadata"] + max_metadata_value_size = metadata_params.get("max_metadata_value_size") or 0 + outputs = metadata_params["outputs"] + + datatypes_registry = validate_and_load_datatypes_config(datatypes_config) + tool_provided_metadata = load_job_metadata(job_metadata) + + def set_meta(new_dataset_instance, file_dict): + set_meta_with_tool_provided(new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size) + + for output_name, output_dict in outputs.items(): + filename_in = os.path.join("metadata/metadata_in_%s" % output_name) + filename_kwds = os.path.join("metadata/metadata_kwds_%s" % output_name) + filename_out = os.path.join("metadata/metadata_out_%s" % output_name) + filename_results_code = os.path.join("metadata/metadata_results_%s" % output_name) + override_metadata = os.path.join("metadata/metadata_override_%s" % output_name) + dataset_filename_override = output_dict["filename_override"] + + # Same block as below... + set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds))) # load kwds; need to ensure our keywords are not unicode + try: + dataset = cPickle.load(open(filename_in, 'rb')) # load DatasetInstance + dataset.dataset.external_filename = dataset_filename_override + store_by = metadata_params.get("object_store_store_by", "id") + extra_files_dir_name = "dataset_%s_files" % getattr(dataset.dataset, store_by) + files_path = os.path.abspath(os.path.join(tool_job_working_directory, extra_files_dir_name)) + dataset.dataset.external_extra_files_path = files_path + file_dict = tool_provided_metadata.get_dataset_meta(output_name, dataset.dataset.id) + if 'ext' in file_dict: + dataset.extension = file_dict['ext'] + # Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles + override_metadata = json.load(open(override_metadata)) + for metadata_name, metadata_file_override in override_metadata: + if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value(metadata_file_override): + metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON(metadata_file_override) + setattr(dataset.metadata, metadata_name, metadata_file_override) + set_meta(dataset, file_dict) + dataset.metadata.to_JSON_dict(filename_out) # write out results of set_meta + json.dump((True, 'Metadata has been set successfully'), open(filename_results_code, 'wt+')) # setting metadata has succeeded + except Exception as e: + json.dump((False, str(e)), open(filename_results_code, 'wt+')) # setting metadata has failed somehow + + write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_provided_metadata) + + +def set_metadata_legacy(): + import galaxy.model + galaxy.model.metadata.MetadataTempFile.tmp_dir = tool_job_working_directory = os.path.abspath(os.getcwd()) + + # This is ugly, but to transition from existing jobs without this parameter + # to ones with, smoothly, it has to be the last optional parameter and we + # have to sniff it. + try: + max_metadata_value_size = int(sys.argv[-1]) + sys.argv = sys.argv[:-1] + except ValueError: + max_metadata_value_size = 0 + # max_metadata_value_size is unspecified and should be 0 + + # Set up datatypes registry + datatypes_config = sys.argv.pop(1) + datatypes_registry = validate_and_load_datatypes_config(datatypes_config) + + job_metadata = sys.argv.pop(1) + tool_provided_metadata = load_job_metadata(job_metadata) + + def set_meta(new_dataset_instance, file_dict): + set_meta_with_tool_provided(new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size) + + for filenames in sys.argv[1:]: + fields = filenames.split(',') + filename_in = fields.pop(0) + filename_kwds = fields.pop(0) + filename_out = fields.pop(0) + filename_results_code = fields.pop(0) + dataset_filename_override = fields.pop(0) + override_metadata = fields.pop(0) + set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds))) # load kwds; need to ensure our keywords are not unicode + try: + dataset = cPickle.load(open(filename_in, 'rb')) # load DatasetInstance + dataset.dataset.external_filename = dataset_filename_override + files_path = os.path.abspath(os.path.join(tool_job_working_directory, "dataset_%s_files" % (dataset.dataset.id))) + dataset.dataset.external_extra_files_path = files_path + file_dict = tool_provided_metadata.get_dataset_meta(None, dataset.dataset.id) + if 'ext' in file_dict: + dataset.extension = file_dict['ext'] + # Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles + override_metadata = json.load(open(override_metadata)) + for metadata_name, metadata_file_override in override_metadata: + if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value(metadata_file_override): + metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON(metadata_file_override) + setattr(dataset.metadata, metadata_name, metadata_file_override) + set_meta(dataset, file_dict) + dataset.metadata.to_JSON_dict(filename_out) # write out results of set_meta + json.dump((True, 'Metadata has been set successfully'), open(filename_results_code, 'wt+')) # setting metadata has succeeded + except Exception as e: + json.dump((False, str(e)), open(filename_results_code, 'wt+')) # setting metadata has failed somehow + + write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_provided_metadata) + + +def validate_and_load_datatypes_config(datatypes_config): + galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir)) + + if not os.path.exists(datatypes_config): + # Hack for Pulsar on usegalaxy.org, drop ASAP. + datatypes_config = "configs/registry.xml" + + if not os.path.exists(datatypes_config): + print("Metadata setting failed because registry.xml [%s] could not be found. You may retry setting metadata." % datatypes_config) + sys.exit(1) + import galaxy.datatypes.registry + datatypes_registry = galaxy.datatypes.registry.Registry() + datatypes_registry.load_datatypes(root_dir=galaxy_root, config=datatypes_config) + galaxy.model.set_datatypes_registry(datatypes_registry) + return datatypes_registry + + +def load_job_metadata(job_metadata): + return parse_tool_provided_metadata(job_metadata) + + +def write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_provided_metadata): + for i, file_dict in enumerate(tool_provided_metadata.get_new_datasets_for_metadata_collection(), start=1): + filename = file_dict["filename"] + new_dataset_filename = os.path.join(tool_job_working_directory, "working", filename) + new_dataset = galaxy.model.Dataset(id=-i, external_filename=new_dataset_filename) + extra_files = file_dict.get('extra_files', None) + if extra_files is not None: + new_dataset._extra_files_path = os.path.join(tool_job_working_directory, "working", extra_files) + new_dataset.state = new_dataset.states.OK + new_dataset_instance = galaxy.model.HistoryDatasetAssociation(id=-i, dataset=new_dataset, extension=file_dict.get('ext', 'data')) + set_meta(new_dataset_instance, file_dict) + file_dict['metadata'] = json.loads(new_dataset_instance.metadata.to_JSON_dict()) # storing metadata in external form, need to turn back into dict, then later jsonify + + tool_provided_metadata.rewrite() + clear_mappers() diff --git a/lib/galaxy_ext/metadata/_provided_metadata.py b/lib/galaxy_ext/metadata/_provided_metadata.py deleted file mode 120000 index b44719c00dc..00000000000 --- a/lib/galaxy_ext/metadata/_provided_metadata.py +++ /dev/null @@ -1 +0,0 @@ -../../galaxy/tool_util/provided_metadata.py \ No newline at end of file diff --git a/lib/galaxy_ext/metadata/set_metadata.py b/lib/galaxy_ext/metadata/set_metadata.py index 9121bd8a4e6..380e3de3336 100644 --- a/lib/galaxy_ext/metadata/set_metadata.py +++ b/lib/galaxy_ext/metadata/set_metadata.py @@ -10,217 +10,13 @@ set to the path of the dataset on which metadata is being set (output_filename_override could previously be left empty and the path would be constructed automatically). """ -import json -import logging + import os import sys # insert *this* galaxy before all others on sys.path sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))) -from six.moves import cPickle -from sqlalchemy.orm import clear_mappers +from galaxy.metadata.set_metadata import set_metadata -import galaxy.model.mapping # need to load this before we unpickle, in order to setup properties assigned by the mappers -from galaxy.model.custom_types import total_size -from galaxy.util import stringify_dictionary_keys -from ._provided_metadata import parse_tool_provided_metadata - -# ensure supported version -assert sys.version_info[:2] >= (2, 7), 'Python version must be at least 2.7, this is: %s' % sys.version - -logging.basicConfig() -log = logging.getLogger(__name__) - -galaxy.model.Job() # this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here - - -def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size): - # This method is somewhat odd, in that we set the metadata attributes from tool, - # then call set_meta, then set metadata attributes from tool again. - # This is intentional due to interplay of overwrite kwd, the fact that some metadata - # parameters may rely on the values of others, and that we are accepting the - # values provided by the tool as Truth. - extension = dataset_instance.extension - if extension == "_sniff_": - try: - from galaxy.datatypes import sniff - extension = sniff.handle_uploaded_dataset_file(dataset_instance.dataset.external_filename, datatypes_registry) - # We need to both set the extension so it is available to set_meta - # and record it in the metadata so it can be reloaded on the server - # side and the model updated (see MetadataCollection.{from,to}_JSON_dict) - dataset_instance.extension = extension - # Set special metadata property that will reload this on server side. - setattr(dataset_instance.metadata, "__extension__", extension) - except Exception: - log.exception("Problem sniffing datatype.") - - for metadata_name, metadata_value in file_dict.get('metadata', {}).items(): - setattr(dataset_instance.metadata, metadata_name, metadata_value) - dataset_instance.datatype.set_meta(dataset_instance, **set_meta_kwds) - for metadata_name, metadata_value in file_dict.get('metadata', {}).items(): - setattr(dataset_instance.metadata, metadata_name, metadata_value) - - if max_metadata_value_size: - for k, v in list(dataset_instance.metadata.items()): - if total_size(v) > max_metadata_value_size: - log.info("Key %s too large for metadata, discarding" % k) - dataset_instance.metadata.remove_key(k) - - -def set_metadata(): - if len(sys.argv) == 1: - set_metadata_portable() - else: - set_metadata_legacy() - - -def set_metadata_portable(): - import galaxy.model - tool_job_working_directory = os.path.abspath(os.getcwd()) - galaxy.model.metadata.MetadataTempFile.tmp_dir = os.path.join(tool_job_working_directory, "metadata") - - metadata_params_path = os.path.join("metadata", "params.json") - try: - with open(metadata_params_path, "r") as f: - metadata_params = json.load(f) - except IOError: - raise Exception("Failed to find metadata/params.json from cwd [%s]" % tool_job_working_directory) - datatypes_config = metadata_params["datatypes_config"] - job_metadata = metadata_params["job_metadata"] - max_metadata_value_size = metadata_params.get("max_metadata_value_size") or 0 - outputs = metadata_params["outputs"] - - datatypes_registry = validate_and_load_datatypes_config(datatypes_config) - tool_provided_metadata = load_job_metadata(job_metadata) - - def set_meta(new_dataset_instance, file_dict): - set_meta_with_tool_provided(new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size) - - for output_name, output_dict in outputs.items(): - filename_in = os.path.join("metadata/metadata_in_%s" % output_name) - filename_kwds = os.path.join("metadata/metadata_kwds_%s" % output_name) - filename_out = os.path.join("metadata/metadata_out_%s" % output_name) - filename_results_code = os.path.join("metadata/metadata_results_%s" % output_name) - override_metadata = os.path.join("metadata/metadata_override_%s" % output_name) - dataset_filename_override = output_dict["filename_override"] - - # Same block as below... - set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds))) # load kwds; need to ensure our keywords are not unicode - try: - dataset = cPickle.load(open(filename_in, 'rb')) # load DatasetInstance - dataset.dataset.external_filename = dataset_filename_override - store_by = metadata_params.get("object_store_store_by", "id") - extra_files_dir_name = "dataset_%s_files" % getattr(dataset.dataset, store_by) - files_path = os.path.abspath(os.path.join(tool_job_working_directory, extra_files_dir_name)) - dataset.dataset.external_extra_files_path = files_path - file_dict = tool_provided_metadata.get_dataset_meta(output_name, dataset.dataset.id) - if 'ext' in file_dict: - dataset.extension = file_dict['ext'] - # Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles - override_metadata = json.load(open(override_metadata)) - for metadata_name, metadata_file_override in override_metadata: - if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value(metadata_file_override): - metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON(metadata_file_override) - setattr(dataset.metadata, metadata_name, metadata_file_override) - set_meta(dataset, file_dict) - dataset.metadata.to_JSON_dict(filename_out) # write out results of set_meta - json.dump((True, 'Metadata has been set successfully'), open(filename_results_code, 'wt+')) # setting metadata has succeeded - except Exception as e: - json.dump((False, str(e)), open(filename_results_code, 'wt+')) # setting metadata has failed somehow - - write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_provided_metadata) - - -def set_metadata_legacy(): - import galaxy.model - galaxy.model.metadata.MetadataTempFile.tmp_dir = tool_job_working_directory = os.path.abspath(os.getcwd()) - - # This is ugly, but to transition from existing jobs without this parameter - # to ones with, smoothly, it has to be the last optional parameter and we - # have to sniff it. - try: - max_metadata_value_size = int(sys.argv[-1]) - sys.argv = sys.argv[:-1] - except ValueError: - max_metadata_value_size = 0 - # max_metadata_value_size is unspecified and should be 0 - - # Set up datatypes registry - datatypes_config = sys.argv.pop(1) - datatypes_registry = validate_and_load_datatypes_config(datatypes_config) - - job_metadata = sys.argv.pop(1) - tool_provided_metadata = load_job_metadata(job_metadata) - - def set_meta(new_dataset_instance, file_dict): - set_meta_with_tool_provided(new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size) - - for filenames in sys.argv[1:]: - fields = filenames.split(',') - filename_in = fields.pop(0) - filename_kwds = fields.pop(0) - filename_out = fields.pop(0) - filename_results_code = fields.pop(0) - dataset_filename_override = fields.pop(0) - override_metadata = fields.pop(0) - set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds))) # load kwds; need to ensure our keywords are not unicode - try: - dataset = cPickle.load(open(filename_in, 'rb')) # load DatasetInstance - dataset.dataset.external_filename = dataset_filename_override - files_path = os.path.abspath(os.path.join(tool_job_working_directory, "dataset_%s_files" % (dataset.dataset.id))) - dataset.dataset.external_extra_files_path = files_path - file_dict = tool_provided_metadata.get_dataset_meta(None, dataset.dataset.id) - if 'ext' in file_dict: - dataset.extension = file_dict['ext'] - # Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles - override_metadata = json.load(open(override_metadata)) - for metadata_name, metadata_file_override in override_metadata: - if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value(metadata_file_override): - metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON(metadata_file_override) - setattr(dataset.metadata, metadata_name, metadata_file_override) - set_meta(dataset, file_dict) - dataset.metadata.to_JSON_dict(filename_out) # write out results of set_meta - json.dump((True, 'Metadata has been set successfully'), open(filename_results_code, 'wt+')) # setting metadata has succeeded - except Exception as e: - json.dump((False, str(e)), open(filename_results_code, 'wt+')) # setting metadata has failed somehow - - write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_provided_metadata) - - -def validate_and_load_datatypes_config(datatypes_config): - galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir)) - - if not os.path.exists(datatypes_config): - # Hack for Pulsar on usegalaxy.org, drop ASAP. - datatypes_config = "configs/registry.xml" - - if not os.path.exists(datatypes_config): - print("Metadata setting failed because registry.xml [%s] could not be found. You may retry setting metadata." % datatypes_config) - sys.exit(1) - import galaxy.datatypes.registry - datatypes_registry = galaxy.datatypes.registry.Registry() - datatypes_registry.load_datatypes(root_dir=galaxy_root, config=datatypes_config) - galaxy.model.set_datatypes_registry(datatypes_registry) - return datatypes_registry - - -def load_job_metadata(job_metadata): - return parse_tool_provided_metadata(job_metadata) - - -def write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_provided_metadata): - for i, file_dict in enumerate(tool_provided_metadata.get_new_datasets_for_metadata_collection(), start=1): - filename = file_dict["filename"] - new_dataset_filename = os.path.join(tool_job_working_directory, "working", filename) - new_dataset = galaxy.model.Dataset(id=-i, external_filename=new_dataset_filename) - extra_files = file_dict.get('extra_files', None) - if extra_files is not None: - new_dataset._extra_files_path = os.path.join(tool_job_working_directory, "working", extra_files) - new_dataset.state = new_dataset.states.OK - new_dataset_instance = galaxy.model.HistoryDatasetAssociation(id=-i, dataset=new_dataset, extension=file_dict.get('ext', 'data')) - set_meta(new_dataset_instance, file_dict) - file_dict['metadata'] = json.loads(new_dataset_instance.metadata.to_JSON_dict()) # storing metadata in external form, need to turn back into dict, then later jsonify - - tool_provided_metadata.rewrite() - clear_mappers() +__all__ = ('set_metadata', ) diff --git a/packages/job_execution/setup.py b/packages/job_execution/setup.py index 8b988e66385..e0d5cb5c879 100644 --- a/packages/job_execution/setup.py +++ b/packages/job_execution/setup.py @@ -36,6 +36,7 @@ PACKAGES = [ ] ENTRY_POINTS = ''' [console_scripts] + galaxy-set-metadata=galaxy.metadata.set_metadata:set_metadata ''' PACKAGE_DATA = { # Be sure to update MANIFEST.in for source dist.