diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 6522f2c3035..d0d8914d3d4 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -39,10 +39,9 @@ from .output_checker import check_output log = logging.getLogger(__name__) -# This file, if created in the job's working directory, will be used for -# setting advanced metadata properties on the job and its associated outputs. -# This interface is currently experimental, is only used by the upload tool, -# and should eventually become API'd +# Legacy definition - this is read by certain misbehaving tool wrappers +# that import Galaxy internals - but it shouldn't be used in Galaxy's code +# itself. TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json' # Override with config.default_job_shell. @@ -1622,46 +1621,10 @@ class JobWrapper(object, HasResourceParameters): if self.tool_provided_job_metadata is not None: return self.tool_provided_job_metadata -<<<<<<< HEAD - # Look for JSONified job metadata - self.tool_provided_job_metadata = [] - meta_file = os.path.join(self.tool_working_directory, TOOL_PROVIDED_JOB_METADATA_FILE) - # LEGACY: Remove in 17.XX - if not os.path.exists(meta_file): - # Maybe this is a legacy job, use the job working directory instead - meta_file = os.path.join(self.working_directory, TOOL_PROVIDED_JOB_METADATA_FILE) - - if os.path.exists(meta_file): - for line in open(meta_file, 'r'): - try: - line = loads(line) - assert 'type' in line - except: - log.exception('(%s) Got JSON data from tool, but data is improperly formatted or no "type" key in data' % self.job_id) - log.debug('Offending data was: %s' % line) - continue - # Set the dataset id if it's a dataset entry and isn't set. - # This isn't insecure. We loop the job's output datasets in - # the finish method, so if a tool writes out metadata for a - # dataset id that it doesn't own, it'll just be ignored. - if line['type'] == 'dataset' and 'dataset_id' not in line: - try: - line['dataset_id'] = self.get_output_file_id(line['dataset']) - except KeyError: - log.warning('(%s) Tool provided job dataset-specific metadata without specifying a dataset' % self.job_id) - continue - self.tool_provided_job_metadata.append(line) + self.tool_provided_job_metadata = self.tool.tool_provided_metadata(self) return self.tool_provided_job_metadata - def get_dataset_finish_context(self, job_context, dataset): - for meta in self.get_tool_provided_job_metadata(): - if meta['type'] == 'dataset' and meta['dataset_id'] == dataset.id: - return ExpressionContext(meta, job_context) -======= - self.tool_provided_job_metadata = self.tool.tool_provided_metadata( self ) - return self.tool_provided_job_metadata - - def get_dataset_finish_context( self, job_context, output_dataset_assoc ): + def get_dataset_finish_context(self, job_context, output_dataset_assoc): meta = {} tool_provided_metadata = self.get_tool_provided_job_metadata() if hasattr(tool_provided_metadata, "get_meta_by_dataset_id"): @@ -1670,8 +1633,7 @@ class JobWrapper(object, HasResourceParameters): meta = tool_provided_metadata.get_meta_by_name(output_dataset_assoc.name) if meta: - return ExpressionContext( meta, job_context ) ->>>>>>> a73a0275a9... Overhaul of tool provided job metadata. + return ExpressionContext(meta, job_context) return job_context def invalidate_external_metadata(self): @@ -1691,13 +1653,8 @@ class JobWrapper(object, HasResourceParameters): if set_extension: for output_dataset_assoc in job.output_datasets: if output_dataset_assoc.dataset.ext == 'auto': -<<<<<<< HEAD - context = self.get_dataset_finish_context(dict(), output_dataset_assoc.dataset.dataset) + context = self.get_dataset_finish_context(dict(), output_dataset_assoc) output_dataset_assoc.dataset.extension = context.get('ext', 'data') -======= - context = self.get_dataset_finish_context( dict(), output_dataset_assoc ) - output_dataset_assoc.dataset.extension = context.get( 'ext', 'data' ) ->>>>>>> a73a0275a9... Overhaul of tool provided job metadata. self.sa_session.flush() if tmp_dir is None: # this dir should should relative to the exec_dir @@ -1720,7 +1677,7 @@ class JobWrapper(object, HasResourceParameters): config_root=config_root, config_file=config_file, datatypes_config=datatypes_config, - job_metadata=os.path.join(self.tool_working_directory, TOOL_PROVIDED_JOB_METADATA_FILE), + job_metadata=os.path.join(self.tool_working_directory, self.tool.provided_metadata_file), max_metadata_value_size=self.app.config.max_metadata_value_size, **kwds) if resolve_metadata_dependencies: diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index e24b343346f..5b68499e1fe 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -19,7 +19,6 @@ from paste import httpexceptions from six import string_types from six.moves.urllib.parse import unquote_plus -import galaxy.jobs import tool_shed.util.repository_util as repository_util import tool_shed.util.shed_util_common @@ -687,6 +686,8 @@ class Tool(object, Dictifiable): ) self.options = Bunch(** self.options) + # Read in name of galaxy.json metadata file and how to parse it. + self.provided_metadata_file = tool_source.parse_provided_metadata_file() self.provided_metadata_style = tool_source.parse_provided_metadata_style() # Parse tool inputs (if there are any required) @@ -808,11 +809,19 @@ class Tool(object, Dictifiable): return self.__tests def tool_provided_metadata(self, job_wrapper): +<<<<<<< HEAD meta_file = os.path.join(job_wrapper.tool_working_directory, galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE) +======= + meta_file = os.path.join( job_wrapper.tool_working_directory, self.provided_metadata_file ) +>>>>>>> 0df820586b... Allow override of provided metadata location. # LEGACY: Remove in 17.XX if not os.path.exists(meta_file): # Maybe this is a legacy job, use the job working directory instead +<<<<<<< HEAD meta_file = os.path.join(job_wrapper.working_directory, galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE) +======= + meta_file = os.path.join( job_wrapper.working_directory, self.provided_metadata_file ) +>>>>>>> 0df820586b... Allow override of provided metadata location. if not os.path.exists(meta_file): return output_collect.NullToolProvidedMetadata() @@ -2082,7 +2091,7 @@ class OutputParameterJSONTool(Tool): json_params = {} json_params['param_dict'] = self._prepare_json_param_dict(param_dict) # it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones? json_params['output_data'] = [] - json_params['job_config'] = dict(GALAXY_DATATYPES_CONF_FILE=param_dict.get('GALAXY_DATATYPES_CONF_FILE'), GALAXY_ROOT_DIR=param_dict.get('GALAXY_ROOT_DIR'), TOOL_PROVIDED_JOB_METADATA_FILE=galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE) + json_params['job_config'] = dict(GALAXY_DATATYPES_CONF_FILE=param_dict.get('GALAXY_DATATYPES_CONF_FILE'), GALAXY_ROOT_DIR=param_dict.get('GALAXY_ROOT_DIR'), TOOL_PROVIDED_JOB_METADATA_FILE=self.provided_metadata_file) json_filename = None for i, (out_name, data) in enumerate(out_data.items()): # use wrapped dataset to access certain values @@ -2134,7 +2143,7 @@ class DataSourceTool(OutputParameterJSONTool): json_params = {} json_params['param_dict'] = self._prepare_json_param_dict(param_dict) # it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones? json_params['output_data'] = [] - json_params['job_config'] = dict(GALAXY_DATATYPES_CONF_FILE=param_dict.get('GALAXY_DATATYPES_CONF_FILE'), GALAXY_ROOT_DIR=param_dict.get('GALAXY_ROOT_DIR'), TOOL_PROVIDED_JOB_METADATA_FILE=galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE) + json_params['job_config'] = dict(GALAXY_DATATYPES_CONF_FILE=param_dict.get('GALAXY_DATATYPES_CONF_FILE'), GALAXY_ROOT_DIR=param_dict.get('GALAXY_ROOT_DIR'), TOOL_PROVIDED_JOB_METADATA_FILE=self.provided_metadata_file) json_filename = None for i, (out_name, data) in enumerate(out_data.items()): # use wrapped dataset to access certain values diff --git a/lib/galaxy/tools/parser/interface.py b/lib/galaxy/tools/parser/interface.py index c37452d5579..a86ce1b4ece 100644 --- a/lib/galaxy/tools/parser/interface.py +++ b/lib/galaxy/tools/parser/interface.py @@ -155,6 +155,10 @@ class ToolSource(object): """ return "default" + def parse_provided_metadata_file(self): + """Return location of provided metadata file (e.g. galaxy.json).""" + return "galaxy.json" + @abstractmethod def parse_outputs(self, tool): """ Return a pair of output and output collections ordered diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py index 02530ff2262..114b4598879 100644 --- a/lib/galaxy/tools/parser/xml.py +++ b/lib/galaxy/tools/parser/xml.py @@ -209,6 +209,14 @@ class XmlToolSource(ToolSource): assert style in ["legacy", "default"] return style + def parse_provided_metadata_file(self): + provided_metadata_file = "galaxy.json" + out_elem = self.root.find("outputs") + if out_elem and "provided_metadata_file" in out_elem.attrib: + provided_metadata_file = out_elem.attrib["provided_metadata_file"] + + return provided_metadata_file + def parse_outputs(self, tool): out_elem = self.root.find("outputs") outputs = odict() diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index 9c430a2aea4..45c2f65aa20 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -3569,6 +3569,17 @@ the ```` documentation. tools and tools without a specified profile. A discussion of the differences between the styles can be found at https://github.com/galaxyproject/galaxy/pull/4437. +]]> + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index ee8e9017ef2..18e55f282f1 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -23,6 +23,7 @@ + diff --git a/test/functional/tools/tool_provided_metadata_9.xml b/test/functional/tools/tool_provided_metadata_9.xml new file mode 100644 index 00000000000..797ede1c871 --- /dev/null +++ b/test/functional/tools/tool_provided_metadata_9.xml @@ -0,0 +1,35 @@ + + + + echo "This is a line of text." > $out1; + cp $c1 not_galaxy.json; + + + {"out1": { + "name": "my dynamic name", + "ext": "txt", + "info": "my dynamic info", + "dbkey": "cust1" +}} + + + + + + + + + + + + + + + + + + + + +