Allow override of provided metadata location.

Eliminates that terrible circular dependency between tools and jobs.
2026-09-24 16:30:27 +08:00 · 2017-08-17 15:41:11 -04:00
parent 39d4411f8c
commit a8bec7a275
7 changed files with 79 additions and 54 deletions
@@ -39,10 +39,9 @@ from .output_checker import check_output

 log = logging.getLogger(__name__)

-# This file, if created in the job's working directory, will be used for
-# setting advanced metadata properties on the job and its associated outputs.
-# This interface is currently experimental, is only used by the upload tool,
-# and should eventually become API'd
+# Legacy definition - this is read by certain misbehaving tool wrappers
+# that import Galaxy internals - but it shouldn't be used in Galaxy's code
+# itself.
 TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'

 # Override with config.default_job_shell.
@@ -1622,46 +1621,10 @@ class JobWrapper(object, HasResourceParameters):
        if self.tool_provided_job_metadata is not None:
            return self.tool_provided_job_metadata

-<<<<<<< HEAD
-        # Look for JSONified job metadata
-        self.tool_provided_job_metadata = []
-        meta_file = os.path.join(self.tool_working_directory, TOOL_PROVIDED_JOB_METADATA_FILE)
-        # LEGACY: Remove in 17.XX
-        if not os.path.exists(meta_file):
-            # Maybe this is a legacy job, use the job working directory instead
-            meta_file = os.path.join(self.working_directory, TOOL_PROVIDED_JOB_METADATA_FILE)
-
-        if os.path.exists(meta_file):
-            for line in open(meta_file, 'r'):
-                try:
-                    line = loads(line)
-                    assert 'type' in line
-                except:
-                    log.exception('(%s) Got JSON data from tool, but data is improperly formatted or no "type" key in data' % self.job_id)
-                    log.debug('Offending data was: %s' % line)
-                    continue
-                # Set the dataset id if it's a dataset entry and isn't set.
-                # This isn't insecure.  We loop the job's output datasets in
-                # the finish method, so if a tool writes out metadata for a
-                # dataset id that it doesn't own, it'll just be ignored.
-                if line['type'] == 'dataset' and 'dataset_id' not in line:
-                    try:
-                        line['dataset_id'] = self.get_output_file_id(line['dataset'])
-                    except KeyError:
-                        log.warning('(%s) Tool provided job dataset-specific metadata without specifying a dataset' % self.job_id)
-                        continue
-                self.tool_provided_job_metadata.append(line)
+        self.tool_provided_job_metadata = self.tool.tool_provided_metadata(self)
        return self.tool_provided_job_metadata

-    def get_dataset_finish_context(self, job_context, dataset):
-        for meta in self.get_tool_provided_job_metadata():
-            if meta['type'] == 'dataset' and meta['dataset_id'] == dataset.id:
-                return ExpressionContext(meta, job_context)
-=======
-        self.tool_provided_job_metadata = self.tool.tool_provided_metadata( self )
-        return self.tool_provided_job_metadata
-
-    def get_dataset_finish_context( self, job_context, output_dataset_assoc ):
+    def get_dataset_finish_context(self, job_context, output_dataset_assoc):
        meta = {}
        tool_provided_metadata = self.get_tool_provided_job_metadata()
        if hasattr(tool_provided_metadata, "get_meta_by_dataset_id"):
@@ -1670,8 +1633,7 @@ class JobWrapper(object, HasResourceParameters):
            meta = tool_provided_metadata.get_meta_by_name(output_dataset_assoc.name)

        if meta:
-            return ExpressionContext( meta, job_context )
->>>>>>> a73a0275a9... Overhaul of tool provided job metadata.
+            return ExpressionContext(meta, job_context)
        return job_context

    def invalidate_external_metadata(self):
@@ -1691,13 +1653,8 @@ class JobWrapper(object, HasResourceParameters):
        if set_extension:
            for output_dataset_assoc in job.output_datasets:
                if output_dataset_assoc.dataset.ext == 'auto':
-<<<<<<< HEAD
-                    context = self.get_dataset_finish_context(dict(), output_dataset_assoc.dataset.dataset)
+                    context = self.get_dataset_finish_context(dict(), output_dataset_assoc)
                    output_dataset_assoc.dataset.extension = context.get('ext', 'data')
-=======
-                    context = self.get_dataset_finish_context( dict(), output_dataset_assoc )
-                    output_dataset_assoc.dataset.extension = context.get( 'ext', 'data' )
->>>>>>> a73a0275a9... Overhaul of tool provided job metadata.
            self.sa_session.flush()
        if tmp_dir is None:
            # this dir should should relative to the exec_dir
@@ -1720,7 +1677,7 @@ class JobWrapper(object, HasResourceParameters):
                                                                        config_root=config_root,
                                                                        config_file=config_file,
                                                                        datatypes_config=datatypes_config,
-                                                                        job_metadata=os.path.join(self.tool_working_directory, TOOL_PROVIDED_JOB_METADATA_FILE),
+                                                                        job_metadata=os.path.join(self.tool_working_directory, self.tool.provided_metadata_file),
                                                                        max_metadata_value_size=self.app.config.max_metadata_value_size,
                                                                        **kwds)
        if resolve_metadata_dependencies:
@@ -19,7 +19,6 @@ from paste import httpexceptions
 from six import string_types
 from six.moves.urllib.parse import unquote_plus

-import galaxy.jobs
 import tool_shed.util.repository_util as repository_util
 import tool_shed.util.shed_util_common

@@ -687,6 +686,8 @@ class Tool(object, Dictifiable):
        )
        self.options = Bunch(** self.options)

+        # Read in name of galaxy.json metadata file and how to parse it.
+        self.provided_metadata_file = tool_source.parse_provided_metadata_file()
        self.provided_metadata_style = tool_source.parse_provided_metadata_style()

        # Parse tool inputs (if there are any required)
@@ -808,11 +809,19 @@ class Tool(object, Dictifiable):
        return self.__tests

    def tool_provided_metadata(self, job_wrapper):
+<<<<<<< HEAD
        meta_file = os.path.join(job_wrapper.tool_working_directory, galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE)
+=======
+        meta_file = os.path.join( job_wrapper.tool_working_directory, self.provided_metadata_file )
+>>>>>>> 0df820586b... Allow override of provided metadata location.
        # LEGACY: Remove in 17.XX
        if not os.path.exists(meta_file):
            # Maybe this is a legacy job, use the job working directory instead
+<<<<<<< HEAD
            meta_file = os.path.join(job_wrapper.working_directory, galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE)
+=======
+            meta_file = os.path.join( job_wrapper.working_directory, self.provided_metadata_file )
+>>>>>>> 0df820586b... Allow override of provided metadata location.

        if not os.path.exists(meta_file):
            return output_collect.NullToolProvidedMetadata()
@@ -2082,7 +2091,7 @@ class OutputParameterJSONTool(Tool):
        json_params = {}
        json_params['param_dict'] = self._prepare_json_param_dict(param_dict)  # it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones?
        json_params['output_data'] = []
-        json_params['job_config'] = dict(GALAXY_DATATYPES_CONF_FILE=param_dict.get('GALAXY_DATATYPES_CONF_FILE'), GALAXY_ROOT_DIR=param_dict.get('GALAXY_ROOT_DIR'), TOOL_PROVIDED_JOB_METADATA_FILE=galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE)
+        json_params['job_config'] = dict(GALAXY_DATATYPES_CONF_FILE=param_dict.get('GALAXY_DATATYPES_CONF_FILE'), GALAXY_ROOT_DIR=param_dict.get('GALAXY_ROOT_DIR'), TOOL_PROVIDED_JOB_METADATA_FILE=self.provided_metadata_file)
        json_filename = None
        for i, (out_name, data) in enumerate(out_data.items()):
            # use wrapped dataset to access certain values
@@ -2134,7 +2143,7 @@ class DataSourceTool(OutputParameterJSONTool):
        json_params = {}
        json_params['param_dict'] = self._prepare_json_param_dict(param_dict)  # it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones?
        json_params['output_data'] = []
-        json_params['job_config'] = dict(GALAXY_DATATYPES_CONF_FILE=param_dict.get('GALAXY_DATATYPES_CONF_FILE'), GALAXY_ROOT_DIR=param_dict.get('GALAXY_ROOT_DIR'), TOOL_PROVIDED_JOB_METADATA_FILE=galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE)
+        json_params['job_config'] = dict(GALAXY_DATATYPES_CONF_FILE=param_dict.get('GALAXY_DATATYPES_CONF_FILE'), GALAXY_ROOT_DIR=param_dict.get('GALAXY_ROOT_DIR'), TOOL_PROVIDED_JOB_METADATA_FILE=self.provided_metadata_file)
        json_filename = None
        for i, (out_name, data) in enumerate(out_data.items()):
            # use wrapped dataset to access certain values
@@ -155,6 +155,10 @@ class ToolSource(object):
        """
        return "default"

+    def parse_provided_metadata_file(self):
+        """Return location of provided metadata file (e.g. galaxy.json)."""
+        return "galaxy.json"
+
    @abstractmethod
    def parse_outputs(self, tool):
        """ Return a pair of output and output collections ordered
@@ -209,6 +209,14 @@ class XmlToolSource(ToolSource):
        assert style in ["legacy", "default"]
        return style

+    def parse_provided_metadata_file(self):
+        provided_metadata_file = "galaxy.json"
+        out_elem = self.root.find("outputs")
+        if out_elem and "provided_metadata_file" in out_elem.attrib:
+            provided_metadata_file = out_elem.attrib["provided_metadata_file"]
+
+        return provided_metadata_file
+
    def parse_outputs(self, tool):
        out_elem = self.root.find("outputs")
        outputs = odict()
@@ -3569,6 +3569,17 @@ the ``<inputs>`` documentation.
          tools and tools without a specified profile. A discussion of the
          differences between the styles can be found at
          https://github.com/galaxyproject/galaxy/pull/4437.
+]]></xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="provided_metadata_file" type="xs:string" default="galaxy.json">
+      <xs:annotation>
+        <xs:documentation xml:lang="en"><![CDATA[
+          Path relative to tool's working directory to load tool provided
+          metadata from. This metadata can describe dynamic datasets to load,
+          dynamic collection contents, as well as simple metadata (e.g. name, dbkey, etc...) and datatype specific metadata for declared outputs.
+          More information can be found [here](http://planemo.readthedocs.io/en/latest/writing_advanced.html#tool-provided-metadata). The
+          default is galaxy.json.
 ]]></xs:documentation>
      </xs:annotation>
    </xs:attribute>
@@ -23,6 +23,7 @@
  <tool file="tool_provided_metadata_6.xml" />
  <tool file="tool_provided_metadata_7.xml" />
  <tool file="tool_provided_metadata_8.xml" />
+  <tool file="tool_provided_metadata_9.xml" />
  <tool file="inputs_as_json.xml" />
  <tool file="dbkey_filter_input.xml" />
  <tool file="dbkey_filter_multi_input.xml" />
@@ -0,0 +1,35 @@
+<tool id="tool_provided_metadata_9" name="tool_provided_metadata_9" profile="17.09" version="1.0.0">
+    <!-- Demonstrate overriding the location of the tool provided metadata file to
+         be something other than galaxy.json. -->
+    <command>
+      echo "This is a line of text." > $out1;
+      cp $c1 not_galaxy.json;
+    </command>
+    <configfiles>
+      <configfile name="c1">{"out1": {
+  "name": "my dynamic name",
+  "ext": "txt",
+  "info": "my dynamic info",
+  "dbkey": "cust1"
+}}
+</configfile>
+    </configfiles>
+    <inputs>
+        <param name="input1" type="data" label="Input Dataset"/>
+    </inputs>
+    <outputs provided_metadata_file="not_galaxy.json">
+        <data name="out1" format="auto" />
+    </outputs>
+    <help>
+    </help>
+    <tests>
+      <test>
+        <param name="input1" value="simple_line.txt" />
+        <output name="out1" file="simple_line.txt" ftype="txt">
+          <metadata name="name" value="my dynamic name" />
+          <metadata name="info" value="my dynamic info" />
+          <metadata name="dbkey" value="cust1" />
+        </output>
+      </test>
+    </tests>
+</tool>