Merge pull request #2697 from jmchilton/metadata_fixing

Fixes and improvements for galaxy.json.
2026-09-24 16:30:27 +08:00 · 2016-08-17 09:12:57 +02:00
parent 0210fb31b6 1e8dd73b6d
commit 5cc3613ff5
9 changed files with 162 additions and 25 deletions
@@ -1298,11 +1298,10 @@ class JobWrapper( object ):
                            dataset.set_peek( is_multi_byte=True )
                        else:
                            dataset.set_peek()
-                    try:
-                        # set the name if provided by the tool
-                        dataset.name = context['name']
-                    except:
-                        pass
+                    for context_key in ['name', 'info', 'dbkey']:
+                        if context_key in context:
+                            context_value = context[context_key]
+                            setattr(dataset, context_key, context_value)
                else:
                    dataset.blurb = "empty"
                    if dataset.ext == 'auto':
@@ -314,6 +314,8 @@ def collect_primary_datasets( tool, output, job_working_directory, input_ext, in
                            )
            metadata_dict = new_primary_datasets_attributes.get( 'metadata', None )
            if metadata_dict:
+                if "dbkey" in new_primary_datasets_attributes:
+                    metadata_dict["dbkey"] = new_primary_datasets_attributes["dbkey"]
                primary_data.metadata.from_JSON_dict( json_dict=metadata_dict )
            else:
                primary_data.set_meta()
@@ -391,15 +391,7 @@ def __parse_output_elem( output_elem ):
    if name is None:
        raise Exception( "Test output does not have a 'name'" )

-    file, attributes = __parse_test_attributes( output_elem, attrib )
-    primary_datasets = {}
-    for primary_elem in ( output_elem.findall( "discovered_dataset" ) or [] ):
-        primary_attrib = dict( primary_elem.attrib )
-        designation = primary_attrib.pop( 'designation', None )
-        if designation is None:
-            raise Exception( "Test primary dataset does not have a 'designation'" )
-        primary_datasets[ designation ] = __parse_test_attributes( primary_elem, primary_attrib )
-    attributes[ "primary_datasets" ] = primary_datasets
+    file, attributes = __parse_test_attributes( output_elem, attrib, parse_discovered_datasets=True )
    return name, file, attributes


@@ -436,7 +428,7 @@ def __parse_element_tests( parent_element ):
    return element_tests


-def __parse_test_attributes( output_elem, attrib, parse_elements=False ):
+def __parse_test_attributes( output_elem, attrib, parse_elements=False, parse_discovered_datasets=False ):
    assert_list = __parse_assert_list( output_elem )

    # Allow either file or value to specify a target file to compare result with
@@ -466,8 +458,18 @@ def __parse_test_attributes( output_elem, attrib, parse_elements=False ):
    if parse_elements:
        element_tests = __parse_element_tests( output_elem )

+    primary_datasets = {}
+    if parse_discovered_datasets:
+        for primary_elem in ( output_elem.findall( "discovered_dataset" ) or [] ):
+            primary_attrib = dict( primary_elem.attrib )
+            designation = primary_attrib.pop( 'designation', None )
+            if designation is None:
+                raise Exception( "Test primary dataset does not have a 'designation'" )
+            primary_datasets[ designation ] = __parse_test_attributes( primary_elem, primary_attrib )
+
    has_checksum = md5sum or checksum
-    if not (assert_list or file or extra_files or metadata or has_checksum or element_tests):
+    has_nested_tests = extra_files or element_tests or primary_datasets
+    if not (assert_list or file or metadata or has_checksum or has_nested_tests):
        raise Exception( "Test output defines nothing to check (e.g. must have a 'file' check against, assertions to check, metadata or checksum tests, etc...)")
    attributes['assert_list'] = assert_list
    attributes['extra_files'] = extra_files
@@ -475,6 +477,7 @@ def __parse_test_attributes( output_elem, attrib, parse_elements=False ):
    attributes['md5'] = md5sum
    attributes['checksum'] = checksum
    attributes['elements'] = element_tests
+    attributes['primary_datasets'] = primary_datasets
    return file, attributes


@@ -141,10 +141,11 @@ def set_metadata():
            json.dump( ( False, str( e ) ), open( filename_results_code, 'wb+' ) )  # setting metadata has failed somehow

    for i, ( filename, file_dict ) in enumerate( new_job_metadata_dict.iteritems(), start=1 ):
-        new_dataset = galaxy.model.Dataset( id=-i, external_filename=os.path.join( tool_job_working_directory, file_dict[ 'filename' ] ) )
+        new_dataset_filename = os.path.join( tool_job_working_directory, "working", file_dict[ 'filename' ] )
+        new_dataset = galaxy.model.Dataset( id=-i, external_filename=new_dataset_filename )
        extra_files = file_dict.get( 'extra_files', None )
        if extra_files is not None:
-            new_dataset._extra_files_path = os.path.join( tool_job_working_directory, extra_files )
+            new_dataset._extra_files_path = os.path.join( tool_job_working_directory, "working", extra_files )
        new_dataset.state = new_dataset.states.OK
        new_dataset_instance = galaxy.model.HistoryDatasetAssociation( id=-i, dataset=new_dataset, extension=file_dict.get( 'ext', 'data' ) )
        set_meta_with_tool_provided( new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry )
@@ -7,6 +7,7 @@ from logging import getLogger

 from requests import get, post, delete, patch
 from six import StringIO
+from six import text_type

 from galaxy import util
 from galaxy.tools.parser.interface import TestCollectionDef
@@ -87,27 +88,45 @@ class GalaxyInteractorApi( object ):

    def verify_output_dataset( self, history_id, hda_id, outfile, attributes, shed_tool_id ):
        fetcher = self.__dataset_fetcher( history_id )
-        self.twill_test_case.verify_hid( outfile, hda_id=hda_id, attributes=attributes, dataset_fetcher=fetcher, shed_tool_id=shed_tool_id )
+        self.twill_test_case.verify_hid(
+            outfile,
+            hda_id=hda_id,
+            attributes=attributes,
+            dataset_fetcher=fetcher,
+            shed_tool_id=shed_tool_id
+        )
        self._verify_metadata( history_id, hda_id, attributes )

    def _verify_metadata( self, history_id, hid, attributes ):
+        """Check dataset metadata.
+
+        ftype on output maps to `file_ext` on the hda's API description, `name`, `info`,
+        and `dbkey` all map to the API description directly. Other metadata attributes
+        are assumed to be datatype-specific and mapped with a prefix of `metadata_`.
+        """
        metadata = attributes.get( 'metadata', {} ).copy()
        for key, value in metadata.copy().items():
-            new_key = "metadata_%s" % key
-            metadata[ new_key ] = metadata[ key ]
-            del metadata[ key ]
+            if key not in ['name', 'info']:
+                new_key = "metadata_%s" % key
+                metadata[ new_key ] = metadata[ key ]
+                del metadata[ key ]
+            elif key == "info":
+                metadata[ "misc_info" ] = metadata[ "info" ]
+                del metadata[ "info" ]
        expected_file_type = attributes.get( 'ftype', None )
        if expected_file_type:
            metadata[ "file_ext" ] = expected_file_type

        if metadata:
+            import time
+            time.sleep(5)
            dataset = self._get( "histories/%s/contents/%s" % ( history_id, hid ) ).json()
            for key, value in metadata.items():
                try:
                    dataset_value = dataset.get( key, None )
-                    if dataset_value != value:
-                        msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s]."
-                        msg_params = ( key, value, dataset_value )
+                    if text_type(dataset_value) != text_type(value):
+                        msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s]. Dataset API value was [%s]."
+                        msg_params = ( key, value, dataset_value, dataset )
                        msg = msg % msg_params
                        raise Exception( msg )
                except KeyError:
@@ -15,6 +15,9 @@
  <tool file="multi_output.xml" />
  <tool file="multi_output_configured.xml" />
  <tool file="multi_output_assign_primary.xml" />
+  <tool file="tool_provided_metadata_1.xml" />
+  <tool file="tool_provided_metadata_2.xml" />
+  <tool file="tool_provided_metadata_3.xml" />
  <tool file="inputs_as_json.xml" />
  <tool file="dbkey_filter_input.xml" />
  <tool file="dbkey_filter_multi_input.xml" />
@@ -0,0 +1,29 @@
+<tool id="tool_provided_metadata_1" name="tool_provided_metadata_1">
+    <command>
+      echo "This is a line of text." > $out1;
+      cp $c1 galaxy.json;
+    </command>
+    <configfiles>
+      <configfile name="c1">{"type": "dataset", "dataset_id": $out1.dataset.dataset.id, "name": "my dynamic name", "ext": "txt", "info": "my dynamic info", "dbkey": "cust1"}</configfile>
+    </configfiles>
+    <inputs>
+        <param name="input1" type="data" label="Input Dataset"/>
+    </inputs>
+    <outputs>
+        <!-- Set format="auto" to read from galaxy.json, use auto_format="true"
+             to sniff. -->
+        <data name="out1" format="auto" />
+    </outputs>
+    <help>
+    </help>
+    <tests>
+      <test>
+        <param name="input1" value="simple_line.txt" />
+        <output name="out1" file="simple_line.txt" ftype="txt">
+          <metadata name="name" value="my dynamic name" />
+          <metadata name="info" value="my dynamic info" />
+          <metadata name="dbkey" value="cust1" />
+        </output>
+      </test>
+    </tests>
+</tool>
@@ -0,0 +1,38 @@
+<tool id="tool_provided_metadata_2" name="tool_provided_metadata_2">
+  <command>
+    echo "1" > sample1.report.tsv;
+    echo "2" > sample2.report.tsv;
+    cp $c1 galaxy.json;
+  </command>
+  <configfiles>
+      <configfile name="c1">{"type": "new_primary_dataset", "filename": "sample1.report.tsv", "name": "cool name 1", "ext": "txt", "info": "cool 1 info", "dbkey": "hg19"}
+{"type": "new_primary_dataset", "filename": "sample2.report.tsv", "name": "cool name 2", "ext": "txt", "info": "cool 2 info", "dbkey": "hg19"}
+</configfile>
+  </configfiles>
+  <inputs>
+    <param name="input" type="data" />
+  </inputs>
+  <outputs>
+    <data name="sample">
+      <discover_datasets pattern="(?P&lt;designation&gt;.+)\.report\.tsv" visible="true" />
+    </data>
+  </outputs>
+  <tests>
+    <test>
+      <param name="input" ftype="txt" value="simple_line.txt"/>
+      <output name="sample">
+        <discovered_dataset designation="sample1" ftype="txt">
+          <assert_contents><has_line line="1" /></assert_contents>
+          <metadata name="name" value="cool name 1" />
+          <metadata name="dbkey" value="hg19" />
+          <metadata name="info" value="cool 1 info" />
+        </discovered_dataset>
+        <discovered_dataset designation="sample2" ftype="txt">
+          <assert_contents><has_line line="2" /></assert_contents>
+          <metadata name="name" value="cool name 2" />
+          <metadata name="info" value="cool 2 info" />
+        </discovered_dataset>
+      </output>
+    </test>
+  </tests>
+</tool>
@@ -0,0 +1,43 @@
+<tool id="tool_provided_metadata_3" name="tool_provided_metadata_3">
+  <command>
+    echo "1" > sample1.report.tsv;
+    echo "2" > sample2.report.tsv;
+    cp $c1 galaxy.json;
+  </command>
+  <configfiles>
+      <configfile name="c1">{"type": "new_primary_dataset", "filename": "sample1.report.tsv", "name": "cool name 1", "ext": "txt", "info": "cool 1 info", "dbkey": "hg19", "metadata": {"data_lines": 10, "foo": "bar"}}
+{"type": "new_primary_dataset", "filename": "sample2.report.tsv", "name": "cool name 2", "ext": "txt", "info": "cool 2 info", "dbkey": "hg19", "metadata": {"data_lines": 20, "foo": "bar"}}
+</configfile>
+  </configfiles>
+  <inputs>
+    <param name="input" type="data" />
+  </inputs>
+  <outputs>
+    <data name="sample">
+      <discover_datasets pattern="(?P&lt;designation&gt;.+)\.report\.tsv" visible="true" />
+    </data>
+  </outputs>
+  <tests>
+    <test>
+      <param name="input" ftype="txt" value="simple_line.txt"/>
+      <output name="sample">
+        <discovered_dataset designation="sample1" ftype="txt">
+          <assert_contents><has_line line="1" /></assert_contents>
+          <!-- Datatype defined metadata can be overridden/specified directly.
+          -->
+          <metadata name="data_lines" value="10" />
+          <!-- Non-datatype defined metadata values are ignored by the framework.
+               Uncommenting the following test will break this test.
+          -->
+          <!--
+          <metadata name="foo" value="bar" />
+          -->
+        </discovered_dataset>
+        <discovered_dataset designation="sample2" ftype="txt">
+          <assert_contents><has_line line="2" /></assert_contents>
+          <metadata name="data_lines" value="20" />
+        </discovered_dataset>
+      </output>
+    </test>
+  </tests>
+</tool>