Merge pull request #7670 from jmchilton/created_from_basename

Track basename at the time of creation for datasets.

lib/galaxy/jobs/__init__.py

@@ -46,7 +46,7 @@ log = logging.getLogger(__name__)
 # that import Galaxy internals - but it shouldn't be used in Galaxy's code
 # itself.
 TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'
-TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey']
+TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey', 'created_from_basename']
 
 # Override with config.default_job_shell.
 DEFAULT_JOB_SHELL = '/bin/bash'

lib/galaxy/managers/hdas.py

@@ -300,7 +300,8 @@ class HDASerializer(  # datasets._UnflattenedMetadataDatasetAssociationSerialize
 
             'annotation',
 
-            'api_type'
+            'api_type',
+            'created_from_basename',
         ], include_keys_from='summary')
 
         self.add_view('extended', [
@@ -354,7 +355,8 @@ class HDASerializer(  # datasets._UnflattenedMetadataDatasetAssociationSerialize
             # TODO: to DatasetAssociationSerializer
             'accessible'    : lambda i, k, user=None, **c: self.manager.is_accessible(i, user, **c),
             'api_type'      : lambda *a, **c: 'file',
-            'type'          : lambda *a, **c: 'file'
+            'type'          : lambda *a, **c: 'file',
+            'created_from_basename' : lambda i, k, **c: i.created_from_basename,
         })
 
     def serialize(self, hda, keys, user=None, **context):

lib/galaxy/model/__init__.py

@@ -2510,6 +2510,16 @@ class DatasetInstance(object):
         """Detects whether there is any data"""
         return self.dataset.has_data()
 
+    def get_created_from_basename(self):
+        return self.dataset.created_from_basename
+
+    def set_created_from_basename(self, created_from_basename):
+        if self.dataset.created_from_basename is not None:
+            raise Exception("Underlying dataset already has a created_from_basename set.")
+        self.dataset.created_from_basename = created_from_basename
+
+    created_from_basename = property(get_created_from_basename, set_created_from_basename)
+
     def get_raw_data(self):
         """Returns the full data. To stream it open the file_name and read/write as needed"""
         return self.datatype.get_raw_data(self)
@@ -3393,6 +3403,7 @@ class LibraryDataset(RepresentById):
                     state=ldda.state,
                     name=ldda.name,
                     file_name=ldda.file_name,
+                    created_from_basename=ldda.created_from_basename,
                     uploaded_by=ldda.user.email,
                     message=ldda.message,
                     date_uploaded=ldda.create_time.isoformat(),
@@ -3545,7 +3556,8 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById):
                     data_type=ldda.datatype.__class__.__module__ + '.' + ldda.datatype.__class__.__name__,
                     genome_build=ldda.dbkey,
                     misc_info=ldda.info,
-                    misc_blurb=ldda.blurb)
+                    misc_blurb=ldda.blurb,
+                    created_from_basename=ldda.created_from_basename)
         if ldda.dataset.uuid is None:
             rval['uuid'] = None
         else:

lib/galaxy/model/mapping.py

@@ -263,6 +263,7 @@ model.Dataset.table = Table(
     Column("object_store_id", TrimmedString(255), index=True),
     Column("external_filename", TEXT),
     Column("_extra_files_path", TEXT),
+    Column("created_from_basename", TEXT),
     Column('file_size', Numeric(15, 0)),
     Column('total_size', Numeric(15, 0)),
     Column('uuid', UUIDType()))

lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py

@@ -6,9 +6,10 @@ from __future__ import print_function
 
 import logging
 
-from sqlalchemy import Column, MetaData, Table
+from sqlalchemy import Column, MetaData
 
 from galaxy.model.custom_types import UUIDType
+from galaxy.model.migrate.versions.util import add_column, drop_column
 
 log = logging.getLogger(__name__)
 
@@ -19,14 +20,8 @@ def upgrade(migrate_engine):
     metadata.bind = migrate_engine
     metadata.reflect()
 
-    metadata_file_table = Table("metadata_file", metadata, autoload=True)
-
-    try:
-        uuid_column = Column('uuid', UUIDType())
-        uuid_column.create(metadata_file_table)
-        assert uuid_column is metadata_file_table.c.uuid
-    except Exception:
-        log.exception("Adding column 'uuid' to `MetadataFile` table failed.")
+    uuid_column = Column('uuid', UUIDType())
+    add_column(uuid_column, 'metadata_file', metadata)
 
 
 def downgrade(migrate_engine):
@@ -34,10 +29,4 @@ def downgrade(migrate_engine):
     metadata.bind = migrate_engine
     metadata.reflect()
 
-    metadata_file_table = Table("metadata_file", metadata, autoload=True)
-
-    try:
-        column = metadata_file_table.c.uuid
-        column.drop()
-    except Exception:
-        log.exception("Dropping 'uuid' column from `metadata_file` table failed.")
+    drop_column('uuid', 'metadata_file', metadata)

lib/galaxy/model/migrate/versions/0154_created_from_basename.py

@@ -0,0 +1,29 @@
+"""
+Adds created_from_basename to dataset.
+"""
+import datetime
+import logging
+
+from sqlalchemy import Column, MetaData, TEXT
+
+from galaxy.model.migrate.versions.util import add_column, drop_column
+
+now = datetime.datetime.utcnow
+log = logging.getLogger(__name__)
+metadata = MetaData()
+
+
+def upgrade(migrate_engine):
+    metadata.bind = migrate_engine
+    print(__doc__)
+    metadata.reflect()
+
+    created_from_basename_column = Column("created_from_basename", TEXT, default=None)
+    add_column(created_from_basename_column, 'dataset', metadata)
+
+
+def downgrade(migrate_engine):
+    metadata.bind = migrate_engine
+    metadata.reflect()
+
+    drop_column('created_from_basename', 'dataset', metadata)

lib/galaxy/model/store/discover.py

@@ -55,6 +55,7 @@ class ModelPersistenceContext(object):
         tag_list=[],
         sources=[],
         hashes=[],
+        created_from_basename=None,
     ):
         sa_session = self.sa_session
 
@@ -116,6 +117,9 @@ class ModelPersistenceContext(object):
             hash_object.hash_value = hash_dict["hash_value"]
             primary_data.dataset.hashes.append(hash_object)
 
+        if created_from_basename is not None:
+            primary_data.created_from_basename = created_from_basename
+
         self.flush()
 
         if tag_list:
@@ -204,6 +208,7 @@ class ModelPersistenceContext(object):
 
             sources = discovered_file.match.sources
             hashes = discovered_file.match.hashes
+            created_from_basename = discovered_file.match.created_from_basename
 
             dataset = self.create_dataset(
                 ext=ext,
@@ -217,6 +222,7 @@ class ModelPersistenceContext(object):
                 tag_list=tag_list,
                 sources=sources,
                 hashes=hashes,
+                created_from_basename=created_from_basename,
             )
             log.debug(
                 "(%s) Created dynamic collection dataset for path [%s] with element identifier [%s] for output [%s] %s",
@@ -475,6 +481,8 @@ def persist_elements_to_folder(model_persistence_context, elements, library_fold
 
             sources = fields_match.sources
             hashes = fields_match.hashes
+            created_from_basename = fields_match.created_from_basename
+
             model_persistence_context.create_dataset(
                 ext=ext,
                 designation=designation,
@@ -487,6 +495,7 @@ def persist_elements_to_folder(model_persistence_context, elements, library_fold
                 link_data=link_data,
                 sources=sources,
                 hashes=hashes,
+                created_from_basename=created_from_basename,
             )
 
 
@@ -517,6 +526,8 @@ def persist_hdas(elements, model_persistence_context):
 
                 sources = fields_match.sources
                 hashes = fields_match.hashes
+                created_from_basename = fields_match.created_from_basename
+
                 dataset = model_persistence_context.create_dataset(
                     ext=ext,
                     designation=designation,
@@ -529,6 +540,7 @@ def persist_hdas(elements, model_persistence_context):
                     primary_data=primary_dataset,
                     sources=sources,
                     hashes=hashes,
+                    created_from_basename=created_from_basename,
                 )
                 dataset.raw_set_dataset_state('ok')
                 if not hda_id:
@@ -701,6 +713,10 @@ class JsonCollectedDatasetMatch(object):
     def hashes(self):
         return self.as_dict.get("hashes", [])
 
+    @property
+    def created_from_basename(self):
+        return self.as_dict.get("created_from_basename")
+
 
 class RegexCollectedDatasetMatch(JsonCollectedDatasetMatch):
 

lib/galaxy/tools/actions/__init__.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import os
 import re
 from json import dumps
 
@@ -391,6 +392,10 @@ class DefaultToolAction(object):
                             break
 
                 data = app.model.HistoryDatasetAssociation(extension=ext, dataset=dataset, create_dataset=create_datasets, flush=False)
+                if create_datasets:
+                    from_work_dir = output.from_work_dir
+                    if from_work_dir is not None:
+                        data.dataset.created_from_basename = os.path.basename(from_work_dir)
                 if hidden is None:
                     hidden = output.hidden
                 if not hidden and dataset_collection_elements is not None:  # Mapping over a collection - hide datasets

lib/galaxy/tools/data_fetch.py

@@ -108,6 +108,7 @@ def _fetch_target(upload_config, target):
         dbkey = item.get("dbkey", "?")
         requested_ext = item.get("ext", "auto")
         info = item.get("info", None)
+        created_from_basename = item.get("created_from_basename", None)
         tags = item.get("tags", [])
         object_id = item.get("object_id", None)
         link_data_only = upload_config.link_data_only
@@ -162,6 +163,8 @@ def _fetch_target(upload_config, target):
             rval["object_id"] = object_id
         if tags:
             rval["tags"] = tags
+        if created_from_basename:
+            rval["created_from_basename"] = created_from_basename
         return rval
 
     elements = elements_tree_map(_resolve_src, items)

lib/galaxy/tools/verify/interactor.py

@@ -194,7 +194,7 @@ class GalaxyInteractorApi(object):
         """
         metadata = attributes.get('metadata', {}).copy()
         for key, value in metadata.copy().items():
-            if key not in ['name', 'info', 'tags']:
+            if key not in ['name', 'info', 'tags', 'created_from_basename']:
                 new_key = "metadata_%s" % key
                 metadata[new_key] = metadata[key]
                 del metadata[key]

test/api/test_libraries.py

@@ -101,7 +101,7 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets):
 
     def test_fetch_upload_to_folder(self):
         history_id, library, destination = self._setup_fetch_to_folder("flat_zip")
-        items = [{"src": "files", "dbkey": "hg19", "info": "my cool bed"}]
+        items = [{"src": "files", "dbkey": "hg19", "info": "my cool bed", "created_from_basename": "4.bed"}]
         targets = [{
             "destination": destination,
             "items": items
@@ -117,6 +117,7 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets):
         assert dataset["genome_build"] == "hg19", dataset
         assert dataset["misc_info"] == "my cool bed", dataset
         assert dataset["file_ext"] == "bed", dataset
+        assert dataset["created_from_basename"] == "4.bed"
 
     def test_fetch_zip_to_folder(self):
         history_id, library, destination = self._setup_fetch_to_folder("flat_zip")

test/functional/tools/output_format.xml

@@ -32,11 +32,13 @@
       <param name="input_text" value="foo" />
       <output name="direct_output" ftype="txt">
         <assert_contents><has_line line="test" /></assert_contents>
+        <metadata name="created_from_basename" value="1" />
       </output>
       <!-- In this case input_based_output ftype is "randomly" either
            fastqsanger or fastqsolexa -->
       <output name="format_source_1_output" ftype="fastqsanger">
         <assert_contents><has_line line="test" /></assert_contents>
+        <metadata name="created_from_basename" value="3" />
       </output>
       <output name="format_source_2_output" ftype="fastqsolexa">
         <assert_contents><has_line line="test" /></assert_contents>

test/functional/tools/tool_provided_metadata_9.xml

@@ -10,7 +10,8 @@
   "name": "my dynamic name",
   "ext": "txt",
   "info": "my dynamic info",
-  "dbkey": "cust1"
+  "dbkey": "cust1",
+  "created_from_basename": "my name.txt"
 }}
 </configfile>
     </configfiles>
@@ -29,6 +30,7 @@
           <metadata name="name" value="my dynamic name" />
           <metadata name="info" value="my dynamic info" />
           <metadata name="dbkey" value="cust1" />
+          <metadata name="created_from_basename" value="my name.txt" />
         </output>
       </test>
     </tests>