diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index e3460c38fc8..8cbc4bff221 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -46,7 +46,7 @@ log = logging.getLogger(__name__)
# that import Galaxy internals - but it shouldn't be used in Galaxy's code
# itself.
TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'
-TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey']
+TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey', 'created_from_basename']
# Override with config.default_job_shell.
DEFAULT_JOB_SHELL = '/bin/bash'
diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py
index 1842665ea78..08bd08e505f 100644
--- a/lib/galaxy/managers/hdas.py
+++ b/lib/galaxy/managers/hdas.py
@@ -300,7 +300,8 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize
'annotation',
- 'api_type'
+ 'api_type',
+ 'created_from_basename',
], include_keys_from='summary')
self.add_view('extended', [
@@ -354,7 +355,8 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize
# TODO: to DatasetAssociationSerializer
'accessible' : lambda i, k, user=None, **c: self.manager.is_accessible(i, user, **c),
'api_type' : lambda *a, **c: 'file',
- 'type' : lambda *a, **c: 'file'
+ 'type' : lambda *a, **c: 'file',
+ 'created_from_basename' : lambda i, k, **c: i.created_from_basename,
})
def serialize(self, hda, keys, user=None, **context):
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index fe26813a4e2..00684f71ae2 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -2510,6 +2510,16 @@ class DatasetInstance(object):
"""Detects whether there is any data"""
return self.dataset.has_data()
+ def get_created_from_basename(self):
+ return self.dataset.created_from_basename
+
+ def set_created_from_basename(self, created_from_basename):
+ if self.dataset.created_from_basename is not None:
+ raise Exception("Underlying dataset already has a created_from_basename set.")
+ self.dataset.created_from_basename = created_from_basename
+
+ created_from_basename = property(get_created_from_basename, set_created_from_basename)
+
def get_raw_data(self):
"""Returns the full data. To stream it open the file_name and read/write as needed"""
return self.datatype.get_raw_data(self)
@@ -3393,6 +3403,7 @@ class LibraryDataset(RepresentById):
state=ldda.state,
name=ldda.name,
file_name=ldda.file_name,
+ created_from_basename=ldda.created_from_basename,
uploaded_by=ldda.user.email,
message=ldda.message,
date_uploaded=ldda.create_time.isoformat(),
@@ -3545,7 +3556,8 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById):
data_type=ldda.datatype.__class__.__module__ + '.' + ldda.datatype.__class__.__name__,
genome_build=ldda.dbkey,
misc_info=ldda.info,
- misc_blurb=ldda.blurb)
+ misc_blurb=ldda.blurb,
+ created_from_basename=ldda.created_from_basename)
if ldda.dataset.uuid is None:
rval['uuid'] = None
else:
diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py
index f7f563c54c8..25f91256140 100644
--- a/lib/galaxy/model/mapping.py
+++ b/lib/galaxy/model/mapping.py
@@ -263,6 +263,7 @@ model.Dataset.table = Table(
Column("object_store_id", TrimmedString(255), index=True),
Column("external_filename", TEXT),
Column("_extra_files_path", TEXT),
+ Column("created_from_basename", TEXT),
Column('file_size', Numeric(15, 0)),
Column('total_size', Numeric(15, 0)),
Column('uuid', UUIDType()))
diff --git a/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py b/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py
index f9d26724a37..0f25287393c 100644
--- a/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py
+++ b/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py
@@ -6,9 +6,10 @@ from __future__ import print_function
import logging
-from sqlalchemy import Column, MetaData, Table
+from sqlalchemy import Column, MetaData
from galaxy.model.custom_types import UUIDType
+from galaxy.model.migrate.versions.util import add_column, drop_column
log = logging.getLogger(__name__)
@@ -19,14 +20,8 @@ def upgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
- metadata_file_table = Table("metadata_file", metadata, autoload=True)
-
- try:
- uuid_column = Column('uuid', UUIDType())
- uuid_column.create(metadata_file_table)
- assert uuid_column is metadata_file_table.c.uuid
- except Exception:
- log.exception("Adding column 'uuid' to `MetadataFile` table failed.")
+ uuid_column = Column('uuid', UUIDType())
+ add_column(uuid_column, 'metadata_file', metadata)
def downgrade(migrate_engine):
@@ -34,10 +29,4 @@ def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
- metadata_file_table = Table("metadata_file", metadata, autoload=True)
-
- try:
- column = metadata_file_table.c.uuid
- column.drop()
- except Exception:
- log.exception("Dropping 'uuid' column from `metadata_file` table failed.")
+ drop_column('uuid', 'metadata_file', metadata)
diff --git a/lib/galaxy/model/migrate/versions/0154_created_from_basename.py b/lib/galaxy/model/migrate/versions/0154_created_from_basename.py
new file mode 100644
index 00000000000..c4664b48e97
--- /dev/null
+++ b/lib/galaxy/model/migrate/versions/0154_created_from_basename.py
@@ -0,0 +1,29 @@
+"""
+Adds created_from_basename to dataset.
+"""
+import datetime
+import logging
+
+from sqlalchemy import Column, MetaData, TEXT
+
+from galaxy.model.migrate.versions.util import add_column, drop_column
+
+now = datetime.datetime.utcnow
+log = logging.getLogger(__name__)
+metadata = MetaData()
+
+
+def upgrade(migrate_engine):
+ metadata.bind = migrate_engine
+ print(__doc__)
+ metadata.reflect()
+
+ created_from_basename_column = Column("created_from_basename", TEXT, default=None)
+ add_column(created_from_basename_column, 'dataset', metadata)
+
+
+def downgrade(migrate_engine):
+ metadata.bind = migrate_engine
+ metadata.reflect()
+
+ drop_column('created_from_basename', 'dataset', metadata)
diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py
index 697aefc15b0..2f5964d4ed9 100644
--- a/lib/galaxy/model/store/discover.py
+++ b/lib/galaxy/model/store/discover.py
@@ -55,6 +55,7 @@ class ModelPersistenceContext(object):
tag_list=[],
sources=[],
hashes=[],
+ created_from_basename=None,
):
sa_session = self.sa_session
@@ -116,6 +117,9 @@ class ModelPersistenceContext(object):
hash_object.hash_value = hash_dict["hash_value"]
primary_data.dataset.hashes.append(hash_object)
+ if created_from_basename is not None:
+ primary_data.created_from_basename = created_from_basename
+
self.flush()
if tag_list:
@@ -204,6 +208,7 @@ class ModelPersistenceContext(object):
sources = discovered_file.match.sources
hashes = discovered_file.match.hashes
+ created_from_basename = discovered_file.match.created_from_basename
dataset = self.create_dataset(
ext=ext,
@@ -217,6 +222,7 @@ class ModelPersistenceContext(object):
tag_list=tag_list,
sources=sources,
hashes=hashes,
+ created_from_basename=created_from_basename,
)
log.debug(
"(%s) Created dynamic collection dataset for path [%s] with element identifier [%s] for output [%s] %s",
@@ -475,6 +481,8 @@ def persist_elements_to_folder(model_persistence_context, elements, library_fold
sources = fields_match.sources
hashes = fields_match.hashes
+ created_from_basename = fields_match.created_from_basename
+
model_persistence_context.create_dataset(
ext=ext,
designation=designation,
@@ -487,6 +495,7 @@ def persist_elements_to_folder(model_persistence_context, elements, library_fold
link_data=link_data,
sources=sources,
hashes=hashes,
+ created_from_basename=created_from_basename,
)
@@ -517,6 +526,8 @@ def persist_hdas(elements, model_persistence_context):
sources = fields_match.sources
hashes = fields_match.hashes
+ created_from_basename = fields_match.created_from_basename
+
dataset = model_persistence_context.create_dataset(
ext=ext,
designation=designation,
@@ -529,6 +540,7 @@ def persist_hdas(elements, model_persistence_context):
primary_data=primary_dataset,
sources=sources,
hashes=hashes,
+ created_from_basename=created_from_basename,
)
dataset.raw_set_dataset_state('ok')
if not hda_id:
@@ -701,6 +713,10 @@ class JsonCollectedDatasetMatch(object):
def hashes(self):
return self.as_dict.get("hashes", [])
+ @property
+ def created_from_basename(self):
+ return self.as_dict.get("created_from_basename")
+
class RegexCollectedDatasetMatch(JsonCollectedDatasetMatch):
diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py
index 7f4ecd03c92..676e2cd5a4a 100644
--- a/lib/galaxy/tools/actions/__init__.py
+++ b/lib/galaxy/tools/actions/__init__.py
@@ -1,5 +1,6 @@
import json
import logging
+import os
import re
from json import dumps
@@ -392,6 +393,10 @@ class DefaultToolAction(object):
break
data = app.model.HistoryDatasetAssociation(extension=ext, dataset=dataset, create_dataset=create_datasets, flush=False)
+ if create_datasets:
+ from_work_dir = output.from_work_dir
+ if from_work_dir is not None:
+ data.dataset.created_from_basename = os.path.basename(from_work_dir)
if hidden is None:
hidden = output.hidden
if not hidden and dataset_collection_elements is not None: # Mapping over a collection - hide datasets
diff --git a/lib/galaxy/tools/data_fetch.py b/lib/galaxy/tools/data_fetch.py
index 75eaa204d48..90b35f64c69 100644
--- a/lib/galaxy/tools/data_fetch.py
+++ b/lib/galaxy/tools/data_fetch.py
@@ -108,6 +108,7 @@ def _fetch_target(upload_config, target):
dbkey = item.get("dbkey", "?")
requested_ext = item.get("ext", "auto")
info = item.get("info", None)
+ created_from_basename = item.get("created_from_basename", None)
tags = item.get("tags", [])
object_id = item.get("object_id", None)
link_data_only = upload_config.link_data_only
@@ -162,6 +163,8 @@ def _fetch_target(upload_config, target):
rval["object_id"] = object_id
if tags:
rval["tags"] = tags
+ if created_from_basename:
+ rval["created_from_basename"] = created_from_basename
return rval
elements = elements_tree_map(_resolve_src, items)
diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py
index 81435bcf2d9..89c7d066efa 100644
--- a/lib/galaxy/tools/verify/interactor.py
+++ b/lib/galaxy/tools/verify/interactor.py
@@ -194,7 +194,7 @@ class GalaxyInteractorApi(object):
"""
metadata = attributes.get('metadata', {}).copy()
for key, value in metadata.copy().items():
- if key not in ['name', 'info', 'tags']:
+ if key not in ['name', 'info', 'tags', 'created_from_basename']:
new_key = "metadata_%s" % key
metadata[new_key] = metadata[key]
del metadata[key]
diff --git a/test/api/test_libraries.py b/test/api/test_libraries.py
index 8535511e480..a9f8d5f4f05 100644
--- a/test/api/test_libraries.py
+++ b/test/api/test_libraries.py
@@ -101,7 +101,7 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets):
def test_fetch_upload_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder("flat_zip")
- items = [{"src": "files", "dbkey": "hg19", "info": "my cool bed"}]
+ items = [{"src": "files", "dbkey": "hg19", "info": "my cool bed", "created_from_basename": "4.bed"}]
targets = [{
"destination": destination,
"items": items
@@ -117,6 +117,7 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets):
assert dataset["genome_build"] == "hg19", dataset
assert dataset["misc_info"] == "my cool bed", dataset
assert dataset["file_ext"] == "bed", dataset
+ assert dataset["created_from_basename"] == "4.bed"
def test_fetch_zip_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder("flat_zip")
diff --git a/test/functional/tools/output_format.xml b/test/functional/tools/output_format.xml
index 4f95a20f238..8c5005879ef 100644
--- a/test/functional/tools/output_format.xml
+++ b/test/functional/tools/output_format.xml
@@ -32,11 +32,13 @@