From 29dfc3085be63664e6e903dbdb2ef44ccd3ce234 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 20 Apr 2019 08:18:39 -0400 Subject: [PATCH] Track basename at the time of creation for datasets. --- lib/galaxy/jobs/__init__.py | 2 +- lib/galaxy/managers/hdas.py | 6 ++-- lib/galaxy/model/__init__.py | 14 ++++++++- lib/galaxy/model/mapping.py | 1 + .../versions/0152_add_metadata_file_uuid.py | 21 ++++---------- .../versions/0154_created_from_basename.py | 29 +++++++++++++++++++ lib/galaxy/model/store/discover.py | 16 ++++++++++ lib/galaxy/tools/actions/__init__.py | 5 ++++ lib/galaxy/tools/data_fetch.py | 3 ++ lib/galaxy/tools/verify/interactor.py | 2 +- test/api/test_libraries.py | 3 +- test/functional/tools/output_format.xml | 2 ++ .../tools/tool_provided_metadata_9.xml | 4 ++- 13 files changed, 85 insertions(+), 23 deletions(-) create mode 100644 lib/galaxy/model/migrate/versions/0154_created_from_basename.py diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index e3460c38fc8..8cbc4bff221 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -46,7 +46,7 @@ log = logging.getLogger(__name__) # that import Galaxy internals - but it shouldn't be used in Galaxy's code # itself. TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json' -TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey'] +TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey', 'created_from_basename'] # Override with config.default_job_shell. DEFAULT_JOB_SHELL = '/bin/bash' diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index 1842665ea78..08bd08e505f 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -300,7 +300,8 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize 'annotation', - 'api_type' + 'api_type', + 'created_from_basename', ], include_keys_from='summary') self.add_view('extended', [ @@ -354,7 +355,8 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize # TODO: to DatasetAssociationSerializer 'accessible' : lambda i, k, user=None, **c: self.manager.is_accessible(i, user, **c), 'api_type' : lambda *a, **c: 'file', - 'type' : lambda *a, **c: 'file' + 'type' : lambda *a, **c: 'file', + 'created_from_basename' : lambda i, k, **c: i.created_from_basename, }) def serialize(self, hda, keys, user=None, **context): diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index fe26813a4e2..00684f71ae2 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2510,6 +2510,16 @@ class DatasetInstance(object): """Detects whether there is any data""" return self.dataset.has_data() + def get_created_from_basename(self): + return self.dataset.created_from_basename + + def set_created_from_basename(self, created_from_basename): + if self.dataset.created_from_basename is not None: + raise Exception("Underlying dataset already has a created_from_basename set.") + self.dataset.created_from_basename = created_from_basename + + created_from_basename = property(get_created_from_basename, set_created_from_basename) + def get_raw_data(self): """Returns the full data. To stream it open the file_name and read/write as needed""" return self.datatype.get_raw_data(self) @@ -3393,6 +3403,7 @@ class LibraryDataset(RepresentById): state=ldda.state, name=ldda.name, file_name=ldda.file_name, + created_from_basename=ldda.created_from_basename, uploaded_by=ldda.user.email, message=ldda.message, date_uploaded=ldda.create_time.isoformat(), @@ -3545,7 +3556,8 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): data_type=ldda.datatype.__class__.__module__ + '.' + ldda.datatype.__class__.__name__, genome_build=ldda.dbkey, misc_info=ldda.info, - misc_blurb=ldda.blurb) + misc_blurb=ldda.blurb, + created_from_basename=ldda.created_from_basename) if ldda.dataset.uuid is None: rval['uuid'] = None else: diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f7f563c54c8..25f91256140 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -263,6 +263,7 @@ model.Dataset.table = Table( Column("object_store_id", TrimmedString(255), index=True), Column("external_filename", TEXT), Column("_extra_files_path", TEXT), + Column("created_from_basename", TEXT), Column('file_size', Numeric(15, 0)), Column('total_size', Numeric(15, 0)), Column('uuid', UUIDType())) diff --git a/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py b/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py index f9d26724a37..0f25287393c 100644 --- a/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py +++ b/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py @@ -6,9 +6,10 @@ from __future__ import print_function import logging -from sqlalchemy import Column, MetaData, Table +from sqlalchemy import Column, MetaData from galaxy.model.custom_types import UUIDType +from galaxy.model.migrate.versions.util import add_column, drop_column log = logging.getLogger(__name__) @@ -19,14 +20,8 @@ def upgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() - metadata_file_table = Table("metadata_file", metadata, autoload=True) - - try: - uuid_column = Column('uuid', UUIDType()) - uuid_column.create(metadata_file_table) - assert uuid_column is metadata_file_table.c.uuid - except Exception: - log.exception("Adding column 'uuid' to `MetadataFile` table failed.") + uuid_column = Column('uuid', UUIDType()) + add_column(uuid_column, 'metadata_file', metadata) def downgrade(migrate_engine): @@ -34,10 +29,4 @@ def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() - metadata_file_table = Table("metadata_file", metadata, autoload=True) - - try: - column = metadata_file_table.c.uuid - column.drop() - except Exception: - log.exception("Dropping 'uuid' column from `metadata_file` table failed.") + drop_column('uuid', 'metadata_file', metadata) diff --git a/lib/galaxy/model/migrate/versions/0154_created_from_basename.py b/lib/galaxy/model/migrate/versions/0154_created_from_basename.py new file mode 100644 index 00000000000..c4664b48e97 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0154_created_from_basename.py @@ -0,0 +1,29 @@ +""" +Adds created_from_basename to dataset. +""" +import datetime +import logging + +from sqlalchemy import Column, MetaData, TEXT + +from galaxy.model.migrate.versions.util import add_column, drop_column + +now = datetime.datetime.utcnow +log = logging.getLogger(__name__) +metadata = MetaData() + + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print(__doc__) + metadata.reflect() + + created_from_basename_column = Column("created_from_basename", TEXT, default=None) + add_column(created_from_basename_column, 'dataset', metadata) + + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + + drop_column('created_from_basename', 'dataset', metadata) diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py index 697aefc15b0..2f5964d4ed9 100644 --- a/lib/galaxy/model/store/discover.py +++ b/lib/galaxy/model/store/discover.py @@ -55,6 +55,7 @@ class ModelPersistenceContext(object): tag_list=[], sources=[], hashes=[], + created_from_basename=None, ): sa_session = self.sa_session @@ -116,6 +117,9 @@ class ModelPersistenceContext(object): hash_object.hash_value = hash_dict["hash_value"] primary_data.dataset.hashes.append(hash_object) + if created_from_basename is not None: + primary_data.created_from_basename = created_from_basename + self.flush() if tag_list: @@ -204,6 +208,7 @@ class ModelPersistenceContext(object): sources = discovered_file.match.sources hashes = discovered_file.match.hashes + created_from_basename = discovered_file.match.created_from_basename dataset = self.create_dataset( ext=ext, @@ -217,6 +222,7 @@ class ModelPersistenceContext(object): tag_list=tag_list, sources=sources, hashes=hashes, + created_from_basename=created_from_basename, ) log.debug( "(%s) Created dynamic collection dataset for path [%s] with element identifier [%s] for output [%s] %s", @@ -475,6 +481,8 @@ def persist_elements_to_folder(model_persistence_context, elements, library_fold sources = fields_match.sources hashes = fields_match.hashes + created_from_basename = fields_match.created_from_basename + model_persistence_context.create_dataset( ext=ext, designation=designation, @@ -487,6 +495,7 @@ def persist_elements_to_folder(model_persistence_context, elements, library_fold link_data=link_data, sources=sources, hashes=hashes, + created_from_basename=created_from_basename, ) @@ -517,6 +526,8 @@ def persist_hdas(elements, model_persistence_context): sources = fields_match.sources hashes = fields_match.hashes + created_from_basename = fields_match.created_from_basename + dataset = model_persistence_context.create_dataset( ext=ext, designation=designation, @@ -529,6 +540,7 @@ def persist_hdas(elements, model_persistence_context): primary_data=primary_dataset, sources=sources, hashes=hashes, + created_from_basename=created_from_basename, ) dataset.raw_set_dataset_state('ok') if not hda_id: @@ -701,6 +713,10 @@ class JsonCollectedDatasetMatch(object): def hashes(self): return self.as_dict.get("hashes", []) + @property + def created_from_basename(self): + return self.as_dict.get("created_from_basename") + class RegexCollectedDatasetMatch(JsonCollectedDatasetMatch): diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 7f4ecd03c92..676e2cd5a4a 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -1,5 +1,6 @@ import json import logging +import os import re from json import dumps @@ -392,6 +393,10 @@ class DefaultToolAction(object): break data = app.model.HistoryDatasetAssociation(extension=ext, dataset=dataset, create_dataset=create_datasets, flush=False) + if create_datasets: + from_work_dir = output.from_work_dir + if from_work_dir is not None: + data.dataset.created_from_basename = os.path.basename(from_work_dir) if hidden is None: hidden = output.hidden if not hidden and dataset_collection_elements is not None: # Mapping over a collection - hide datasets diff --git a/lib/galaxy/tools/data_fetch.py b/lib/galaxy/tools/data_fetch.py index 75eaa204d48..90b35f64c69 100644 --- a/lib/galaxy/tools/data_fetch.py +++ b/lib/galaxy/tools/data_fetch.py @@ -108,6 +108,7 @@ def _fetch_target(upload_config, target): dbkey = item.get("dbkey", "?") requested_ext = item.get("ext", "auto") info = item.get("info", None) + created_from_basename = item.get("created_from_basename", None) tags = item.get("tags", []) object_id = item.get("object_id", None) link_data_only = upload_config.link_data_only @@ -162,6 +163,8 @@ def _fetch_target(upload_config, target): rval["object_id"] = object_id if tags: rval["tags"] = tags + if created_from_basename: + rval["created_from_basename"] = created_from_basename return rval elements = elements_tree_map(_resolve_src, items) diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py index 81435bcf2d9..89c7d066efa 100644 --- a/lib/galaxy/tools/verify/interactor.py +++ b/lib/galaxy/tools/verify/interactor.py @@ -194,7 +194,7 @@ class GalaxyInteractorApi(object): """ metadata = attributes.get('metadata', {}).copy() for key, value in metadata.copy().items(): - if key not in ['name', 'info', 'tags']: + if key not in ['name', 'info', 'tags', 'created_from_basename']: new_key = "metadata_%s" % key metadata[new_key] = metadata[key] del metadata[key] diff --git a/test/api/test_libraries.py b/test/api/test_libraries.py index 8535511e480..a9f8d5f4f05 100644 --- a/test/api/test_libraries.py +++ b/test/api/test_libraries.py @@ -101,7 +101,7 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets): def test_fetch_upload_to_folder(self): history_id, library, destination = self._setup_fetch_to_folder("flat_zip") - items = [{"src": "files", "dbkey": "hg19", "info": "my cool bed"}] + items = [{"src": "files", "dbkey": "hg19", "info": "my cool bed", "created_from_basename": "4.bed"}] targets = [{ "destination": destination, "items": items @@ -117,6 +117,7 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets): assert dataset["genome_build"] == "hg19", dataset assert dataset["misc_info"] == "my cool bed", dataset assert dataset["file_ext"] == "bed", dataset + assert dataset["created_from_basename"] == "4.bed" def test_fetch_zip_to_folder(self): history_id, library, destination = self._setup_fetch_to_folder("flat_zip") diff --git a/test/functional/tools/output_format.xml b/test/functional/tools/output_format.xml index 4f95a20f238..8c5005879ef 100644 --- a/test/functional/tools/output_format.xml +++ b/test/functional/tools/output_format.xml @@ -32,11 +32,13 @@ + + diff --git a/test/functional/tools/tool_provided_metadata_9.xml b/test/functional/tools/tool_provided_metadata_9.xml index 797ede1c871..285d5c88089 100644 --- a/test/functional/tools/tool_provided_metadata_9.xml +++ b/test/functional/tools/tool_provided_metadata_9.xml @@ -10,7 +10,8 @@ "name": "my dynamic name", "ext": "txt", "info": "my dynamic info", - "dbkey": "cust1" + "dbkey": "cust1", + "created_from_basename": "my name.txt" }} @@ -29,6 +30,7 @@ +