opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Track basename at the time of creation for datasets.

Browse Source
This commit is contained in:
John Chilton
2019-04-20 08:18:39 -04:00
parent eaff93c6b5
commit 29dfc3085b
13 changed files with 85 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/jobs/__init__.py
+1 -1
View File
@@ -46,7 +46,7 @@ log = logging.getLogger(__name__)
# that import Galaxy internals - but it shouldn't be used in Galaxy's code
# itself.
TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'
TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey']
TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey', 'created_from_basename']
# Override with config.default_job_shell.
DEFAULT_JOB_SHELL = '/bin/bash'
lib/galaxy/managers/hdas.py
+4 -2
View File
@@ -300,7 +300,8 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize
'annotation',
'api_type'
'api_type',
'created_from_basename',
], include_keys_from='summary')
self.add_view('extended', [
@@ -354,7 +355,8 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize
# TODO: to DatasetAssociationSerializer
'accessible' : lambda i, k, user=None, **c: self.manager.is_accessible(i, user, **c),
'api_type' : lambda *a, **c: 'file',
'type' : lambda *a, **c: 'file'
'type' : lambda *a, **c: 'file',
'created_from_basename' : lambda i, k, **c: i.created_from_basename,
})
def serialize(self, hda, keys, user=None, **context):
lib/galaxy/model/__init__.py
+13 -1
View File
@@ -2510,6 +2510,16 @@ class DatasetInstance(object):
"""Detects whether there is any data"""
return self.dataset.has_data()
def get_created_from_basename(self):
return self.dataset.created_from_basename
def set_created_from_basename(self, created_from_basename):
if self.dataset.created_from_basename is not None:
raise Exception("Underlying dataset already has a created_from_basename set.")
self.dataset.created_from_basename = created_from_basename
created_from_basename = property(get_created_from_basename, set_created_from_basename)
def get_raw_data(self):
"""Returns the full data. To stream it open the file_name and read/write as needed"""
return self.datatype.get_raw_data(self)
@@ -3393,6 +3403,7 @@ class LibraryDataset(RepresentById):
state=ldda.state,
name=ldda.name,
file_name=ldda.file_name,
created_from_basename=ldda.created_from_basename,
uploaded_by=ldda.user.email,
message=ldda.message,
date_uploaded=ldda.create_time.isoformat(),
@@ -3545,7 +3556,8 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById):
data_type=ldda.datatype.__class__.__module__ + '.' + ldda.datatype.__class__.__name__,
genome_build=ldda.dbkey,
misc_info=ldda.info,
misc_blurb=ldda.blurb)
misc_blurb=ldda.blurb,
created_from_basename=ldda.created_from_basename)
if ldda.dataset.uuid is None:
rval['uuid'] = None
else:
lib/galaxy/model/mapping.py
+1
View File
@@ -263,6 +263,7 @@ model.Dataset.table = Table(
Column("object_store_id", TrimmedString(255), index=True),
Column("external_filename", TEXT),
Column("_extra_files_path", TEXT),
Column("created_from_basename", TEXT),
Column('file_size', Numeric(15, 0)),
Column('total_size', Numeric(15, 0)),
Column('uuid', UUIDType()))
lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py
+5 -16
View File
@@ -6,9 +6,10 @@ from __future__ import print_function
import logging
from sqlalchemy import Column, MetaData, Table
from sqlalchemy import Column, MetaData
from galaxy.model.custom_types import UUIDType
from galaxy.model.migrate.versions.util import add_column, drop_column
log = logging.getLogger(__name__)
@@ -19,14 +20,8 @@ def upgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
metadata_file_table = Table("metadata_file", metadata, autoload=True)
try:
uuid_column = Column('uuid', UUIDType())
uuid_column.create(metadata_file_table)
assert uuid_column is metadata_file_table.c.uuid
except Exception:
log.exception("Adding column 'uuid' to `MetadataFile` table failed.")
uuid_column = Column('uuid', UUIDType())
add_column(uuid_column, 'metadata_file', metadata)
def downgrade(migrate_engine):
@@ -34,10 +29,4 @@ def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
metadata_file_table = Table("metadata_file", metadata, autoload=True)
try:
column = metadata_file_table.c.uuid
column.drop()
except Exception:
log.exception("Dropping 'uuid' column from `metadata_file` table failed.")
drop_column('uuid', 'metadata_file', metadata)
lib/galaxy/model/migrate/versions/0154_created_from_basename.py
+29
View File
@@ -0,0 +1,29 @@
"""
Adds created_from_basename to dataset.
"""
import datetime
import logging
from sqlalchemy import Column, MetaData, TEXT
from galaxy.model.migrate.versions.util import add_column, drop_column
now = datetime.datetime.utcnow
log = logging.getLogger(__name__)
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print(__doc__)
metadata.reflect()
created_from_basename_column = Column("created_from_basename", TEXT, default=None)
add_column(created_from_basename_column, 'dataset', metadata)
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
drop_column('created_from_basename', 'dataset', metadata)
lib/galaxy/model/store/discover.py
+16
View File
@@ -55,6 +55,7 @@ class ModelPersistenceContext(object):
tag_list=[],
sources=[],
hashes=[],
created_from_basename=None,
):
sa_session = self.sa_session
@@ -116,6 +117,9 @@ class ModelPersistenceContext(object):
hash_object.hash_value = hash_dict["hash_value"]
primary_data.dataset.hashes.append(hash_object)
if created_from_basename is not None:
primary_data.created_from_basename = created_from_basename
self.flush()
if tag_list:
@@ -204,6 +208,7 @@ class ModelPersistenceContext(object):
sources = discovered_file.match.sources
hashes = discovered_file.match.hashes
created_from_basename = discovered_file.match.created_from_basename
dataset = self.create_dataset(
ext=ext,
@@ -217,6 +222,7 @@ class ModelPersistenceContext(object):
tag_list=tag_list,
sources=sources,
hashes=hashes,
created_from_basename=created_from_basename,
)
log.debug(
"(%s) Created dynamic collection dataset for path [%s] with element identifier [%s] for output [%s] %s",
@@ -475,6 +481,8 @@ def persist_elements_to_folder(model_persistence_context, elements, library_fold
sources = fields_match.sources
hashes = fields_match.hashes
created_from_basename = fields_match.created_from_basename
model_persistence_context.create_dataset(
ext=ext,
designation=designation,
@@ -487,6 +495,7 @@ def persist_elements_to_folder(model_persistence_context, elements, library_fold
link_data=link_data,
sources=sources,
hashes=hashes,
created_from_basename=created_from_basename,
)
@@ -517,6 +526,8 @@ def persist_hdas(elements, model_persistence_context):
sources = fields_match.sources
hashes = fields_match.hashes
created_from_basename = fields_match.created_from_basename
dataset = model_persistence_context.create_dataset(
ext=ext,
designation=designation,
@@ -529,6 +540,7 @@ def persist_hdas(elements, model_persistence_context):
primary_data=primary_dataset,
sources=sources,
hashes=hashes,
created_from_basename=created_from_basename,
)
dataset.raw_set_dataset_state('ok')
if not hda_id:
@@ -701,6 +713,10 @@ class JsonCollectedDatasetMatch(object):
def hashes(self):
return self.as_dict.get("hashes", [])
@property
def created_from_basename(self):
return self.as_dict.get("created_from_basename")
class RegexCollectedDatasetMatch(JsonCollectedDatasetMatch):
lib/galaxy/tools/actions/__init__.py
+5
View File
@@ -1,5 +1,6 @@
import json
import logging
import os
import re
from json import dumps
@@ -392,6 +393,10 @@ class DefaultToolAction(object):
break
data = app.model.HistoryDatasetAssociation(extension=ext, dataset=dataset, create_dataset=create_datasets, flush=False)
if create_datasets:
from_work_dir = output.from_work_dir
if from_work_dir is not None:
data.dataset.created_from_basename = os.path.basename(from_work_dir)
if hidden is None:
hidden = output.hidden
if not hidden and dataset_collection_elements is not None: # Mapping over a collection - hide datasets
lib/galaxy/tools/data_fetch.py
+3
View File
@@ -108,6 +108,7 @@ def _fetch_target(upload_config, target):
dbkey = item.get("dbkey", "?")
requested_ext = item.get("ext", "auto")
info = item.get("info", None)
created_from_basename = item.get("created_from_basename", None)
tags = item.get("tags", [])
object_id = item.get("object_id", None)
link_data_only = upload_config.link_data_only
@@ -162,6 +163,8 @@ def _fetch_target(upload_config, target):
rval["object_id"] = object_id
if tags:
rval["tags"] = tags
if created_from_basename:
rval["created_from_basename"] = created_from_basename
return rval
elements = elements_tree_map(_resolve_src, items)
lib/galaxy/tools/verify/interactor.py
+1 -1
View File
@@ -194,7 +194,7 @@ class GalaxyInteractorApi(object):
"""
metadata = attributes.get('metadata', {}).copy()
for key, value in metadata.copy().items():
if key not in ['name', 'info', 'tags']:
if key not in ['name', 'info', 'tags', 'created_from_basename']:
new_key = "metadata_%s" % key
metadata[new_key] = metadata[key]
del metadata[key]
test/api/test_libraries.py
+2 -1
View File
@@ -101,7 +101,7 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets):
def test_fetch_upload_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder("flat_zip")
items = [{"src": "files", "dbkey": "hg19", "info": "my cool bed"}]
items = [{"src": "files", "dbkey": "hg19", "info": "my cool bed", "created_from_basename": "4.bed"}]
targets = [{
"destination": destination,
"items": items
@@ -117,6 +117,7 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets):
assert dataset["genome_build"] == "hg19", dataset
assert dataset["misc_info"] == "my cool bed", dataset
assert dataset["file_ext"] == "bed", dataset
assert dataset["created_from_basename"] == "4.bed"
def test_fetch_zip_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder("flat_zip")
test/functional/tools/output_format.xml
+2
View File
@@ -32,11 +32,13 @@
<param name="input_text" value="foo" />
<output name="direct_output" ftype="txt">
<assert_contents><has_line line="test" /></assert_contents>
<metadata name="created_from_basename" value="1" />
</output>
<!-- In this case input_based_output ftype is "randomly" either
fastqsanger or fastqsolexa -->
<output name="format_source_1_output" ftype="fastqsanger">
<assert_contents><has_line line="test" /></assert_contents>
<metadata name="created_from_basename" value="3" />
</output>
<output name="format_source_2_output" ftype="fastqsolexa">
<assert_contents><has_line line="test" /></assert_contents>
test/functional/tools/tool_provided_metadata_9.xml
+3 -1
View File
@@ -10,7 +10,8 @@
"name": "my dynamic name",
"ext": "txt",
"info": "my dynamic info",
"dbkey": "cust1"
"dbkey": "cust1",
"created_from_basename": "my name.txt"
}}
</configfile>
</configfiles>
@@ -29,6 +30,7 @@
<metadata name="name" value="my dynamic name" />
<metadata name="info" value="my dynamic info" />
<metadata name="dbkey" value="cust1" />
<metadata name="created_from_basename" value="my name.txt" />
</output>
</test>
</tests>