From 69b447169fb96048ea492a1d99d653585571c15b Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 1 Nov 2018 15:22:15 -0400 Subject: [PATCH 1/3] Allow deferring 'to_ext' extension to the server, implement for collections. --- client/galaxy/scripts/mvc/collection/collection-model.js | 4 +++- lib/galaxy/datatypes/data.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/client/galaxy/scripts/mvc/collection/collection-model.js b/client/galaxy/scripts/mvc/collection/collection-model.js index 75d0b030fb5..a3cc67371b4 100644 --- a/client/galaxy/scripts/mvc/collection/collection-model.js +++ b/client/galaxy/scripts/mvc/collection/collection-model.js @@ -149,7 +149,9 @@ var DatasetDCE = DATASET_MODEL.DatasetAssociation.extend( ), _downloadQueryParameters: function() { - var fileExt = this.get("file_ext"); + // Setting the file extension to just 'data' defers that + // decision to the serverside, setting based on the datatype. + var fileExt = this.get("file_ext") || "data"; var elementIdentifier = this.get("element_identifier"); var parentHdcaId = this.get("parent_hdca_id"); return `?to_ext=${fileExt}&hdca_id=${parentHdcaId}&element_identifier=${elementIdentifier}`; diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 0dbeeb6493d..0eabb3b2d8b 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -452,7 +452,9 @@ class Data(object): def escape(raw_identifier): return ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in raw_identifier)[0:150] - if not to_ext: + if not to_ext or to_ext == "data": + # If a client requests to_ext with the extension 'data', they are + # deferring to the server, set it based on datatype. to_ext = dataset.extension template_values = { From 3be8d66d71bcdb3eccde2d1a37731442ed1aa63a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 5 Nov 2018 10:04:48 +0100 Subject: [PATCH 2/3] Remove unncecessary implementations of display_data These were not updated to use `_download_filename`, and so this fixes the download name for collection elements to include the element identifier. Note that the Data class should handle composite data just fine. --- lib/galaxy/datatypes/binary.py | 12 +---------- lib/galaxy/datatypes/data.py | 9 ++++---- lib/galaxy/datatypes/molecules.py | 8 ------- lib/galaxy/datatypes/neo4j.py | 36 ------------------------------- 4 files changed, 6 insertions(+), 59 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 980aa88802f..a36f0ff6c74 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -23,7 +23,7 @@ from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_ from galaxy import util from galaxy.datatypes import metadata from galaxy.datatypes.metadata import DictParameter, ListParameter, MetadataElement, MetadataParameter -from galaxy.util import FILENAME_VALID_CHARS, nice_size, sqlite +from galaxy.util import nice_size, sqlite from galaxy.util.checkers import is_bz2, is_gzip from . import data, dataproviders @@ -59,16 +59,6 @@ class Binary(data.Data): """Returns the mime type of the datatype""" return 'application/octet-stream' - def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, **kwd): - trans.response.set_content_type(dataset.get_mime()) - trans.log_event("Display dataset id: %s" % str(dataset.id)) - trans.response.headers['Content-Length'] = int(os.stat(dataset.file_name).st_size) - to_ext = dataset.extension - fname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in dataset.name)[0:150] - trans.response.set_content_type("application/octet-stream") # force octet-stream so Safari doesn't append mime extensions to filename - trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (dataset.hid, fname, to_ext) - return open(dataset.file_name, mode='rb') - class Ab1(Binary): """Class describing an ab1 binary sequence file""" diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 0eabb3b2d8b..80f9766807e 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -353,10 +353,11 @@ class Data(object): return zip(file_paths, rel_paths) def display_data(self, trans, data, preview=False, filename=None, to_ext=None, **kwd): - """ Old display method, for transition - though still used by API and - test framework. Datatypes should be very careful if overridding this - method and this interface between datatypes and Galaxy will likely - change. + """ + Displays data in central pane if preview is `True`, else handles download. + + Datatypes should be very careful if overridding this method and this interface + between datatypes and Galaxy will likely change. TOOD: Document alternatives to overridding this method (data providers?). diff --git a/lib/galaxy/datatypes/molecules.py b/lib/galaxy/datatypes/molecules.py index 8d29e58f4d1..ee4f4dd42be 100644 --- a/lib/galaxy/datatypes/molecules.py +++ b/lib/galaxy/datatypes/molecules.py @@ -423,14 +423,6 @@ class OBFS(Binary): except Exception: return "OpenBabel Fastsearch Index" - def display_data(self, trans, data, preview=False, filename=None, - to_ext=None, **kwd): - """Apparently an old display method, but still gets called. - - This allows us to format the data shown in the central pane via the "eye" icon. - """ - return b"This is a OpenBabel Fastsearch format. You can speed up your similarity and substructure search with it." - def get_mime(self): """Returns the mime type of the datatype (pretend it is text for peek)""" return 'text/plain' diff --git a/lib/galaxy/datatypes/neo4j.py b/lib/galaxy/datatypes/neo4j.py index 36340d1c263..dba4d9e5f36 100644 --- a/lib/galaxy/datatypes/neo4j.py +++ b/lib/galaxy/datatypes/neo4j.py @@ -2,14 +2,11 @@ Neo4j Composite Dataset """ import logging -import os -import shutil import sys from galaxy.datatypes.data import Data from galaxy.datatypes.images import Html from galaxy.datatypes.metadata import MetadataElement -from galaxy.util import FILENAME_VALID_CHARS gal_Log = logging.getLogger(__name__) verbose = True @@ -61,39 +58,6 @@ class Neo4j(Html): except Exception: return "NEO4J database (multiple files)" - def display_data(self, trans, data, preview=False, filename=None, - to_ext=None, size=None, offset=None, **kwd): - """Documented as an old display method, but still gets called via tests etc - This allows us to format the data shown in the central pane via the "eye" icon. - """ - if not preview: - trans.response.set_content_type(data.get_mime()) - trans.log_event("Display dataset id: %s" % str(data.id)) - - # the target directory name - neo4j_dir_name = '/dataset_{}_files'.format( - data.dataset.id) - dir_name = str(os.path.dirname( - trans.app.object_store.get_filename(data.dataset))) + neo4j_dir_name - - # generate unique filename for this dataset - fname = ''.join( - c in FILENAME_VALID_CHARS and c or '_' for c in data.name)[0:150] - - # zip the target directory (dir_name) using the fname - shutil.make_archive(fname, 'zip', dir_name) - download_zip = fname + '.zip' - - # setup headers for the download - trans.response.headers['Content-Length'] = int( - os.stat(download_zip).st_size) - # force octet-stream so Safari doesn't append mime extensions to filename - trans.response.set_content_type("application/octet-stream") - trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % \ - (data.hid, - download_zip, "zip") - return open(download_zip, mode='rb') - class Neo4jDB(Neo4j, Data): """Class for neo4jDB database files.""" From f8b160535c5f9363e85c6c1d041b69ce5c3e9cc2 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 5 Nov 2018 10:07:56 +0100 Subject: [PATCH 3/3] Some minor fixes for the blast datatype The signature for Data.display_data wasn't quite correct, and the description of display_data was also a little misleading. --- lib/galaxy/datatypes/blast.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/datatypes/blast.py b/lib/galaxy/datatypes/blast.py index f1924449d34..0e3b2265b30 100644 --- a/lib/galaxy/datatypes/blast.py +++ b/lib/galaxy/datatypes/blast.py @@ -30,10 +30,12 @@ Covers the ``blastxml`` format and the BLAST databases. """ +import io import logging import os from time import sleep +from galaxy.util import smart_str from .data import ( Data, get_file_peek, @@ -98,6 +100,7 @@ class BlastXml(GenericXml): % (split_files, output_file)) with open(output_file, "w") as out: h = None + old_header = None for f in split_files: if not os.path.isfile(f): log.warning("BLAST XML file %s missing, retry in 1s..." % f) @@ -149,7 +152,7 @@ class BlastXml(GenericXml): if f == split_files[0]: out.write(header) old_header = header - elif old_header[:300] != header[:300]: + elif old_header is not None and old_header[:300] != header[:300]: # Enough to check and match h.close() raise ValueError("BLAST XML headers don't match for %s and %s - have:\n%s\n...\n\nAnd:\n%s\n...\n" @@ -168,7 +171,7 @@ class BlastXml(GenericXml): merge = staticmethod(merge) -class _BlastDb(object): +class _BlastDb(Data): """Base class for BLAST database datatype.""" def set_peek(self, dataset, is_multi_byte=False): @@ -189,14 +192,19 @@ class _BlastDb(object): def display_data(self, trans, data, preview=False, filename=None, to_ext=None, size=None, offset=None, **kwd): - """Documented as an old display method, but still gets called via tests etc - - This allows us to format the data shown in the central pane via the "eye" icon. """ - if filename is not None and filename != "index": - # Change nothing - important for the unit tests to access child files: - return Data.display_data(self, trans, data, preview, filename, - to_ext, size, offset, **kwd) + If preview is `True` allows us to format the data shown in the central pane via the "eye" icon. + If preview is `False` triggers download. + """ + if not preview: + return super(_BlastDb, self).display_data(trans, + data=data, + preview=preview, + filename=filename, + to_ext=to_ext, + size=size, + offset=offset, + **kwd) if self.file_ext == "blastdbn": title = "This is a nucleotide BLAST database" elif self.file_ext == "blastdbp": @@ -209,14 +217,14 @@ class _BlastDb(object): msg = "" try: # Try to use any text recorded in the dummy index file: - with open(data.file_name, "rU") as handle: + with io.open(data.file_name, "rU", encoding='utf-8') as handle: msg = handle.read().strip() except Exception: pass if not msg: msg = title # Galaxy assumes HTML for the display of composite datatypes, - return "%s
%s
" % (title, msg) + return smart_str("%s
%s
" % (title, msg)) def merge(split_files, output_file): """Merge BLAST databases (not implemented for now)."""