diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 35e4dadfbf3..a1927c1ac54 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -17,7 +17,7 @@ from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_ from galaxy.datatypes.metadata import MetadataElement, MetadataParameter, ListParameter, DictParameter from galaxy.datatypes import metadata -from galaxy.util import nice_size, sqlite, which +from galaxy.util import nice_size, sqlite, which, FILENAME_VALID_CHARS from . import data, dataproviders @@ -78,8 +78,7 @@ class Binary( data.Data ): trans.log_event( "Display dataset id: %s" % str( dataset.id ) ) trans.response.headers['Content-Length'] = int( os.stat( dataset.file_name ).st_size ) to_ext = dataset.extension - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - fname = ''.join(c in valid_chars and c or '_' for c in dataset.name)[0:150] + fname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in dataset.name)[0:150] trans.response.set_content_type( "application/octet-stream" ) # force octet-stream so Safari doesn't append mime extensions to filename trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (dataset.hid, fname, to_ext) return open( dataset.file_name ) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 0a2aedd2bf9..abebd6ad216 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -15,6 +15,7 @@ import six from galaxy import util from galaxy.datatypes.metadata import MetadataElement # import directly to maintain ease of use in Datatype class definitions +from galaxy.util import FILENAME_VALID_CHARS from galaxy.util import inflector from galaxy.util import unicodify from galaxy.util.bunch import Bunch @@ -236,9 +237,8 @@ class Data( object ): def _archive_composite_dataset( self, trans, data=None, **kwd ): # save a composite object into a compressed archive for downloading params = util.Params( kwd ) - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' outfname = data.name[0:150] - outfname = ''.join(c in valid_chars and c or '_' for c in outfname) + outfname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in outfname) if params.do_action is None: params.do_action = 'zip' # default msg = util.restore_text( params.get( 'msg', '' ) ) @@ -308,8 +308,7 @@ class Data( object ): def _serve_raw(self, trans, dataset, to_ext): trans.response.headers['Content-Length'] = int( os.stat( dataset.file_name ).st_size ) - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - fname = ''.join(c in valid_chars and c or '_' for c in dataset.name)[0:150] + fname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in dataset.name)[0:150] trans.response.set_content_type( "application/octet-stream" ) # force octet-stream so Safari doesn't append mime extensions to filename trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (dataset.hid, fname, to_ext) return open( dataset.file_name ) @@ -358,8 +357,7 @@ class Data( object ): trans.response.headers['Content-Length'] = int( os.stat( data.file_name ).st_size ) if not to_ext: to_ext = data.extension - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150] + fname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in data.name)[0:150] trans.response.set_content_type( "application/octet-stream" ) # force octet-stream so Safari doesn't append mime extensions to filename trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (data.hid, fname, to_ext) return open( data.file_name ) diff --git a/lib/galaxy/datatypes/neo4j.py b/lib/galaxy/datatypes/neo4j.py index 4dc202608f0..91f8be3a9a7 100644 --- a/lib/galaxy/datatypes/neo4j.py +++ b/lib/galaxy/datatypes/neo4j.py @@ -5,8 +5,10 @@ import logging import sys import shutil import os -from galaxy.datatypes.images import Html + from galaxy.datatypes.data import Data +from galaxy.datatypes.images import Html +from galaxy.util import FILENAME_VALID_CHARS gal_Log = logging.getLogger(__name__) verbose = True @@ -69,8 +71,7 @@ class Neo4j(Html): dir_name = str(os.path.dirname(trans.app.object_store.get_filename(data.dataset))) + neo4j_dir_name # generate unique filename for this dataset - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150] + fname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in data.name)[0:150] # zip the target directory (dir_name) using the fname shutil.make_archive(fname, 'zip', dir_name) diff --git a/lib/galaxy/managers/datasets.py b/lib/galaxy/managers/datasets.py index 0466331f903..4de5b758073 100644 --- a/lib/galaxy/managers/datasets.py +++ b/lib/galaxy/managers/datasets.py @@ -427,7 +427,12 @@ class _UnflattenedMetadataDatasetAssociationSerializer( base.ModelSerializer, meta_files = [] for meta_type in dataset_assoc.metadata.spec.keys(): if isinstance( dataset_assoc.metadata.spec[ meta_type ].param, galaxy.datatypes.metadata.FileParameter ): - meta_files.append( dict( file_type=meta_type ) ) + meta_files.append( + dict( file_type=meta_type, + download_url=self.url_for( 'history_contents_metadata_file', + history_id=self.app.security.encode_id(dataset_assoc.history_id), + history_content_id=self.app.security.encode_id(dataset_assoc.id), + metadata_file=meta_type) ) ) return meta_files def serialize_metadata( self, dataset_assoc, key, excluded=None, **context ): diff --git a/lib/galaxy/tools/imp_exp/export_history.py b/lib/galaxy/tools/imp_exp/export_history.py index f035f6e610e..39520925d59 100644 --- a/lib/galaxy/tools/imp_exp/export_history.py +++ b/lib/galaxy/tools/imp_exp/export_history.py @@ -12,13 +12,14 @@ import sys import tarfile from json import dumps, loads +from galaxy.util import FILENAME_VALID_CHARS + def get_dataset_filename( name, ext, hid ): """ Builds a filename for a dataset using its name an extension. """ - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - base = ''.join( c in valid_chars and c or '_' for c in name ) + base = ''.join( c in FILENAME_VALID_CHARS and c or '_' for c in name ) return base + "_%s.%s" % (hid, ext) @@ -67,7 +68,7 @@ def create_archive( history_attrs_file, datasets_attrs_file, jobs_attrs_file, ou dataset_extra_files_path = 'datasets/extra_files_path_%s' % dataset_hid for fname in file_list: history_archive.add( os.path.join( extra_files_path, fname ), - arcname=( os.path.join( dataset_extra_files_path, fname ) ) ) + arcname=( os.path.join( dataset_extra_files_path, fname ) ) ) dataset_attrs[ 'extra_files_path' ] = dataset_extra_files_path else: dataset_attrs[ 'extra_files_path' ] = '' diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index f208a62efa2..fda218d6a88 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -62,6 +62,7 @@ bz2_magic = 'BZh' DEFAULT_ENCODING = os.environ.get('GALAXY_DEFAULT_ENCODING', 'utf-8') NULL_CHAR = '\000' BINARY_CHARS = [ NULL_CHAR ] +FILENAME_VALID_CHARS = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' def remove_protocol_from_url( url ): diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index 48ece995c38..b58ce404d25 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -317,6 +317,24 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ): return rval + @web.expose_api_raw_anonymous + def get_metadata_file( self, trans, history_content_id, history_id, metadata_file=None, **kwd ): + decoded_content_id = self.decode_id( history_content_id ) + rval = '' + try: + hda = self.hda_manager.get_accessible( decoded_content_id, trans.user ) + file_ext = hda.metadata.spec.get(metadata_file).get("file_ext", metadata_file) + fname = ''.join(c in util.FILENAME_VALID_CHARS and c or '_' for c in hda.name)[0:150] + trans.response.headers["Content-Type"] = "application/octet-stream" + trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (hda.hid, fname, file_ext) + return open(hda.metadata.get(metadata_file).file_name) + except Exception as exception: + log.error( "Error getting metadata_file (%s) for dataset (%s) from history (%s): %s", + metadata_file, history_content_id, history_id, str( exception ), exc_info=True ) + trans.response.status = 500 + rval = ( "Could not get display data for dataset: " + str( exception ) ) + return rval + @web._future_expose_api_anonymous def converted( self, trans, dataset_id, ext, **kwargs ): """ diff --git a/lib/galaxy/webapps/galaxy/api/lda_datasets.py b/lib/galaxy/webapps/galaxy/api/lda_datasets.py index 8e94edb29d2..14dc75158b7 100644 --- a/lib/galaxy/webapps/galaxy/api/lda_datasets.py +++ b/lib/galaxy/webapps/galaxy/api/lda_datasets.py @@ -709,9 +709,8 @@ class LibraryDatasetsController( BaseAPIController, UsesVisualizationMixin ): fStat = os.stat( dataset.file_name ) trans.response.set_content_type( ldda.get_mime() ) trans.response.headers[ 'Content-Length' ] = int( fStat.st_size ) - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' fname = ldda.name - fname = ''.join( c in valid_chars and c or '_' for c in fname )[ 0:150 ] + fname = ''.join( c in util.FILENAME_VALID_CHARS and c or '_' for c in fname )[ 0:150 ] trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s"' % fname try: return open( dataset.file_name ) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 40eb4623303..5643d7afdc4 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -197,6 +197,11 @@ def populate_api_routes( webapp, app ): controller="datasets", action="display", conditions=dict(method=["GET"])) + webapp.mapper.connect( "history_contents_metadata_file", + "/api/histories/{history_id}/contents/{history_content_id}/metadata_file", + controller="datasets", + action="get_metadata_file", + conditions=dict(method=["GET"])) webapp.mapper.resource( 'user', 'users', controller='group_users', diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index 184019204d0..d90dc300b1a 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -181,8 +181,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesItemRatings, Uses if not data or not self._can_access_dataset( trans, data ): return trans.show_error_message( "You are not allowed to access this dataset" ) - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150] + fname = ''.join(c in util.FILENAME_VALID_CHARS and c or '_' for c in data.name)[0:150] file_ext = data.metadata.spec.get(metadata_name).get("file_ext", metadata_name) trans.response.headers["Content-Type"] = "application/octet-stream" diff --git a/lib/galaxy/webapps/galaxy/controllers/library_common.py b/lib/galaxy/webapps/galaxy/controllers/library_common.py index 9fe8f5b605c..6f58eb17291 100644 --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -19,7 +19,7 @@ from sqlalchemy.orm import eagerload_all from galaxy import util, web from galaxy.security import Action from galaxy.tools.actions import upload_common -from galaxy.util import inflector, unicodify +from galaxy.util import inflector, unicodify, FILENAME_VALID_CHARS from galaxy.util.streamball import StreamBall from galaxy.web.base.controller import BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMetadataMixin, UsesLibraryMixinItems from galaxy.web.form_builder import AddressField, CheckboxField, SelectField, build_select_field @@ -1519,9 +1519,8 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet trans.response.set_content_type( ldda.get_mime() ) fStat = os.stat( ldda.file_name ) trans.response.headers[ 'Content-Length' ] = int( fStat.st_size ) - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' fname = ldda.name - fname = ''.join( c in valid_chars and c or '_' for c in fname )[ 0:150 ] + fname = ''.join( c in FILENAME_VALID_CHARS and c or '_' for c in fname )[ 0:150 ] trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s"' % fname try: return open( ldda.file_name ) diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index 4f6829d2927..4e593424e96 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -9,7 +9,7 @@ from paste.httpexceptions import HTTPNotFound, HTTPBadGateway from galaxy import web from galaxy import util -from galaxy.util import listify, Params, string_as_bool +from galaxy.util import listify, Params, string_as_bool, FILENAME_VALID_CHARS from galaxy.web.base import controller from galaxy.model.item_attrs import UsesAnnotations @@ -217,9 +217,8 @@ class RootController( controller.JSAppLauncher, UsesAnnotations ): trans.response.headers['Content-Length'] = int(fStat.st_size) if toext[0:1] != ".": toext = "." + toext - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' fname = data.name - fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150] + fname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in fname)[0:150] trans.response.headers["Content-Disposition"] = 'attachment; filename="GalaxyHistoryItem-%s-[%s]%s"' % (data.hid, fname, toext) trans.log_event( "Display dataset id: %s" % str(id) ) try: diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index 3381b569654..2489e91d050 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -18,7 +18,7 @@ from galaxy import web from galaxy.managers import workflows from galaxy.model.item_attrs import UsesItemRatings from galaxy.model.mapping import desc -from galaxy.util import unicodify +from galaxy.util import unicodify, FILENAME_VALID_CHARS from galaxy.util.sanitize_html import sanitize_html from galaxy.web import error, url_for from galaxy.web.base.controller import BaseUIController, SharableMixin, UsesStoredWorkflowMixin @@ -815,9 +815,8 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix # This workflow has a tool that's missing from the distribution trans.response.status = 400 return "Workflow cannot be exported due to missing tools." - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' sname = stored.name - sname = ''.join(c in valid_chars and c or '_' for c in sname)[0:150] + sname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in sname)[0:150] trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy-Workflow-%s.ga"' % ( sname ) trans.response.set_content_type( 'application/galaxy-archive' ) return stored_dict diff --git a/tools/genomespace/genomespace_file_browser.py b/tools/genomespace/genomespace_file_browser.py index 18c9c342bf9..3fceb937eea 100644 --- a/tools/genomespace/genomespace_file_browser.py +++ b/tools/genomespace/genomespace_file_browser.py @@ -13,6 +13,7 @@ from galaxy.datatypes.registry import Registry GENOMESPACE_API_VERSION_STRING = "v1.0" GENOMESPACE_SERVER_URL_PROPERTIES = "https://dm.genomespace.org/config/%s/serverurl.properties" % ( GENOMESPACE_API_VERSION_STRING ) DEFAULT_GENOMESPACE_TOOLNAME = 'Galaxy' +FILENAME_VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ' CHUNK_SIZE = 2**20 # 1mb @@ -46,8 +47,6 @@ GENOMESPACE_EXT_TO_GALAXY_EXT = {'rifles': 'rifles', GENOMESPACE_UNKNOWN_FORMAT_KEY = 'unknown' GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN = None -VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ' - def chunk_write( source_stream, target_stream, source_method="read", target_method="write" ): source_method = getattr( source_stream, source_method ) @@ -166,7 +165,7 @@ def download_from_genomespace_file_browser( json_parameter_file, genomespace_sit metadata_dict = None original_filename = filename if output_filename is None: - filename = ''.join( c in VALID_CHARS and c or '-' for c in filename ) + filename = ''.join( c in FILENAME_VALID_CHARS and c or '-' for c in filename ) while filename in used_filenames: filename = "-%s" % filename used_filenames.append( filename ) diff --git a/tools/genomespace/genomespace_importer.py b/tools/genomespace/genomespace_importer.py index e356d573e14..f1bbef46ff9 100644 --- a/tools/genomespace/genomespace_importer.py +++ b/tools/genomespace/genomespace_importer.py @@ -16,6 +16,7 @@ from galaxy.datatypes.registry import Registry GENOMESPACE_API_VERSION_STRING = "v1.0" GENOMESPACE_SERVER_URL_PROPERTIES = "https://dm.genomespace.org/config/%s/serverurl.properties" % ( GENOMESPACE_API_VERSION_STRING ) DEFAULT_GENOMESPACE_TOOLNAME = 'Galaxy' +FILENAME_VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ' CHUNK_SIZE = 2**20 # 1mb @@ -45,8 +46,6 @@ GENOMESPACE_EXT_TO_GALAXY_EXT = {'rifles': 'rifles', 'gmt': 'gmt', 'gct': 'gct'} -VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ' - def chunk_write( source_stream, target_stream, source_method="read", target_method="write" ): source_method = getattr( source_stream, source_method ) @@ -192,7 +191,7 @@ def download_from_genomespace_importer( username, token, json_parameter_file, ge # if using tmp file, move the file to the new file path dir to get scooped up later if using_temp_file: original_filename = filename - filename = ''.join( c in VALID_CHARS and c or '-' for c in filename ) + filename = ''.join( c in FILENAME_VALID_CHARS and c or '-' for c in filename ) while filename in used_filenames: filename = "-%s" % filename used_filenames.append( filename )