opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2741 from dannon/expose_metadata_api

Browse Source 
Metadata file fetching via the API
This commit is contained in:
John Chilton
2016-08-11 11:51:00 -04:00
committed by GitHub
parent 49f58dd017 c9ac723174
commit b40e9938d6
15 changed files with 56 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/datatypes/binary.py
+2 -3
View File
@@ -17,7 +17,7 @@ from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_
from galaxy.datatypes.metadata import MetadataElement, MetadataParameter, ListParameter, DictParameter
from galaxy.datatypes import metadata
from galaxy.util import nice_size, sqlite, which
from galaxy.util import nice_size, sqlite, which, FILENAME_VALID_CHARS
from . import data, dataproviders
@@ -78,8 +78,7 @@ class Binary( data.Data ):
trans.log_event( "Display dataset id: %s" % str( dataset.id ) )
trans.response.headers['Content-Length'] = int( os.stat( dataset.file_name ).st_size )
to_ext = dataset.extension
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
fname = ''.join(c in valid_chars and c or '_' for c in dataset.name)[0:150]
fname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in dataset.name)[0:150]
trans.response.set_content_type( "application/octet-stream" ) # force octet-stream so Safari doesn't append mime extensions to filename
trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (dataset.hid, fname, to_ext)
return open( dataset.file_name )
lib/galaxy/datatypes/data.py
+4 -6
View File
@@ -15,6 +15,7 @@ import six
from galaxy import util
from galaxy.datatypes.metadata import MetadataElement # import directly to maintain ease of use in Datatype class definitions
from galaxy.util import FILENAME_VALID_CHARS
from galaxy.util import inflector
from galaxy.util import unicodify
from galaxy.util.bunch import Bunch
@@ -236,9 +237,8 @@ class Data( object ):
def _archive_composite_dataset( self, trans, data=None, **kwd ):
# save a composite object into a compressed archive for downloading
params = util.Params( kwd )
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
outfname = data.name[0:150]
outfname = ''.join(c in valid_chars and c or '_' for c in outfname)
outfname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in outfname)
if params.do_action is None:
params.do_action = 'zip' # default
msg = util.restore_text( params.get( 'msg', '' ) )
@@ -308,8 +308,7 @@ class Data( object ):
def _serve_raw(self, trans, dataset, to_ext):
trans.response.headers['Content-Length'] = int( os.stat( dataset.file_name ).st_size )
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
fname = ''.join(c in valid_chars and c or '_' for c in dataset.name)[0:150]
fname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in dataset.name)[0:150]
trans.response.set_content_type( "application/octet-stream" ) # force octet-stream so Safari doesn't append mime extensions to filename
trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (dataset.hid, fname, to_ext)
return open( dataset.file_name )
@@ -358,8 +357,7 @@ class Data( object ):
trans.response.headers['Content-Length'] = int( os.stat( data.file_name ).st_size )
if not to_ext:
to_ext = data.extension
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150]
fname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in data.name)[0:150]
trans.response.set_content_type( "application/octet-stream" ) # force octet-stream so Safari doesn't append mime extensions to filename
trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (data.hid, fname, to_ext)
return open( data.file_name )
lib/galaxy/datatypes/neo4j.py
+4 -3
View File
@@ -5,8 +5,10 @@ import logging
import sys
import shutil
import os
from galaxy.datatypes.images import Html
from galaxy.datatypes.data import Data
from galaxy.datatypes.images import Html
from galaxy.util import FILENAME_VALID_CHARS
gal_Log = logging.getLogger(__name__)
verbose = True
@@ -69,8 +71,7 @@ class Neo4j(Html):
dir_name = str(os.path.dirname(trans.app.object_store.get_filename(data.dataset))) + neo4j_dir_name
# generate unique filename for this dataset
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150]
fname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in data.name)[0:150]
# zip the target directory (dir_name) using the fname
shutil.make_archive(fname, 'zip', dir_name)
lib/galaxy/managers/datasets.py
+6 -1
View File
@@ -427,7 +427,12 @@ class _UnflattenedMetadataDatasetAssociationSerializer( base.ModelSerializer,
meta_files = []
for meta_type in dataset_assoc.metadata.spec.keys():
if isinstance( dataset_assoc.metadata.spec[ meta_type ].param, galaxy.datatypes.metadata.FileParameter ):
meta_files.append( dict( file_type=meta_type ) )
meta_files.append(
dict( file_type=meta_type,
download_url=self.url_for( 'history_contents_metadata_file',
history_id=self.app.security.encode_id(dataset_assoc.history_id),
history_content_id=self.app.security.encode_id(dataset_assoc.id),
metadata_file=meta_type) ) )
return meta_files
def serialize_metadata( self, dataset_assoc, key, excluded=None, **context ):
lib/galaxy/tools/imp_exp/export_history.py
+4 -3
View File
@@ -12,13 +12,14 @@ import sys
import tarfile
from json import dumps, loads
from galaxy.util import FILENAME_VALID_CHARS
def get_dataset_filename( name, ext, hid ):
"""
Builds a filename for a dataset using its name an extension.
"""
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
base = ''.join( c in valid_chars and c or '_' for c in name )
base = ''.join( c in FILENAME_VALID_CHARS and c or '_' for c in name )
return base + "_%s.%s" % (hid, ext)
@@ -67,7 +68,7 @@ def create_archive( history_attrs_file, datasets_attrs_file, jobs_attrs_file, ou
dataset_extra_files_path = 'datasets/extra_files_path_%s' % dataset_hid
for fname in file_list:
history_archive.add( os.path.join( extra_files_path, fname ),
arcname=( os.path.join( dataset_extra_files_path, fname ) ) )
arcname=( os.path.join( dataset_extra_files_path, fname ) ) )
dataset_attrs[ 'extra_files_path' ] = dataset_extra_files_path
else:
dataset_attrs[ 'extra_files_path' ] = ''
lib/galaxy/util/__init__.py
+1
View File
@@ -62,6 +62,7 @@ bz2_magic = 'BZh'
DEFAULT_ENCODING = os.environ.get('GALAXY_DEFAULT_ENCODING', 'utf-8')
NULL_CHAR = '\000'
BINARY_CHARS = [ NULL_CHAR ]
FILENAME_VALID_CHARS = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
def remove_protocol_from_url( url ):
lib/galaxy/webapps/galaxy/api/datasets.py
+18
View File
@@ -317,6 +317,24 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ):
return rval
@web.expose_api_raw_anonymous
def get_metadata_file( self, trans, history_content_id, history_id, metadata_file=None, **kwd ):
decoded_content_id = self.decode_id( history_content_id )
rval = ''
try:
hda = self.hda_manager.get_accessible( decoded_content_id, trans.user )
file_ext = hda.metadata.spec.get(metadata_file).get("file_ext", metadata_file)
fname = ''.join(c in util.FILENAME_VALID_CHARS and c or '_' for c in hda.name)[0:150]
trans.response.headers["Content-Type"] = "application/octet-stream"
trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (hda.hid, fname, file_ext)
return open(hda.metadata.get(metadata_file).file_name)
except Exception as exception:
log.error( "Error getting metadata_file (%s) for dataset (%s) from history (%s): %s",
metadata_file, history_content_id, history_id, str( exception ), exc_info=True )
trans.response.status = 500
rval = ( "Could not get display data for dataset: " + str( exception ) )
return rval
@web._future_expose_api_anonymous
def converted( self, trans, dataset_id, ext, **kwargs ):
"""
lib/galaxy/webapps/galaxy/api/lda_datasets.py
+1 -2
View File
@@ -709,9 +709,8 @@ class LibraryDatasetsController( BaseAPIController, UsesVisualizationMixin ):
fStat = os.stat( dataset.file_name )
trans.response.set_content_type( ldda.get_mime() )
trans.response.headers[ 'Content-Length' ] = int( fStat.st_size )
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
fname = ldda.name
fname = ''.join( c in valid_chars and c or '_' for c in fname )[ 0:150 ]
fname = ''.join( c in util.FILENAME_VALID_CHARS and c or '_' for c in fname )[ 0:150 ]
trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s"' % fname
try:
return open( dataset.file_name )
lib/galaxy/webapps/galaxy/buildapp.py
+5
View File
@@ -197,6 +197,11 @@ def populate_api_routes( webapp, app ):
controller="datasets",
action="display",
conditions=dict(method=["GET"]))
webapp.mapper.connect( "history_contents_metadata_file",
"/api/histories/{history_id}/contents/{history_content_id}/metadata_file",
controller="datasets",
action="get_metadata_file",
conditions=dict(method=["GET"]))
webapp.mapper.resource( 'user',
'users',
controller='group_users',
lib/galaxy/webapps/galaxy/controllers/dataset.py
+1 -2
View File
@@ -181,8 +181,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesItemRatings, Uses
if not data or not self._can_access_dataset( trans, data ):
return trans.show_error_message( "You are not allowed to access this dataset" )
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150]
fname = ''.join(c in util.FILENAME_VALID_CHARS and c or '_' for c in data.name)[0:150]
file_ext = data.metadata.spec.get(metadata_name).get("file_ext", metadata_name)
trans.response.headers["Content-Type"] = "application/octet-stream"
lib/galaxy/webapps/galaxy/controllers/library_common.py
+2 -3
View File
@@ -19,7 +19,7 @@ from sqlalchemy.orm import eagerload_all
from galaxy import util, web
from galaxy.security import Action
from galaxy.tools.actions import upload_common
from galaxy.util import inflector, unicodify
from galaxy.util import inflector, unicodify, FILENAME_VALID_CHARS
from galaxy.util.streamball import StreamBall
from galaxy.web.base.controller import BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMetadataMixin, UsesLibraryMixinItems
from galaxy.web.form_builder import AddressField, CheckboxField, SelectField, build_select_field
@@ -1519,9 +1519,8 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet
trans.response.set_content_type( ldda.get_mime() )
fStat = os.stat( ldda.file_name )
trans.response.headers[ 'Content-Length' ] = int( fStat.st_size )
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
fname = ldda.name
fname = ''.join( c in valid_chars and c or '_' for c in fname )[ 0:150 ]
fname = ''.join( c in FILENAME_VALID_CHARS and c or '_' for c in fname )[ 0:150 ]
trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s"' % fname
try:
return open( ldda.file_name )
lib/galaxy/webapps/galaxy/controllers/root.py
+2 -3
View File
@@ -9,7 +9,7 @@ from paste.httpexceptions import HTTPNotFound, HTTPBadGateway
from galaxy import web
from galaxy import util
from galaxy.util import listify, Params, string_as_bool
from galaxy.util import listify, Params, string_as_bool, FILENAME_VALID_CHARS
from galaxy.web.base import controller
from galaxy.model.item_attrs import UsesAnnotations
@@ -217,9 +217,8 @@ class RootController( controller.JSAppLauncher, UsesAnnotations ):
trans.response.headers['Content-Length'] = int(fStat.st_size)
if toext[0:1] != ".":
toext = "." + toext
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
fname = data.name
fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150]
fname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in fname)[0:150]
trans.response.headers["Content-Disposition"] = 'attachment; filename="GalaxyHistoryItem-%s-[%s]%s"' % (data.hid, fname, toext)
trans.log_event( "Display dataset id: %s" % str(id) )
try:
lib/galaxy/webapps/galaxy/controllers/workflow.py
+2 -3
View File
@@ -18,7 +18,7 @@ from galaxy import web
from galaxy.managers import workflows
from galaxy.model.item_attrs import UsesItemRatings
from galaxy.model.mapping import desc
from galaxy.util import unicodify
from galaxy.util import unicodify, FILENAME_VALID_CHARS
from galaxy.util.sanitize_html import sanitize_html
from galaxy.web import error, url_for
from galaxy.web.base.controller import BaseUIController, SharableMixin, UsesStoredWorkflowMixin
@@ -815,9 +815,8 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix
# This workflow has a tool that's missing from the distribution
trans.response.status = 400
return "Workflow cannot be exported due to missing tools."
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
sname = stored.name
sname = ''.join(c in valid_chars and c or '_' for c in sname)[0:150]
sname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in sname)[0:150]
trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy-Workflow-%s.ga"' % ( sname )
trans.response.set_content_type( 'application/galaxy-archive' )
return stored_dict
tools/genomespace/genomespace_file_browser.py
+2 -3
View File
@@ -13,6 +13,7 @@ from galaxy.datatypes.registry import Registry
GENOMESPACE_API_VERSION_STRING = "v1.0"
GENOMESPACE_SERVER_URL_PROPERTIES = "https://dm.genomespace.org/config/%s/serverurl.properties" % ( GENOMESPACE_API_VERSION_STRING )
DEFAULT_GENOMESPACE_TOOLNAME = 'Galaxy'
FILENAME_VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ '
CHUNK_SIZE = 2**20 # 1mb
@@ -46,8 +47,6 @@ GENOMESPACE_EXT_TO_GALAXY_EXT = {'rifles': 'rifles',
GENOMESPACE_UNKNOWN_FORMAT_KEY = 'unknown'
GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN = None
VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ '
def chunk_write( source_stream, target_stream, source_method="read", target_method="write" ):
source_method = getattr( source_stream, source_method )
@@ -166,7 +165,7 @@ def download_from_genomespace_file_browser( json_parameter_file, genomespace_sit
metadata_dict = None
original_filename = filename
if output_filename is None:
filename = ''.join( c in VALID_CHARS and c or '-' for c in filename )
filename = ''.join( c in FILENAME_VALID_CHARS and c or '-' for c in filename )
while filename in used_filenames:
filename = "-%s" % filename
used_filenames.append( filename )
tools/genomespace/genomespace_importer.py
+2 -3
View File
@@ -16,6 +16,7 @@ from galaxy.datatypes.registry import Registry
GENOMESPACE_API_VERSION_STRING = "v1.0"
GENOMESPACE_SERVER_URL_PROPERTIES = "https://dm.genomespace.org/config/%s/serverurl.properties" % ( GENOMESPACE_API_VERSION_STRING )
DEFAULT_GENOMESPACE_TOOLNAME = 'Galaxy'
FILENAME_VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ '
CHUNK_SIZE = 2**20 # 1mb
@@ -45,8 +46,6 @@ GENOMESPACE_EXT_TO_GALAXY_EXT = {'rifles': 'rifles',
'gmt': 'gmt',
'gct': 'gct'}
VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ '
def chunk_write( source_stream, target_stream, source_method="read", target_method="write" ):
source_method = getattr( source_stream, source_method )
@@ -192,7 +191,7 @@ def download_from_genomespace_importer( username, token, json_parameter_file, ge
# if using tmp file, move the file to the new file path dir to get scooped up later
if using_temp_file:
original_filename = filename
filename = ''.join( c in VALID_CHARS and c or '-' for c in filename )
filename = ''.join( c in FILENAME_VALID_CHARS and c or '-' for c in filename )
while filename in used_filenames:
filename = "-%s" % filename
used_filenames.append( filename )