From a25c5c025a28f837441d67f024a3c7e5d219dec0 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Tue, 17 Mar 2015 13:08:06 -0400 Subject: [PATCH 1/3] Some basic Caching for the GenomeSpace Export Tool's list of available target directories, so we don't have to reload and download everything every time, especially now that we are calling the parameter's get options method 5 times (6 on reload) when a user loads the tool interface. For now, we'll use 30 seconds as the cache valid time length. (cherry picked from commit feb0a26f8d24c6eb008c02f00230777d978a62cb) --- tools/genomespace/genomespace_exporter.py | 44 ++++++++++++++++++----- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/tools/genomespace/genomespace_exporter.py b/tools/genomespace/genomespace_exporter.py index 7540b2fb5ef..a5bd30dfa09 100644 --- a/tools/genomespace/genomespace_exporter.py +++ b/tools/genomespace/genomespace_exporter.py @@ -4,6 +4,7 @@ import base64 import binascii import cgi import cookielib +import datetime import hashlib import json import logging @@ -20,6 +21,13 @@ GENOMESPACE_SERVER_URL_PROPERTIES = "https://dm.genomespace.org/config/%s/server CHUNK_SIZE = 2**20 #1mb +# Some basic Caching, so we don't have to reload and download everything every time, +# especially now that we are calling the parameter's get options method 5 times +# (6 on reload) when a user loads the tool interface +# For now, we'll use 30 seconds as the cache valid time +CACHE_TIME = datetime.timedelta( seconds=30 ) +GENOMESPACE_DIRECTORIES_BY_USER = {} + def chunk_write( source_stream, target_stream, source_method = "read", target_method="write" ): source_method = getattr( source_stream, source_method ) @@ -150,14 +158,32 @@ def galaxy_code_get_genomespace_folders( genomespace_site='prod', trans=None, va username = trans.user.preferences.get( 'genomespace_username', None ) token = trans.user.preferences.get( 'genomespace_token', None ) if None not in ( username, token ): - url_opener = get_cookie_opener( username, token ) - genomespace_site_dict = get_genomespace_site_urls()[ genomespace_site ] - dm_url = genomespace_site_dict['dmServer'] - #get export root directory - #directory_dict = get_default_directory( url_opener, dm_url ).get( 'directory', None ) #This directory contains shares and other items outside of the users home - directory_dict = get_personal_directory( url_opener, dm_url ).get( 'directory', None ) #Limit export list to only user's home dir - if directory_dict is not None: - recurse_directory_dict( url_opener, rval, directory_dict.get( 'url' ) ) + # NB: it is possible, but unlikely for a user to swap GenomeSpace accounts around + # in the middle of interacting with tools, so we'll have several layers of caching by ids/values + if trans.user in GENOMESPACE_DIRECTORIES_BY_USER: + if username in GENOMESPACE_DIRECTORIES_BY_USER[ trans.user ]: + if token in GENOMESPACE_DIRECTORIES_BY_USER[ trans.user ][ username ]: + cache_dict = GENOMESPACE_DIRECTORIES_BY_USER[ trans.user ][ username ][ token ] + if datetime.datetime.now() - cache_dict.get( 'time_loaded' ) > CACHE_TIME: + # cache too old, need to reload, we'll just kill the whole trans.user + del GENOMESPACE_DIRECTORIES_BY_USER[ trans.user ] + else: + rval = cache_dict.get( 'rval' ) + else: + del GENOMESPACE_DIRECTORIES_BY_USER[ trans.user ] + else: + del GENOMESPACE_DIRECTORIES_BY_USER[ trans.user ] + if not rval: + url_opener = get_cookie_opener( username, token ) + genomespace_site_dict = get_genomespace_site_urls()[ genomespace_site ] + dm_url = genomespace_site_dict['dmServer'] + #get export root directory + #directory_dict = get_default_directory( url_opener, dm_url ).get( 'directory', None ) #This directory contains shares and other items outside of the users home + directory_dict = get_personal_directory( url_opener, dm_url ).get( 'directory', None ) #Limit export list to only user's home dir + if directory_dict is not None: + recurse_directory_dict( url_opener, rval, directory_dict.get( 'url' ) ) + # Save the cache + GENOMESPACE_DIRECTORIES_BY_USER[ trans.user ] = { username: { token: { 'time_loaded': datetime.datetime.now(), 'rval': rval } } } if not rval: if not base_url: base_url = '..' @@ -180,7 +206,7 @@ def send_file_to_genomespace( genomespace_site, username, token, source_filename #get upload url upload_url = "uploadurl" content_length = os.path.getsize( source_filename ) - input_file = open( source_filename ) + input_file = open( source_filename, 'rb' ) content_md5 = hashlib.md5() chunk_write( input_file, content_md5, target_method="update" ) input_file.seek( 0 ) #back to start, for uploading From 153bef1c52948a8ef12c027399bf04b83160bd6f Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Thu, 2 Apr 2015 10:16:02 -0400 Subject: [PATCH 2/3] Enable multi-part upload for the GenomeSpace export tool. Files greater than 5gb can once again be send to GenomeSpace from Galaxy. (cherry picked from commit 126ae073afd39a83db90b3f8002f714be9dfdec2) --- tools/genomespace/genomespace_exporter.py | 100 +++++++++++++++++---- tools/genomespace/genomespace_exporter.xml | 4 +- 2 files changed, 83 insertions(+), 21 deletions(-) diff --git a/tools/genomespace/genomespace_exporter.py b/tools/genomespace/genomespace_exporter.py index a5bd30dfa09..ec88fa26af6 100644 --- a/tools/genomespace/genomespace_exporter.py +++ b/tools/genomespace/genomespace_exporter.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python #Dan Blankenberg import base64 @@ -10,17 +11,37 @@ import json import logging import optparse import os +import tempfile import urllib import urllib2 from urlparse import urljoin log = logging.getLogger( "tools.genomespace.genomespace_exporter" )#( __name__ ) +try: + from galaxy import eggs + eggs.require('boto') +except ImportError: + pass + +try: + import boto + from boto.s3.connection import S3Connection +except ImportError: + boto = None + GENOMESPACE_API_VERSION_STRING = "v1.0" GENOMESPACE_SERVER_URL_PROPERTIES = "https://dm.genomespace.org/config/%s/serverurl.properties" % ( GENOMESPACE_API_VERSION_STRING ) CHUNK_SIZE = 2**20 #1mb +# TODO: TARGET_SPLIT_SIZE and TARGET_SIMPLE_PUT_UPLOAD_SIZE are arbitrarily defined +# we should programmatically determine these, based upon the current environment +TARGET_SPLIT_SIZE = 250 * 1024 * 1024 # 250 mb +MIN_MULTIPART_UPLOAD_SIZE = 5 * 1024 * 1024 # 5mb +MAX_SIMPLE_PUT_UPLOAD_SIZE = 5 * 1024 * 1024 * 1024 # 5gb +TARGET_SIMPLE_PUT_UPLOAD_SIZE = MAX_SIMPLE_PUT_UPLOAD_SIZE / 2 + # Some basic Caching, so we don't have to reload and download everything every time, # especially now that we are calling the parameter's get options method 5 times # (6 on reload) when a user loads the tool interface @@ -203,28 +224,69 @@ def send_file_to_genomespace( genomespace_site, username, token, source_filename directory_dict = get_personal_directory( url_opener, dm_url )['directory'] #this is the base for the auto-generated galaxy export directories #what directory to stuff this in target_directory_dict = create_directory( url_opener, directory_dict, target_directory, dm_url ) - #get upload url - upload_url = "uploadurl" content_length = os.path.getsize( source_filename ) input_file = open( source_filename, 'rb' ) - content_md5 = hashlib.md5() - chunk_write( input_file, content_md5, target_method="update" ) - input_file.seek( 0 ) #back to start, for uploading - - upload_params = { 'Content-Length': content_length, 'Content-MD5': base64.standard_b64encode( content_md5.digest() ), 'Content-Type': content_type } - upload_url = "%s/%s/%s%s/%s?%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], urllib.quote( target_filename, safe='' ), urllib.urlencode( upload_params ) ) - new_file_request = urllib2.Request( upload_url )#, headers = { 'Content-Type': 'application/json', 'Accept': 'application/text' } ) #apparently http://www.genomespace.org/team/specs/updated-dm-rest-api:"Every HTTP request to the Data Manager should include the Accept header with a preference for the media types application/json and application/text." is not correct - new_file_request.get_method = lambda: 'GET' - #get url to upload to - target_upload_url = url_opener.open( new_file_request ).read() - #upload file to determined url - upload_headers = dict( upload_params ) - #upload_headers[ 'x-amz-meta-md5-hash' ] = content_md5.hexdigest() - upload_headers[ 'Accept' ] = 'application/json' - upload_file_request = urllib2.Request( target_upload_url, headers = upload_headers, data = input_file ) - upload_file_request.get_method = lambda: 'PUT' - upload_result = urllib2.urlopen( upload_file_request ).read() + if content_length > TARGET_SIMPLE_PUT_UPLOAD_SIZE: + # Determine sizes of each part. + split_count = content_length / TARGET_SPLIT_SIZE + last_size = content_length - ( split_count * TARGET_SPLIT_SIZE ) + sizes = [ TARGET_SPLIT_SIZE ] * split_count + if last_size: + if last_size < MIN_MULTIPART_UPLOAD_SIZE: + if sizes: + sizes[-1] = sizes[-1] + last_size + else: + sizes = [ last_size ] + else: + sizes.append( last_size ) + print "Performing multi-part upload in %i parts." % ( len( sizes ) ) + #get upload url + upload_url = "uploadinfo" + upload_url = "%s/%s/%s%s/%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], urllib.quote( target_filename, safe='' ) ) + upload_request = urllib2.Request( upload_url, headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' } ) + upload_request.get_method = lambda: 'GET' + upload_info = json.loads( url_opener.open( upload_request ).read() ) + conn = S3Connection( aws_access_key_id=upload_info['amazonCredentials']['accessKey'], + aws_secret_access_key=upload_info['amazonCredentials']['secretKey'], + security_token=upload_info['amazonCredentials']['sessionToken'] ) + # Cannot use conn.get_bucket due to permissions, manually create bucket object + bucket = boto.s3.bucket.Bucket( connection=conn, name=upload_info['s3BucketName'] ) + mp = bucket.initiate_multipart_upload( upload_info['s3ObjectKey'] ) + for i,part_size in enumerate( sizes, start=1 ): + fh = tempfile.TemporaryFile( 'wb+' ) + while part_size: + if CHUNK_SIZE > part_size: + read_size = part_size + else: + read_size = CHUNK_SIZE + chunk = input_file.read( read_size ) + fh.write( chunk ) + part_size = part_size - read_size + fh.flush() + fh.seek(0) + mp.upload_part_from_file( fh, i ) + fh.close() + upload_result = mp.complete_upload() + else: + print 'Performing simple put upload.' + upload_url = "uploadurl" + content_md5 = hashlib.md5() + chunk_write( input_file, content_md5, target_method="update" ) + input_file.seek( 0 ) #back to start, for uploading + upload_params = { 'Content-Length': content_length, 'Content-MD5': base64.standard_b64encode( content_md5.digest() ), 'Content-Type': content_type } + upload_url = "%s/%s/%s%s/%s?%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], urllib.quote( target_filename, safe='' ), urllib.urlencode( upload_params ) ) + new_file_request = urllib2.Request( upload_url )#, headers = { 'Content-Type': 'application/json', 'Accept': 'application/text' } ) #apparently http://www.genomespace.org/team/specs/updated-dm-rest-api:"Every HTTP request to the Data Manager should include the Accept header with a preference for the media types application/json and application/text." is not correct + new_file_request.get_method = lambda: 'GET' + #get url to upload to + target_upload_url = url_opener.open( new_file_request ).read() + #upload file to determined url + upload_headers = dict( upload_params ) + #upload_headers[ 'x-amz-meta-md5-hash' ] = content_md5.hexdigest() + upload_headers[ 'Accept' ] = 'application/json' + upload_file_request = urllib2.Request( target_upload_url, headers = upload_headers, data = input_file ) + upload_file_request.get_method = lambda: 'PUT' + upload_result = urllib2.urlopen( upload_file_request ).read() result_url = "%s/%s" % ( target_directory_dict['url'], urllib.quote( target_filename, safe='' ) ) #determine available gs launch apps web_tools = get_genome_space_launch_apps( genomespace_site_dict['atmServer'], url_opener, result_url, file_type ) diff --git a/tools/genomespace/genomespace_exporter.xml b/tools/genomespace/genomespace_exporter.xml index 344698e2baa..1f9561e11dc 100644 --- a/tools/genomespace/genomespace_exporter.xml +++ b/tools/genomespace/genomespace_exporter.xml @@ -1,5 +1,5 @@ - + - send data to GenomeSpace genomespace_exporter.py --genomespace_site "prod" @@ -36,7 +36,7 @@ - + From 5d3485cd7abdfd120833a3b920085e42180827c3 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 21 Apr 2015 11:41:33 -0400 Subject: [PATCH 3/3] Bugfix: Typo in remap dataset handling in tools. --- lib/galaxy/tools/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index f613efdc480..d0dc1c0caed 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2601,7 +2601,7 @@ class Tool( object, Dictifiable ): hdca_source_dict = {} for hdca in history.dataset_collections: key = '%s_%s' % (hdca.hid, hdca.collection.id) - hdca_source_dict[ hda.collection.id ] = hdca_source_dict[ key ] = hdca + hdca_source_dict[ hdca.collection.id ] = hdca_source_dict[ key ] = hdca # Map dataset or collection to current history def map_to_history(value):