opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #122 from blankenberg/genomespace-export-5g-to-stable

Browse Source 
[STABLE] Genomespace export 5g
This commit is contained in:
John Chilton
2015-04-16 15:31:35 -05:00
parent 1b922ea0e9 153bef1c52
commit 17856e1855
2 changed files with 118 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/genomespace/genomespace_exporter.py
+116 -28
View File
@@ -1,25 +1,54 @@
#!/usr/bin/env python
#Dan Blankenberg
import base64
import binascii
import cgi
import cookielib
import datetime
import hashlib
import json
import logging
import optparse
import os
import tempfile
import urllib
import urllib2
from urlparse import urljoin
log = logging.getLogger( "tools.genomespace.genomespace_exporter" )#( __name__ )
try:
from galaxy import eggs
eggs.require('boto')
except ImportError:
pass
try:
import boto
from boto.s3.connection import S3Connection
except ImportError:
boto = None
GENOMESPACE_API_VERSION_STRING = "v1.0"
GENOMESPACE_SERVER_URL_PROPERTIES = "https://dm.genomespace.org/config/%s/serverurl.properties" % ( GENOMESPACE_API_VERSION_STRING )
CHUNK_SIZE = 2**20 #1mb
# TODO: TARGET_SPLIT_SIZE and TARGET_SIMPLE_PUT_UPLOAD_SIZE are arbitrarily defined
# we should programmatically determine these, based upon the current environment
TARGET_SPLIT_SIZE = 250 * 1024 * 1024 # 250 mb
MIN_MULTIPART_UPLOAD_SIZE = 5 * 1024 * 1024 # 5mb
MAX_SIMPLE_PUT_UPLOAD_SIZE = 5 * 1024 * 1024 * 1024 # 5gb
TARGET_SIMPLE_PUT_UPLOAD_SIZE = MAX_SIMPLE_PUT_UPLOAD_SIZE / 2
# Some basic Caching, so we don't have to reload and download everything every time,
# especially now that we are calling the parameter's get options method 5 times
# (6 on reload) when a user loads the tool interface
# For now, we'll use 30 seconds as the cache valid time
CACHE_TIME = datetime.timedelta( seconds=30 )
GENOMESPACE_DIRECTORIES_BY_USER = {}
def chunk_write( source_stream, target_stream, source_method = "read", target_method="write" ):
source_method = getattr( source_stream, source_method )
@@ -150,14 +179,32 @@ def galaxy_code_get_genomespace_folders( genomespace_site='prod', trans=None, va
username = trans.user.preferences.get( 'genomespace_username', None )
token = trans.user.preferences.get( 'genomespace_token', None )
if None not in ( username, token ):
url_opener = get_cookie_opener( username, token )
genomespace_site_dict = get_genomespace_site_urls()[ genomespace_site ]
dm_url = genomespace_site_dict['dmServer']
#get export root directory
#directory_dict = get_default_directory( url_opener, dm_url ).get( 'directory', None ) #This directory contains shares and other items outside of the users home
directory_dict = get_personal_directory( url_opener, dm_url ).get( 'directory', None ) #Limit export list to only user's home dir
if directory_dict is not None:
recurse_directory_dict( url_opener, rval, directory_dict.get( 'url' ) )
# NB: it is possible, but unlikely for a user to swap GenomeSpace accounts around
# in the middle of interacting with tools, so we'll have several layers of caching by ids/values
if trans.user in GENOMESPACE_DIRECTORIES_BY_USER:
if username in GENOMESPACE_DIRECTORIES_BY_USER[ trans.user ]:
if token in GENOMESPACE_DIRECTORIES_BY_USER[ trans.user ][ username ]:
cache_dict = GENOMESPACE_DIRECTORIES_BY_USER[ trans.user ][ username ][ token ]
if datetime.datetime.now() - cache_dict.get( 'time_loaded' ) > CACHE_TIME:
# cache too old, need to reload, we'll just kill the whole trans.user
del GENOMESPACE_DIRECTORIES_BY_USER[ trans.user ]
else:
rval = cache_dict.get( 'rval' )
else:
del GENOMESPACE_DIRECTORIES_BY_USER[ trans.user ]
else:
del GENOMESPACE_DIRECTORIES_BY_USER[ trans.user ]
if not rval:
url_opener = get_cookie_opener( username, token )
genomespace_site_dict = get_genomespace_site_urls()[ genomespace_site ]
dm_url = genomespace_site_dict['dmServer']
#get export root directory
#directory_dict = get_default_directory( url_opener, dm_url ).get( 'directory', None ) #This directory contains shares and other items outside of the users home
directory_dict = get_personal_directory( url_opener, dm_url ).get( 'directory', None ) #Limit export list to only user's home dir
if directory_dict is not None:
recurse_directory_dict( url_opener, rval, directory_dict.get( 'url' ) )
# Save the cache
GENOMESPACE_DIRECTORIES_BY_USER[ trans.user ] = { username: { token: { 'time_loaded': datetime.datetime.now(), 'rval': rval } } }
if not rval:
if not base_url:
base_url = '..'
@@ -177,28 +224,69 @@ def send_file_to_genomespace( genomespace_site, username, token, source_filename
directory_dict = get_personal_directory( url_opener, dm_url )['directory'] #this is the base for the auto-generated galaxy export directories
#what directory to stuff this in
target_directory_dict = create_directory( url_opener, directory_dict, target_directory, dm_url )
#get upload url
upload_url = "uploadurl"
content_length = os.path.getsize( source_filename )
input_file = open( source_filename )
content_md5 = hashlib.md5()
chunk_write( input_file, content_md5, target_method="update" )
input_file.seek( 0 ) #back to start, for uploading
upload_params = { 'Content-Length': content_length, 'Content-MD5': base64.standard_b64encode( content_md5.digest() ), 'Content-Type': content_type }
upload_url = "%s/%s/%s%s/%s?%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], urllib.quote( target_filename, safe='' ), urllib.urlencode( upload_params ) )
new_file_request = urllib2.Request( upload_url )#, headers = { 'Content-Type': 'application/json', 'Accept': 'application/text' } ) #apparently http://www.genomespace.org/team/specs/updated-dm-rest-api:"Every HTTP request to the Data Manager should include the Accept header with a preference for the media types application/json and application/text." is not correct
new_file_request.get_method = lambda: 'GET'
#get url to upload to
target_upload_url = url_opener.open( new_file_request ).read()
#upload file to determined url
upload_headers = dict( upload_params )
#upload_headers[ 'x-amz-meta-md5-hash' ] = content_md5.hexdigest()
upload_headers[ 'Accept' ] = 'application/json'
upload_file_request = urllib2.Request( target_upload_url, headers = upload_headers, data = input_file )
upload_file_request.get_method = lambda: 'PUT'
upload_result = urllib2.urlopen( upload_file_request ).read()
input_file = open( source_filename, 'rb' )
if content_length > TARGET_SIMPLE_PUT_UPLOAD_SIZE:
# Determine sizes of each part.
split_count = content_length / TARGET_SPLIT_SIZE
last_size = content_length - ( split_count * TARGET_SPLIT_SIZE )
sizes = [ TARGET_SPLIT_SIZE ] * split_count
if last_size:
if last_size < MIN_MULTIPART_UPLOAD_SIZE:
if sizes:
sizes[-1] = sizes[-1] + last_size
else:
sizes = [ last_size ]
else:
sizes.append( last_size )
print "Performing multi-part upload in %i parts." % ( len( sizes ) )
#get upload url
upload_url = "uploadinfo"
upload_url = "%s/%s/%s%s/%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], urllib.quote( target_filename, safe='' ) )
upload_request = urllib2.Request( upload_url, headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' } )
upload_request.get_method = lambda: 'GET'
upload_info = json.loads( url_opener.open( upload_request ).read() )
conn = S3Connection( aws_access_key_id=upload_info['amazonCredentials']['accessKey'],
aws_secret_access_key=upload_info['amazonCredentials']['secretKey'],
security_token=upload_info['amazonCredentials']['sessionToken'] )
# Cannot use conn.get_bucket due to permissions, manually create bucket object
bucket = boto.s3.bucket.Bucket( connection=conn, name=upload_info['s3BucketName'] )
mp = bucket.initiate_multipart_upload( upload_info['s3ObjectKey'] )
for i,part_size in enumerate( sizes, start=1 ):
fh = tempfile.TemporaryFile( 'wb+' )
while part_size:
if CHUNK_SIZE > part_size:
read_size = part_size
else:
read_size = CHUNK_SIZE
chunk = input_file.read( read_size )
fh.write( chunk )
part_size = part_size - read_size
fh.flush()
fh.seek(0)
mp.upload_part_from_file( fh, i )
fh.close()
upload_result = mp.complete_upload()
else:
print 'Performing simple put upload.'
upload_url = "uploadurl"
content_md5 = hashlib.md5()
chunk_write( input_file, content_md5, target_method="update" )
input_file.seek( 0 ) #back to start, for uploading
upload_params = { 'Content-Length': content_length, 'Content-MD5': base64.standard_b64encode( content_md5.digest() ), 'Content-Type': content_type }
upload_url = "%s/%s/%s%s/%s?%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], urllib.quote( target_filename, safe='' ), urllib.urlencode( upload_params ) )
new_file_request = urllib2.Request( upload_url )#, headers = { 'Content-Type': 'application/json', 'Accept': 'application/text' } ) #apparently http://www.genomespace.org/team/specs/updated-dm-rest-api:"Every HTTP request to the Data Manager should include the Accept header with a preference for the media types application/json and application/text." is not correct
new_file_request.get_method = lambda: 'GET'
#get url to upload to
target_upload_url = url_opener.open( new_file_request ).read()
#upload file to determined url
upload_headers = dict( upload_params )
#upload_headers[ 'x-amz-meta-md5-hash' ] = content_md5.hexdigest()
upload_headers[ 'Accept' ] = 'application/json'
upload_file_request = urllib2.Request( target_upload_url, headers = upload_headers, data = input_file )
upload_file_request.get_method = lambda: 'PUT'
upload_result = urllib2.urlopen( upload_file_request ).read()
result_url = "%s/%s" % ( target_directory_dict['url'], urllib.quote( target_filename, safe='' ) )
#determine available gs launch apps
web_tools = get_genome_space_launch_apps( genomespace_site_dict['atmServer'], url_opener, result_url, file_type )
tools/genomespace/genomespace_exporter.xml
+2 -2
View File
@@ -1,5 +1,5 @@
<?xml version="1.0"?>
<tool name="GenomeSpace Exporter" id="genomespace_exporter" require_login="True" version="0.0.2">
<tool name="GenomeSpace Exporter" id="genomespace_exporter" require_login="True" version="0.0.3">
<description> - send data to GenomeSpace</description>
<command interpreter="python">genomespace_exporter.py
--genomespace_site "prod"
@@ -36,7 +36,7 @@
<param format="data" name="input1" type="data" label="Send this dataset to GenomeSpace" />
<param name="base_url" type="baseurl" />
<param name="subdirectory" type="drill_down" display="radio" hierarchy="exact" multiple="False" optional="True" label="Choose Target Directory" dynamic_options="galaxy_code_get_genomespace_folders( genomespace_site = 'prod', trans=__trans__, value=__value__, input_dataset=input1, base_url=base_url )" help="Leave blank to generate automatically"/>
<param name="filename" type="text" size="80" help="Leave blank to generate automatically" />
<param name="filename" type="text" size="80" label="Filename" help="Leave blank to generate automatically" />
</inputs>
<outputs>
<data format="html" name="output_log" />