diff --git a/tool_conf.xml.sample b/tool_conf.xml.sample
index 3e556682819..7d9abf0f65f 100644
--- a/tool_conf.xml.sample
+++ b/tool_conf.xml.sample
@@ -27,14 +27,14 @@
-
-
+
+
\n' )
+ else:
+ log_file.write( 'There are no GenomeSpace applications available for file type: %s
\n' % ( file_type ) )
+ log_file.write( "\n" )
+ return upload_result
+
+if __name__ == '__main__':
+ #Parse Command Line
+ parser = optparse.OptionParser()
+ parser.add_option( '-s', '--genomespace_site', dest='genomespace_site', action='store', type="string", default=None, help='genomespace_site' )
+ parser.add_option( '-t', '--token', dest='token', action='store', type="string", default=None, help='token' )
+ parser.add_option( '-u', '--username', dest='username', action='store', type="string", default=None, help='username' )
+ parser.add_option( '-d', '--dataset', dest='dataset', action='store', type="string", default=None, help='dataset' )
+ parser.add_option( '-f', '--filename', dest='filename', action='store', type="string", default=None, help='filename' )
+ parser.add_option( '-y', '--subdirectory', dest='subdirectory', action='append', type="string", default=None, help='subdirectory' )
+ parser.add_option( '', '--file_type', dest='file_type', action='store', type="string", default=None, help='file_type' )
+ parser.add_option( '-c', '--content_type', dest='content_type', action='store', type="string", default=None, help='content_type' )
+ parser.add_option( '-l', '--log', dest='log', action='store', type="string", default=None, help='log' )
+
+ (options, args) = parser.parse_args()
+
+ send_file_to_genomespace( options.genomespace_site, options.username, options.token, options.dataset, map( binascii.unhexlify, options.subdirectory ), options.filename, options.file_type, options.content_type, options.log )
+
+
diff --git a/tools/genomespace/genomespace_exporter.xml b/tools/genomespace/genomespace_exporter.xml
new file mode 100644
index 00000000000..eddba6d7f71
--- /dev/null
+++ b/tools/genomespace/genomespace_exporter.xml
@@ -0,0 +1,51 @@
+
+
+ - send data to GenomeSpace
+ genomespace_exporter.py
+ --genomespace_site "prod"
+ #assert $__user_id__ != 'Anonymous', Exception( 'You must be logged in to use this tool.' )
+ #set $user = $__app__.model.User.get( $__user_id__ )
+ #set $username = $user.preferences.get( 'genomespace_username', None )
+ #set $token = $user.preferences.get( 'genomespace_token', None )
+ #assert None not in ( $username, $token ), Exception( 'You must associate a GenomeSpace OpenID with your account and log in with it.' )
+ #import binascii
+ --username "${username}"
+ --token "${token}"
+ --dataset "${input1}"
+ #if $subdirectory:
+ #for $subd in str( $subdirectory ).split( '/' ):
+ #if not $subd:
+ --subdirectory "${ binascii.hexlify( '/' ) }"
+ #else:
+ --subdirectory "${ binascii.hexlify( $subd ) }"
+ #end if
+ #end for
+ #else:
+ --subdirectory "${ binascii.hexlify( 'galaxy_export' ) }"
+ --subdirectory "${ binascii.hexlify( str( $base_url ).split( '://', 1 )[-1] ) }" ##Protocol removed by request
+ #end if
+ #if $filename:
+ --filename "${filename}"
+ #else:
+ --filename "Galaxy History Item ${__app__.security.encode_id( $input1.id )} - ${input1.hid}: ${input1.name}.${input1.ext}"
+ #end if
+ --file_type "${input1.ext}"
+ --content_type "${input1.get_mime()}"
+ --log "${output_log}"
+
+
+
+
+
+
+
+
+
+
+
+
+ This Tool allows you to export data to GenomeSpace. You must have logged in using your GenomeSpace OpenID. You can associate your OpenID credentials under the User Preferences panel.
+
+
+
+
\ No newline at end of file
diff --git a/tools/data_source/genomespace_file_browser.py b/tools/genomespace/genomespace_file_browser.py
similarity index 86%
rename from tools/data_source/genomespace_file_browser.py
rename to tools/genomespace/genomespace_file_browser.py
index 73687c43990..788ab2bf69c 100644
--- a/tools/data_source/genomespace_file_browser.py
+++ b/tools/genomespace/genomespace_file_browser.py
@@ -1,6 +1,6 @@
#Dan Blankenberg
-import optparse, os, urllib2, cookielib
+import optparse, os, urllib, urllib2, urlparse, cookielib
from galaxy import eggs
import pkg_resources
@@ -121,17 +121,29 @@ def download_from_genomespace_file_browser( json_parameter_file, genomespace_sit
filetype_key = "%s%i" % ( file_type_prefix, file_num )
filetype_url = datasource_params.get( filetype_key, None )
galaxy_ext = get_galaxy_ext_from_genomespace_format_url( url_opener, filetype_url )
+ formated_download_url = "%s?%s" % ( download_url, urllib.urlencode( [ ( 'dataformat', filetype_url ) ] ) )
+ new_file_request = urllib2.Request( formated_download_url )
+ new_file_request.get_method = lambda: 'GET'
+ target_download_url = url_opener.open( new_file_request )
+ filename = None
+ if 'Content-Disposition' in target_download_url.info():
+ # If the response has Content-Disposition, try to get filename from it
+ content_disposition = dict( map( lambda x: x.strip().split('=') if '=' in x else ( x.strip(),'' ), target_download_url.info()['Content-Disposition'].split( ';' ) ) )
+ if 'filename' in content_disposition:
+ filename = content_disposition[ 'filename' ].strip( "\"'" )
+ if not filename:
+ parsed_url = urlparse.urlparse( download_url )
+ query_params = urlparse.parse_qs( parsed_url[4] )
+ filename = urllib.unquote_plus( parsed_url[2].split( '/' )[-1] )
if output_filename is None:
output_filename = os.path.join( datasource_params['__new_file_path__'], 'primary_%i_output%i_visible_%s' % ( hda_id, file_num, galaxy_ext ) )
else:
if dataset_id is not None:
metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'dataset',
dataset_id = dataset_id,
- ext = galaxy_ext ) ) )
+ ext = galaxy_ext,
+ name = "GenomeSpace import on %s" % ( filename ) ) ) )
output_file = open( output_filename, 'wb' )
- new_file_request = urllib2.Request( download_url )
- new_file_request.get_method = lambda: 'GET'
- target_download_url = url_opener.open( new_file_request )
chunk_write( target_download_url, output_file )
output_file.close()
output_filename = None #only have one filename available
diff --git a/tools/data_source/genomespace_file_browser_dev.xml b/tools/genomespace/genomespace_file_browser_dev.xml
similarity index 100%
rename from tools/data_source/genomespace_file_browser_dev.xml
rename to tools/genomespace/genomespace_file_browser_dev.xml
diff --git a/tools/data_source/genomespace_file_browser_prod.xml b/tools/genomespace/genomespace_file_browser_prod.xml
similarity index 100%
rename from tools/data_source/genomespace_file_browser_prod.xml
rename to tools/genomespace/genomespace_file_browser_prod.xml
diff --git a/tools/data_source/genomespace_file_browser_test.xml b/tools/genomespace/genomespace_file_browser_test.xml
similarity index 100%
rename from tools/data_source/genomespace_file_browser_test.xml
rename to tools/genomespace/genomespace_file_browser_test.xml
diff --git a/tools/genomespace/genomespace_importer.py b/tools/genomespace/genomespace_importer.py
new file mode 100644
index 00000000000..b2deee2083f
--- /dev/null
+++ b/tools/genomespace/genomespace_importer.py
@@ -0,0 +1,156 @@
+#Dan Blankenberg
+
+import optparse, os, urllib2, urllib, cookielib, urlparse
+
+from galaxy import eggs
+import pkg_resources
+
+pkg_resources.require( "simplejson" )
+import simplejson
+
+GENOMESPACE_API_VERSION_STRING = "v1.0"
+GENOMESPACE_SERVER_URL_PROPERTIES = "http://www.genomespace.org/sites/genomespacefiles/config/serverurl.properties"
+
+CHUNK_SIZE = 2**20 #1mb
+
+DEFAULT_GALAXY_EXT = "data"
+
+#genomespace format identifier is the URL
+GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT = {} #TODO: fix this so it is not a global variable
+#TODO: we should use a better way to set up this mapping
+GENOMESPACE_EXT_TO_GALAXY_EXT = {'rifles': 'rifles',
+ 'lifes': 'lifes',
+ 'cn': 'cn',
+ 'GTF': 'gtf',
+ 'res': 'res',
+ 'xcn': 'xcn',
+ 'lowercasetxt': 'lowercasetxt',
+ 'bed': 'bed',
+ 'CBS': 'cbs',
+ 'genomicatab': 'genomicatab',
+ 'gxp': 'gxp',
+ 'reversedtxt': 'reversedtxt',
+ 'nowhitespace': 'nowhitespace',
+ 'unknown': 'unknown',
+ 'txt': 'txt',
+ 'uppercasetxt': 'uppercasetxt',
+ 'GISTIC': 'gistic',
+ 'GFF': 'gff',
+ 'gmt': 'gmt',
+ 'gct': 'gct'}
+
+VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+
+def chunk_write( source_stream, target_stream, source_method = "read", target_method="write" ):
+ source_method = getattr( source_stream, source_method )
+ target_method = getattr( target_stream, target_method )
+ while True:
+ chunk = source_method( CHUNK_SIZE )
+ if chunk:
+ target_method( chunk )
+ else:
+ break
+
+def get_cookie_opener( gs_username, gs_token ):
+ """ Create a GenomeSpace cookie opener """
+ cj = cookielib.CookieJar()
+ for cookie_name, cookie_value in [ ( 'gs-token', gs_token ), ( 'gs-username', gs_username ) ]:
+ #create a super-cookie, valid for all domains
+ cookie = cookielib.Cookie(version=0, name=cookie_name, value=cookie_value, port=None, port_specified=False, domain='', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False )
+ cj.set_cookie( cookie )
+ cookie_opener = urllib2.build_opener( urllib2.HTTPCookieProcessor( cj ) )
+ return cookie_opener
+
+def get_galaxy_ext_from_genomespace_format_url( url_opener, file_format_url ):
+ ext = GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT.get( file_format_url, None )
+ if ext is not None:
+ ext = GENOMESPACE_EXT_TO_GALAXY_EXT.get( ext, None )
+ if ext is None:
+ #could check content type, etc here
+ ext = DEFAULT_GALAXY_EXT
+ return ext
+
+def get_genomespace_site_urls():
+ genomespace_sites = {}
+ for line in urllib2.urlopen( GENOMESPACE_SERVER_URL_PROPERTIES ).read().split( '\n' ):
+ line = line.rstrip()
+ if not line or line.startswith( "#" ):
+ continue
+ server, line = line.split( '.', 1 )
+ if server not in genomespace_sites:
+ genomespace_sites[server] = {}
+ line = line.split( "=", 1 )
+ genomespace_sites[server][line[0]] = line[1]
+ return genomespace_sites
+
+def set_genomespace_format_identifiers( url_opener, dm_site ):
+ gs_request = urllib2.Request( "%s/%s/dataformat/list" % ( dm_site, GENOMESPACE_API_VERSION_STRING ) )
+ gs_request.get_method = lambda: 'GET'
+ opened_gs_request = url_opener.open( gs_request )
+ genomespace_formats = simplejson.loads( opened_gs_request.read() )
+ for format in genomespace_formats:
+ GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT[ format['url'] ] = format['name']
+
+def download_from_genomespace_importer( username, token, json_parameter_file, genomespace_site ):
+ json_params = simplejson.loads( open( json_parameter_file, 'r' ).read() )
+ datasource_params = json_params.get( 'param_dict' )
+ #username = datasource_params.get( "gs-username", None )
+ #token = datasource_params.get( "gs-token", None )
+ assert None not in [ username, token ], "Missing GenomeSpace username or token."
+ output_filename = datasource_params.get( "output_file1", None )
+ dataset_id = json_params['output_data'][0]['dataset_id']
+ hda_id = json_params['output_data'][0]['hda_id']
+ url_opener = get_cookie_opener( username, token )
+ #load and set genomespace format ids to galaxy exts
+ genomespace_site_dict = get_genomespace_site_urls()[ genomespace_site ]
+ set_genomespace_format_identifiers( url_opener, genomespace_site_dict['dmServer'] )
+ file_url_name = "URL"
+ metadata_parameter_file = open( json_params['job_config']['TOOL_PROVIDED_JOB_METADATA_FILE'], 'wb' )
+ url_param = datasource_params.get( file_url_name, None )
+ for download_url in url_param.split( ',' ):
+ parsed_url = urlparse.urlparse( download_url )
+ query_params = urlparse.parse_qs( parsed_url[4] )
+ file_type = DEFAULT_GALAXY_EXT
+ if 'dataformat' in query_params:
+ file_type = query_params[ 'dataformat' ][0]
+ file_type = get_galaxy_ext_from_genomespace_format_url( url_opener, file_type )
+ elif '.' in parsed_url[2]:
+ file_type = parsed_url[2].rsplit( '.', 1 )[-1]
+ file_type = GENOMESPACE_EXT_TO_GALAXY_EXT.get( file_type, file_type )
+ new_file_request = urllib2.Request( download_url )
+ new_file_request.get_method = lambda: 'GET'
+ target_download_url = url_opener.open( new_file_request )
+ filename = None
+ if 'Content-Disposition' in target_download_url.info():
+ content_disposition = dict( map( lambda x: x.strip().split('=') if '=' in x else ( x.strip(),'' ), target_download_url.info()['Content-Disposition'].split( ';' ) ) )
+ if 'filename' in content_disposition:
+ filename = content_disposition[ 'filename' ].strip( "\"'" )
+ if not filename:
+ parsed_url = urlparse.urlparse( download_url )
+ query_params = urlparse.parse_qs( parsed_url[4] )
+ filename = urllib.unquote_plus( parsed_url[2].split( '/' )[-1] )
+ if output_filename is None:
+ output_filename = os.path.join( datasource_params['__new_file_path__'], 'primary_%i_output%s_visible_%s' % ( hda_id, ''.join( c in VALID_CHARS and c or '-' for c in filename ), file_type ) )
+ else:
+ if dataset_id is not None:
+ metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'dataset',
+ dataset_id = dataset_id,
+ ext = file_type,
+ name = "GenomeSpace importer on %s" % ( filename ) ) ) )
+ output_file = open( output_filename, 'wb' )
+ chunk_write( target_download_url, output_file )
+ output_file.close()
+ output_filename = None #only have one filename available
+ metadata_parameter_file.close()
+ return True
+
+if __name__ == '__main__':
+ #Parse Command Line
+ parser = optparse.OptionParser()
+ parser.add_option( '-p', '--json_parameter_file', dest='json_parameter_file', action='store', type="string", default=None, help='json_parameter_file' )
+ parser.add_option( '-s', '--genomespace_site', dest='genomespace_site', action='store', type="string", default=None, help='genomespace_site' )
+ parser.add_option( '-t', '--token', dest='token', action='store', type="string", default=None, help='token' )
+ parser.add_option( '-u', '--username', dest='username', action='store', type="string", default=None, help='username' )
+ (options, args) = parser.parse_args()
+
+ download_from_genomespace_importer( options.username, options.token, options.json_parameter_file, options.genomespace_site )
diff --git a/tools/genomespace/genomespace_importer.xml b/tools/genomespace/genomespace_importer.xml
new file mode 100644
index 00000000000..db6e126726b
--- /dev/null
+++ b/tools/genomespace/genomespace_importer.xml
@@ -0,0 +1,26 @@
+
+
+ - receive data from GenomeSpace
+ genomespace_importer.py
+ --genomespace_site "prod"
+ #assert $__user_id__ != 'Anonymous', Exception( 'You must be logged in to use this tool.' )
+ #set $user = $__app__.model.User.get( $__user_id__ )
+ #set $username = $user.preferences.get( 'genomespace_username', None )
+ #set $token = $user.preferences.get( 'genomespace_token', None )
+ #assert None not in ( $username, $token ), Exception( 'You must associate a GenomeSpace OpenID with your account and log in with it.' )
+ --username "${username}"
+ --token "${token}"
+ --json_parameter_file "${output_file1}"
+
+
+
+
+
+
+
+
+
+ some help text here...
+
+
+