From b2e2b801ea0785821eaa265762d33b1bcc5a0658 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Mon, 3 Aug 2009 12:02:54 -0400 Subject: [PATCH] Enhance data_source.py to take advantage of the content length when known. If content length provided by external application is greater than config.output_size_limit, no data will be retrieved and an error message will be provided to the user. Tools using this script have been updated to provide the max file size on the command line. Resolves ticket #93. --- lib/galaxy/jobs/__init__.py | 2 +- tools/data_source/biomart.xml | 2 +- tools/data_source/biomart_test.xml | 2 +- tools/data_source/data_source.py | 8 ++++++++ tools/data_source/epigraph_import.xml | 2 +- tools/data_source/epigraph_import_test.xml | 2 +- tools/data_source/eupathdb.xml | 2 +- tools/data_source/flymine.xml | 2 +- tools/data_source/flymine_test.xml | 2 +- tools/data_source/gramene_mart.xml | 2 +- tools/data_source/ucsc_tablebrowser.xml | 2 +- tools/data_source/ucsc_tablebrowser_archaea.xml | 2 +- tools/data_source/ucsc_tablebrowser_test.xml | 2 +- tools/data_source/wormbase.xml | 2 +- tools/data_source/wormbase_test.xml | 2 +- 15 files changed, 22 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 3b253eb6394..ddec77802be 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -595,7 +595,7 @@ class JobWrapper( object ): self.real_path = real_path self.false_path = false_path def __str__( self ): - if false_path is None: + if self.false_path is None: return self.real_path else: return self.false_path diff --git a/tools/data_source/biomart.xml b/tools/data_source/biomart.xml index edfa563ebca..4f7e0303c67 100644 --- a/tools/data_source/biomart.xml +++ b/tools/data_source/biomart.xml @@ -9,7 +9,7 @@ --> Central server - data_source.py $output + data_source.py $output $__app__.config.output_size_limit go to BioMart Central $GALAXY_URL diff --git a/tools/data_source/biomart_test.xml b/tools/data_source/biomart_test.xml index 0f4bbdc3517..8893885ea06 100644 --- a/tools/data_source/biomart_test.xml +++ b/tools/data_source/biomart_test.xml @@ -9,7 +9,7 @@ --> Test server - data_source.py $output + data_source.py $output $__app__.config.output_size_limit go to BioMart Central $GALAXY_URL diff --git a/tools/data_source/data_source.py b/tools/data_source/data_source.py index 15b089c8452..f48754d2cc5 100644 --- a/tools/data_source/data_source.py +++ b/tools/data_source/data_source.py @@ -21,6 +21,10 @@ def check_gzip( filename ): def __main__(): filename = sys.argv[1] + try: + max_file_size = int( sys.argv[2] ) + except: + max_file_size = 0 params = {} for line in open( filename, 'r' ): try: @@ -50,6 +54,10 @@ def __main__(): page = urllib.urlopen( URL, urllib.urlencode( params ) ) except Exception, e: stop_err( 'The remote data source application may be off line, please try again later. Error: %s' % str( e ) ) + if max_file_size: + file_size = int( page.info().get( 'Content-Length', 0 ) ) + if file_size > max_file_size: + stop_err( 'The size of the data (%d bytes) you have requested exceeds the maximum allowed (%d bytes) on this server.' % ( file_size, max_file_size ) ) out = open( filename, 'w' ) while 1: chunk = page.read( CHUNK_SIZE ) diff --git a/tools/data_source/epigraph_import.xml b/tools/data_source/epigraph_import.xml index abd0e69847e..ec6c489efa9 100644 --- a/tools/data_source/epigraph_import.xml +++ b/tools/data_source/epigraph_import.xml @@ -6,7 +6,7 @@ --> server - data_source.py $output + data_source.py $output $__app__.config.output_size_limit go to EpiGRAPH server $GALAXY_URL diff --git a/tools/data_source/epigraph_import_test.xml b/tools/data_source/epigraph_import_test.xml index 2d73e1930b3..d3ec24776ee 100644 --- a/tools/data_source/epigraph_import_test.xml +++ b/tools/data_source/epigraph_import_test.xml @@ -6,7 +6,7 @@ --> test server - data_source.py $output + data_source.py $output $__app__.config.output_size_limit go to EpiGRAPH server $GALAXY_URL diff --git a/tools/data_source/eupathdb.xml b/tools/data_source/eupathdb.xml index 17440b3ae3c..d371a4715fa 100644 --- a/tools/data_source/eupathdb.xml +++ b/tools/data_source/eupathdb.xml @@ -5,7 +5,7 @@ --> server - data_source.py $output + data_source.py $output $__app__.config.output_size_limit go to EuPathDB server $GALAXY_URL diff --git a/tools/data_source/flymine.xml b/tools/data_source/flymine.xml index 035f38bf00c..e7d734e9115 100644 --- a/tools/data_source/flymine.xml +++ b/tools/data_source/flymine.xml @@ -6,7 +6,7 @@ --> server - data_source.py $output + data_source.py $output $__app__.config.output_size_limit go to Flymine server $GALAXY_URL diff --git a/tools/data_source/flymine_test.xml b/tools/data_source/flymine_test.xml index 62c7b923a0e..0b6b45791f0 100644 --- a/tools/data_source/flymine_test.xml +++ b/tools/data_source/flymine_test.xml @@ -6,7 +6,7 @@ --> server - data_source.py $output + data_source.py $output $__app__.config.output_size_limit go to Flymine server $GALAXY_URL diff --git a/tools/data_source/gramene_mart.xml b/tools/data_source/gramene_mart.xml index 3eed0531048..0f97a5f259b 100644 --- a/tools/data_source/gramene_mart.xml +++ b/tools/data_source/gramene_mart.xml @@ -9,7 +9,7 @@ --> Central server - data_source.py $output + data_source.py $output $__app__.config.output_size_limit go to GrameneMart Central $GALAXY_URL diff --git a/tools/data_source/ucsc_tablebrowser.xml b/tools/data_source/ucsc_tablebrowser.xml index 250dae000d1..79244951834 100644 --- a/tools/data_source/ucsc_tablebrowser.xml +++ b/tools/data_source/ucsc_tablebrowser.xml @@ -6,7 +6,7 @@ --> table browser - data_source.py $output + data_source.py $output $__app__.config.output_size_limit go to UCSC Table Browser $GALAXY_URL diff --git a/tools/data_source/ucsc_tablebrowser_archaea.xml b/tools/data_source/ucsc_tablebrowser_archaea.xml index 0b6342a957d..cf09e18f0c7 100644 --- a/tools/data_source/ucsc_tablebrowser_archaea.xml +++ b/tools/data_source/ucsc_tablebrowser_archaea.xml @@ -6,7 +6,7 @@ --> table browser - data_source.py $output + data_source.py $output $__app__.config.output_size_limit go to UCSC Table Browser $GALAXY_URL diff --git a/tools/data_source/ucsc_tablebrowser_test.xml b/tools/data_source/ucsc_tablebrowser_test.xml index d09862dd715..67b42ee7b2f 100644 --- a/tools/data_source/ucsc_tablebrowser_test.xml +++ b/tools/data_source/ucsc_tablebrowser_test.xml @@ -6,7 +6,7 @@ --> table browser - data_source.py $output + data_source.py $output $__app__.config.output_size_limit go to UCSC Table Browser $GALAXY_URL diff --git a/tools/data_source/wormbase.xml b/tools/data_source/wormbase.xml index ef7d55b70bb..0005039aeb4 100644 --- a/tools/data_source/wormbase.xml +++ b/tools/data_source/wormbase.xml @@ -1,7 +1,7 @@ server - data_source.py $output + data_source.py $output $__app__.config.output_size_limit go to Wormbase server $GALAXY_URL diff --git a/tools/data_source/wormbase_test.xml b/tools/data_source/wormbase_test.xml index 32b32519f43..44bf34cc241 100644 --- a/tools/data_source/wormbase_test.xml +++ b/tools/data_source/wormbase_test.xml @@ -1,7 +1,7 @@ test server - data_source.py $output + data_source.py $output $__app__.config.output_size_limit go to Wormbase test server $GALAXY_URL