From 9d85ca9e23e8a793c0532e35f93c9dd093fa209f Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 9 Sep 2014 10:06:35 -0400 Subject: [PATCH] Switch to_json_string/from_json_string in galaxy/tools. --- tools/data_source/data_source.py | 24 ++++++++++---------- tools/data_source/upload.py | 8 +++---- tools/filters/gff/gff_filter_by_attribute.py | 22 +++++++++--------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tools/data_source/data_source.py b/tools/data_source/data_source.py index df1137bad05..929e9b1f7d6 100644 --- a/tools/data_source/data_source.py +++ b/tools/data_source/data_source.py @@ -3,7 +3,7 @@ # Data source application parameters are temporarily stored in the dataset file. import socket, urllib, sys, os from galaxy import eggs #eggs needs to be imported so that galaxy.util can find docutils egg... -from galaxy.util.json import from_json_string, to_json_string +from galaxy.util.json import loads, dumps from galaxy.util import get_charset_from_http_headers import galaxy.model # need to import model before sniff to resolve a circular import dependency from galaxy.datatypes import sniff @@ -23,7 +23,7 @@ GALAXY_DATATYPES_CONF_FILE = os.path.join( GALAXY_ROOT_DIR, 'datatypes_conf.xml' def load_input_parameters( filename, erase_file = True ): datasource_params = {} try: - json_params = from_json_string( open( filename, 'r' ).read() ) + json_params = loads( open( filename, 'r' ).read() ) datasource_params = json_params.get( 'param_dict' ) except: json_params = None @@ -44,7 +44,7 @@ def __main__(): max_file_size = int( sys.argv[2] ) except: max_file_size = 0 - + job_params, params = load_input_parameters( filename ) if job_params is None: #using an older tabular file enhanced_handling = False @@ -57,13 +57,13 @@ def __main__(): else: enhanced_handling = True json_file = open( job_params[ 'job_config' ][ 'TOOL_PROVIDED_JOB_METADATA_FILE' ], 'w' ) #specially named file for output junk to pass onto set metadata - + datatypes_registry = Registry() datatypes_registry.load_datatypes( root_dir = job_params[ 'job_config' ][ 'GALAXY_ROOT_DIR' ], config = job_params[ 'job_config' ][ 'GALAXY_DATATYPES_CONF_FILE' ] ) - + URL = params.get( 'URL', None ) #using exactly URL indicates that only one dataset is being downloaded URL_method = params.get( 'URL_method', None ) - + # The Python support for fetching resources from the web is layered. urllib uses the httplib # library, which in turn uses the socket library. As of Python 2.3 you can specify how long # a socket should wait for a response before timing out. By default the socket module has no @@ -71,14 +71,14 @@ def __main__(): # levels. However, you can set the default timeout ( in seconds ) globally for all sockets by # doing the following. socket.setdefaulttimeout( 600 ) - + for data_dict in job_params[ 'output_data' ]: cur_filename = data_dict.get( 'file_name', filename ) cur_URL = params.get( '%s|%s|URL' % ( GALAXY_PARAM_PREFIX, data_dict[ 'out_data_name' ] ), URL ) if not cur_URL: open( cur_filename, 'w' ).write( "" ) stop_err( 'The remote data source application has not sent back a URL parameter in the request.' ) - + # The following calls to urllib.urlopen() will use the above default timeout try: if not URL_method or URL_method == 'get': @@ -96,7 +96,7 @@ def __main__(): cur_filename, is_multi_byte = sniff.stream_to_open_named_file( page, os.open( cur_filename, os.O_WRONLY | os.O_CREAT ), cur_filename, source_encoding=get_charset_from_http_headers( page.headers ) ) except Exception, e: stop_err( 'Unable to fetch %s:\n%s' % ( cur_URL, e ) ) - + #here import checks that upload tool performs if enhanced_handling: try: @@ -106,7 +106,7 @@ def __main__(): info = dict( type = 'dataset', dataset_id = data_dict[ 'dataset_id' ], ext = ext) - - json_file.write( "%s\n" % to_json_string( info ) ) - + + json_file.write( "%s\n" % dumps( info ) ) + if __name__ == "__main__": __main__() diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index f3911191288..42fd67f5fc1 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -36,7 +36,7 @@ def stop_err( msg, ret=1 ): sys.stderr.write( msg ) sys.exit( ret ) def file_err( msg, dataset, json_file ): - json_file.write( to_json_string( dict( type = 'dataset', + json_file.write( dumps( dict( type = 'dataset', ext = 'data', dataset_id = dataset.dataset_id, stderr = msg ) ) + "\n" ) @@ -319,7 +319,7 @@ def add_file( dataset, registry, json_file, output_path ): line_count = line_count ) if dataset.get('uuid', None) is not None: info['uuid'] = dataset.get('uuid') - json_file.write( to_json_string( info ) + "\n" ) + json_file.write( dumps( info ) + "\n" ) if link_data_only == 'copy_files' and datatype.dataset_content_needs_grooming( output_path ): # Groom the dataset content if necessary @@ -358,7 +358,7 @@ def add_composite_file( dataset, registry, json_file, output_path, files_path ): info = dict( type = 'dataset', dataset_id = dataset.dataset_id, stdout = 'uploaded %s file' % dataset.file_type ) - json_file.write( to_json_string( info ) + "\n" ) + json_file.write( dumps( info ) + "\n" ) def output_adjacent_tmpdir( output_path ): @@ -382,7 +382,7 @@ def __main__(): registry.load_datatypes( root_dir=sys.argv[1], config=sys.argv[2] ) for line in open( sys.argv[3], 'r' ): - dataset = from_json_string( line ) + dataset = loads( line ) dataset = util.bunch.Bunch( **safe_dict( dataset ) ) try: output_path = output_paths[int( dataset.dataset_id )][0] diff --git a/tools/filters/gff/gff_filter_by_attribute.py b/tools/filters/gff/gff_filter_by_attribute.py index c84344a3779..6b8a083ae10 100644 --- a/tools/filters/gff/gff_filter_by_attribute.py +++ b/tools/filters/gff/gff_filter_by_attribute.py @@ -1,13 +1,13 @@ #!/usr/bin/env python # This tool takes a gff file as input and creates filters on attributes based on certain properties. # The tool will skip over invalid lines within the file, informing the user about the number of lines skipped. -# TODO: much of this code is copied from the Filter1 tool (filtering.py in tools/stats/). The commonalities should be +# TODO: much of this code is copied from the Filter1 tool (filtering.py in tools/stats/). The commonalities should be # abstracted and leveraged in each filtering tool. from __future__ import division import sys from galaxy import eggs -from galaxy.util.json import to_json_string, from_json_string +from galaxy.util.json import dumps, loads # Older py compatibility try: @@ -44,7 +44,7 @@ def check_for_executable( text, description='' ): except: if operand in secured: stop_err( "Illegal value '%s' in %s '%s'" % ( operand, description, text ) ) - + # # Process inputs. # @@ -52,13 +52,13 @@ def check_for_executable( text, description='' ): in_fname = sys.argv[1] out_fname = sys.argv[2] cond_text = sys.argv[3] -attribute_types = from_json_string( sys.argv[4] ) +attribute_types = loads( sys.argv[4] ) # Convert types from str to type objects. for name, a_type in attribute_types.items(): check_for_executable(a_type) attribute_types[ name ] = eval( a_type ) - + # Unescape if input has been escaped mapped_str = { '__lt__': '<', @@ -72,23 +72,23 @@ mapped_str = { } for key, value in mapped_str.items(): cond_text = cond_text.replace( key, value ) - + # Attempt to determine if the condition includes executable stuff and, if so, exit. check_for_executable( cond_text, 'condition') -# Prepare the column variable names and wrappers for column data types. Only +# Prepare the column variable names and wrappers for column data types. Only # prepare columns up to largest column in condition. attrs, type_casts = [], [] for name, attr_type in attribute_types.items(): attrs.append( name ) type_cast = "get_value('%(name)s', attribute_types['%(name)s'], attribute_values)" % ( {'name': name} ) type_casts.append( type_cast ) - + attr_str = ', '.join( attrs ) # 'c1, c2, c3, c4' type_cast_str = ', '.join( type_casts ) # 'str(c1), int(c2), int(c3), str(c4)' wrap = "%s = %s" % ( attr_str, type_cast_str ) - -# Stats + +# Stats skipped_lines = 0 first_invalid_line = 0 invalid_line = None @@ -102,7 +102,7 @@ def get_value(name, a_type, values_dict): return (a_type)(values_dict[ name ]) else: return None - + # Read and filter input file, skipping invalid lines code = ''' for i, line in enumerate( file( in_fname ) ):