diff --git a/eggs.ini b/eggs.ini index 1c0491984b7..4470e0c103b 100644 --- a/eggs.ini +++ b/eggs.ini @@ -26,7 +26,6 @@ pycrypto = 2.5 pysam = 0.4.2 pysqlite = 2.5.6 python_lzo = 1.08_2.03_static -simplejson = 2.1.1 threadframe = 0.2 guppy = 0.1.8 SQLAlchemy = 0.7.9 diff --git a/lib/galaxy/datatypes/metadata.py b/lib/galaxy/datatypes/metadata.py index 2bf4de1fcd8..54efe4d2949 100644 --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -2,15 +2,13 @@ Galaxy Metadata """ -from galaxy import eggs -eggs.require("simplejson") import copy import cPickle +import json import logging import os import shutil -import simplejson import sys import tempfile import weakref @@ -130,7 +128,7 @@ class MetadataCollection( object ): def from_JSON_dict( self, filename ): dataset = self.parent log.debug( 'loading metadata from file for: %s %s' % ( dataset.__class__.__name__, dataset.id ) ) - JSONified_dict = simplejson.load( open( filename ) ) + JSONified_dict = json.load( open( filename ) ) for name, spec in self.spec.items(): if name in JSONified_dict: dataset._metadata[ name ] = spec.param.from_external_value( JSONified_dict[ name ], dataset ) @@ -146,7 +144,7 @@ class MetadataCollection( object ): for name, spec in self.spec.items(): if name in dataset_meta_dict: meta_dict[ name ] = spec.param.to_external_value( dataset_meta_dict[ name ] ) - simplejson.dump( meta_dict, open( filename, 'wb+' ) ) + json.dump( meta_dict, open( filename, 'wb+' ) ) def __getstate__( self ): return None #cannot pickle a weakref item (self._parent), when data._metadata_collection is None, it will be recreated on demand @@ -456,7 +454,7 @@ class ListParameter( MetadataParameter ): class DictParameter( MetadataParameter ): def to_string( self, value ): - return simplejson.dumps( value ) + return json.dumps( value ) class PythonObjectParameter( MetadataParameter ): @@ -594,7 +592,7 @@ class MetadataTempFile( object ): @classmethod def cleanup_from_JSON_dict_filename( cls, filename ): try: - for key, value in simplejson.load( open( filename ) ).items(): + for key, value in json.load( open( filename ) ).items(): if cls.is_JSONified_value( value ): value = cls.from_JSON( value ) if isinstance( value, cls ) and os.path.exists( value.file_name ): @@ -686,10 +684,10 @@ class JobExternalOutputMetadataWrapper( object ): #file to store a 'return code' indicating the results of the set_meta() call #results code is like (True/False - if setting metadata was successful/failed , exception or string of reason of success/failure ) metadata_files.filename_results_code = abspath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_results_%s_" % key ).name ) - simplejson.dump( ( False, 'External set_meta() not called' ), open( metadata_files.filename_results_code, 'wb+' ) ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) + json.dump( ( False, 'External set_meta() not called' ), open( metadata_files.filename_results_code, 'wb+' ) ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) #file to store kwds passed to set_meta() metadata_files.filename_kwds = abspath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_kwds_%s_" % key ).name ) - simplejson.dump( kwds, open( metadata_files.filename_kwds, 'wb+' ), ensure_ascii=True ) + json.dump( kwds, open( metadata_files.filename_kwds, 'wb+' ), ensure_ascii=True ) #existing metadata file parameters need to be overridden with cluster-writable file locations metadata_files.filename_override_metadata = abspath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_override_%s_" % key ).name ) open( metadata_files.filename_override_metadata, 'wb+' ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) @@ -699,7 +697,7 @@ class JobExternalOutputMetadataWrapper( object ): metadata_temp = MetadataTempFile() shutil.copy( dataset.metadata.get( meta_key, None ).file_name, metadata_temp.file_name ) override_metadata.append( ( meta_key, metadata_temp.to_JSON() ) ) - simplejson.dump( override_metadata, open( metadata_files.filename_override_metadata, 'wb+' ) ) + json.dump( override_metadata, open( metadata_files.filename_override_metadata, 'wb+' ) ) #add to session and flush sa_session.add( metadata_files ) sa_session.flush() @@ -711,7 +709,7 @@ class JobExternalOutputMetadataWrapper( object ): metadata_files = self.get_output_filenames_by_dataset( dataset, sa_session ) if not metadata_files: return False # this file doesn't exist - rval, rstring = simplejson.load( open( metadata_files.filename_results_code ) ) + rval, rstring = json.load( open( metadata_files.filename_results_code ) ) if not rval: log.debug( 'setting metadata externally failed for %s %s: %s' % ( dataset.__class__.__name__, dataset.id, rstring ) ) return rval diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index f63593a7915..87e3cf0b82a 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -2,12 +2,14 @@ Sequence classes """ -import data +from . import data import gzip +import json import logging import os import re import string + from cgi import escape from galaxy import eggs, util @@ -16,8 +18,6 @@ from galaxy.datatypes.checkers import is_gzip from galaxy.datatypes.sniff import get_test_fname, get_headers from galaxy.datatypes.metadata import MetadataElement -eggs.require("simplejson") -import simplejson try: eggs.require( "bx-python" ) @@ -44,8 +44,8 @@ class SequenceSplitLocations( data.Text ): def set_peek( self, dataset, is_multi_byte=False ): if not dataset.dataset.purged: try: - parsed_data = simplejson.load(open(dataset.file_name)) - # dataset.peek = simplejson.dumps(data, sort_keys=True, indent=4) + parsed_data = json.load(open(dataset.file_name)) + # dataset.peek = json.dumps(data, sort_keys=True, indent=4) dataset.peek = data.get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte ) dataset.blurb = '%d sections' % len(parsed_data['sections']) except Exception, e: @@ -60,7 +60,7 @@ class SequenceSplitLocations( data.Text ): def sniff( self, filename ): if os.path.getsize(filename) < 50000: try: - data = simplejson.load(open(filename)) + data = json.load(open(filename)) sections = data['sections'] for section in sections: if 'start' not in section or 'end' not in section or 'sequences' not in section: @@ -155,7 +155,7 @@ class Sequence( data.Text ): do_slow_split = classmethod(do_slow_split) def do_fast_split( cls, input_datasets, toc_file_datasets, subdir_generator_function, split_params): - data = simplejson.load(open(toc_file_datasets[0].file_name)) + data = json.load(open(toc_file_datasets[0].file_name)) sections = data['sections'] total_sequences = long(0) for section in sections: @@ -191,7 +191,7 @@ class Sequence( data.Text ): toc = toc_file_datasets[ds_no] split_data['args']['toc_file'] = toc.file_name f = open(os.path.join(dir, 'split_info_%s.json' % base_name), 'w') - simplejson.dump(split_data, f) + json.dump(split_data, f) f.close() start_sequence += sequences_per_file[part_no] return directories @@ -557,7 +557,7 @@ class Fastq ( Sequence ): sequence_count = long(args['num_sequences']) if 'toc_file' in args: - toc_file = simplejson.load(open(args['toc_file'], 'r')) + toc_file = json.load(open(args['toc_file'], 'r')) commands = Sequence.get_split_commands_with_toc(input_name, output_name, toc_file, start_sequence, sequence_count) else: commands = Sequence.get_split_commands_sequential(is_gzip(input_name), input_name, output_name, start_sequence, sequence_count) diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index 465d5e9f6ad..5e111ca4710 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -2,13 +2,13 @@ Job control via the DRMAA API. """ +import json +import logging import os +import string +import subprocess import sys import time -import string -import logging -import subprocess -import simplejson as json from galaxy import eggs from galaxy import model diff --git a/lib/galaxy/jobs/runners/lwr_client/action_mapper.py b/lib/galaxy/jobs/runners/lwr_client/action_mapper.py index ce09207b38a..3903f515610 100644 --- a/lib/galaxy/jobs/runners/lwr_client/action_mapper.py +++ b/lib/galaxy/jobs/runners/lwr_client/action_mapper.py @@ -1,4 +1,4 @@ -from simplejson import load +from json import load from os.path import abspath from os.path import dirname from os.path import join diff --git a/lib/galaxy/jobs/runners/lwr_client/client.py b/lib/galaxy/jobs/runners/lwr_client/client.py index ff45642647a..b29126f0f3d 100644 --- a/lib/galaxy/jobs/runners/lwr_client/client.py +++ b/lib/galaxy/jobs/runners/lwr_client/client.py @@ -1,7 +1,6 @@ import os import shutil -import simplejson -from simplejson import dumps +from json import dumps, loads from time import sleep from .destination import submit_params @@ -16,7 +15,7 @@ class parseJson(object): def __call__(self, func): def replacement(*args, **kwargs): response = func(*args, **kwargs) - return simplejson.loads(response) + return loads(response) return replacement diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 23d9c9e03df..7075562f274 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6,7 +6,6 @@ the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ from galaxy import eggs -eggs.require("simplejson") eggs.require("pexpect") import codecs @@ -15,7 +14,7 @@ import logging import operator import os import pexpect -import simplejson +import json import socket import time from string import Template @@ -2250,7 +2249,7 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ): template_data[template.name] = tmp_dict return template_data def templates_json( self, use_name=False ): - return simplejson.dumps( self.templates_dict( use_name=use_name ) ) + return json.dumps( self.templates_dict( use_name=use_name ) ) def get_display_name( self ): """ diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index 183484467f4..0096c7bc405 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -1,8 +1,6 @@ from sqlalchemy.types import * -import pkg_resources -pkg_resources.require("simplejson") -import simplejson +import json import pickle import copy import uuid @@ -19,8 +17,8 @@ import logging log = logging.getLogger( __name__ ) # Default JSON encoder and decoder -json_encoder = simplejson.JSONEncoder( sort_keys=True ) -json_decoder = simplejson.JSONDecoder( ) +json_encoder = json.JSONEncoder( sort_keys=True ) +json_decoder = json.JSONDecoder( ) def _sniffnfix_pg9_hex(value): """ diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 565dd009196..6f5f0279e18 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -4,6 +4,7 @@ Classes encapsulating galaxy tools and tool configuration. import binascii import glob +import json import logging import os import pipes @@ -19,14 +20,12 @@ import urllib from math import isinf from galaxy import eggs -eggs.require( "simplejson" ) eggs.require( "MarkupSafe" ) #MarkupSafe must load before mako eggs.require( "Mako" ) eggs.require( "elementtree" ) eggs.require( "Paste" ) eggs.require( "SQLAlchemy >= 0.4" ) -import simplejson from cgi import FieldStorage from elementtree import ElementTree from mako.template import Template @@ -869,7 +868,7 @@ class DefaultToolState( object ): value = params_to_strings( tool.inputs, self.inputs, app ) value["__page__"] = self.page value["__rerun_remap_job_id__"] = self.rerun_remap_job_id - value = simplejson.dumps( value ) + value = json.dumps( value ) # Make it secure if secure: a = hmac_new( app.config.tool_secret, value ) @@ -888,7 +887,7 @@ class DefaultToolState( object ): test = hmac_new( app.config.tool_secret, value ) assert a == test # Restore from string - values = json_fix( simplejson.loads( value ) ) + values = json_fix( json.loads( value ) ) self.page = values.pop( "__page__" ) if '__rerun_remap_job_id__' in values: self.rerun_remap_job_id = values.pop( "__rerun_remap_job_id__" ) @@ -2921,7 +2920,7 @@ class Tool( object, Dictifiable ): try: json_file = open( os.path.join( job_working_directory, jobs.TOOL_PROVIDED_JOB_METADATA_FILE ), 'r' ) for line in json_file: - line = simplejson.loads( line ) + line = json.loads( line ) if line.get( 'type' ) == 'new_primary_dataset': new_primary_datasets[ os.path.split( line.get( 'filename' ) )[-1] ] = line except Exception: @@ -3085,7 +3084,7 @@ class OutputParameterJSONTool( Tool ): if json_filename is None: json_filename = file_name out = open( json_filename, 'w' ) - out.write( simplejson.dumps( json_params ) ) + out.write( json.dumps( json_params ) ) out.close() class DataSourceTool( OutputParameterJSONTool ): @@ -3145,7 +3144,7 @@ class DataSourceTool( OutputParameterJSONTool ): if json_filename is None: json_filename = file_name out = open( json_filename, 'w' ) - out.write( simplejson.dumps( json_params ) ) + out.write( json.dumps( json_params ) ) out.close() class AsyncDataSourceTool( DataSourceTool ): diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index decee9e1e2b..f5091e1c4ba 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -1,9 +1,6 @@ -import pkg_resources - -pkg_resources.require( "simplejson" ) - -import os, errno -import simplejson +import errno +import json +import os from galaxy import util from galaxy.util.odict import odict @@ -226,7 +223,7 @@ class DataManager( object ): #TODO: fix this merging below for output_name, output_dataset in out_data.iteritems(): try: - output_dict = simplejson.loads( open( output_dataset.file_name ).read() ) + output_dict = json.loads( open( output_dataset.file_name ).read() ) except Exception, e: log.warning( 'Error reading DataManagerTool json for "%s": %s' % ( output_name, e ) ) continue diff --git a/lib/galaxy/tools/genome_index/__init__.py b/lib/galaxy/tools/genome_index/__init__.py index 80b2287fba8..0fb36f7be4a 100644 --- a/lib/galaxy/tools/genome_index/__init__.py +++ b/lib/galaxy/tools/genome_index/__init__.py @@ -1,6 +1,11 @@ from __future__ import with_statement -import os, shutil, logging, tempfile, tarfile +import json +import logging +import os +import shutil +import tarfile +import tempfile from galaxy import model, util from galaxy.web.framework.helpers import to_unicode @@ -9,9 +14,6 @@ from galaxy.util.json import * from galaxy.web.base.controller import UsesHistoryMixin from galaxy.tools.data import ToolDataTableManager -import pkg_resources -pkg_resources.require("simplejson") -import simplejson log = logging.getLogger(__name__) @@ -74,7 +76,7 @@ class GenomeIndexToolWrapper( object ): fp = open( gitd.dataset.get_file_name(), 'r' ) deferred = sa_session.query( model.DeferredJob ).filter_by( id=gitd.deferred_job_id ).first() try: - logloc = simplejson.load( fp ) + logloc = json.load( fp ) except ValueError: deferred.state = app.model.DeferredJob.states.ERROR sa_session.add( deferred ) diff --git a/lib/galaxy/tools/genome_index/index_genome.py b/lib/galaxy/tools/genome_index/index_genome.py index 7c7017a9966..fa10450059c 100644 --- a/lib/galaxy/tools/genome_index/index_genome.py +++ b/lib/galaxy/tools/genome_index/index_genome.py @@ -7,11 +7,17 @@ usage: %prog history_attrs dataset_attrs job_attrs out_file """ from __future__ import with_statement -import optparse, sys, os, tempfile, time, subprocess, shlex, tarfile, shutil +import json +import optparse +import os +import shlex +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time -import pkg_resources -pkg_resources.require("simplejson") -import simplejson class ManagedIndexer(): def __init__( self, output_file, infile, workingdir, rsync_url, tooldata ): @@ -76,7 +82,7 @@ class ManagedIndexer(): return result def _flush_files( self ): - simplejson.dump( self.locations, self.outfile ) + json.dump( self.locations, self.outfile ) self.outfile.close() self.logfile.close() @@ -318,4 +324,4 @@ if __name__ == "__main__": returncode = idxobj.run_indexer( indexer ) if not returncode: exit(1) - exit(0) \ No newline at end of file + exit(0) diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index 2639a3fa78d..6b7f8a954bb 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -1,4 +1,4 @@ -import os, shutil, logging, tempfile, simplejson +import os, shutil, logging, tempfile, json from galaxy import model from galaxy.tools.parameters.basic import UnvalidatedValue from galaxy.web.framework.helpers import to_unicode @@ -226,13 +226,13 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations self.sa_session.add( imported_job ) self.sa_session.flush() - class HistoryDatasetAssociationIDEncoder( simplejson.JSONEncoder ): + class HistoryDatasetAssociationIDEncoder( json.JSONEncoder ): """ Custom JSONEncoder for a HistoryDatasetAssociation that encodes an HDA as its ID. """ def default( self, obj ): """ Encode an HDA, default encoding for everything else. """ if isinstance( obj, model.HistoryDatasetAssociation ): return obj.id - return simplejson.JSONEncoder.default( self, obj ) + return json.JSONEncoder.default( self, obj ) # Set parameters. May be useful to look at metadata.py for creating parameters. # TODO: there may be a better way to set parameters, e.g.: @@ -311,7 +311,7 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations del metadata[ name ] return metadata - class HistoryDatasetAssociationEncoder( simplejson.JSONEncoder ): + class HistoryDatasetAssociationEncoder( json.JSONEncoder ): """ Custom JSONEncoder for a HistoryDatasetAssociation. """ def default( self, obj ): """ Encode an HDA, default encoding for everything else. """ @@ -337,7 +337,7 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations } if isinstance( obj, UnvalidatedValue ): return obj.__str__() - return simplejson.JSONEncoder.default( self, obj ) + return json.JSONEncoder.default( self, obj ) # # Create attributes/metadata files for export. diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 2d116466d3e..1790a92e19a 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -2,7 +2,26 @@ Utility functions used systemwide. """ -import binascii, errno, grp, logging, os, pickle, random, re, shutil, smtplib, stat, string, sys, tempfile, threading + +from __future__ import absolute_import + +import binascii +import errno +import grp +import json +import logging +import os +import pickle +import random +import re +import shutil +import smtplib +import stat +import string +import sys +import tempfile +import threading + from email.MIMEText import MIMEText from os.path import relpath @@ -21,12 +40,9 @@ from elementtree import ElementTree, ElementInclude eggs.require( "wchartype" ) import wchartype -from inflection import Inflector, English +from .inflection import Inflector, English inflector = Inflector(English) -eggs.require( "simplejson" ) -import simplejson - log = logging.getLogger(__name__) _lock = threading.RLock() @@ -292,8 +308,8 @@ def shrink_string_by_size( value, size, join_by="..", left_larger=True, beginnin def pretty_print_json(json_data, is_json_string=False): if is_json_string: - json_data = simplejson.loads(json_data) - return simplejson.dumps(json_data, sort_keys=True, indent=4 * ' ') + json_data = json.loads(json_data) + return json.dumps(json_data, sort_keys=True, indent=4 * ' ') # characters that are valid valid_chars = set(string.letters + string.digits + " -=_.()/+*^,:?!") diff --git a/lib/galaxy/util/json.py b/lib/galaxy/util/json.py index 5219a44f52a..7e7d5ba2368 100644 --- a/lib/galaxy/util/json.py +++ b/lib/galaxy/util/json.py @@ -1,16 +1,15 @@ +from __future__ import absolute_import __all__ = [ "to_json_string", "from_json_string", "json_fix", "validate_jsonrpc_request", "validate_jsonrpc_response", "jsonrpc_request", "jsonrpc_response" ] -import random, string, logging +import json +import logging +import random import socket +import string -import pkg_resources -pkg_resources.require( "simplejson" ) - -import simplejson - -to_json_string = simplejson.dumps -from_json_string = simplejson.loads +to_json_string = json.dumps +from_json_string = json.loads log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/visualization/data_providers/phyloviz/baseparser.py b/lib/galaxy/visualization/data_providers/phyloviz/baseparser.py index 0abcafd2f6d..f44d3606a17 100644 --- a/lib/galaxy/visualization/data_providers/phyloviz/baseparser.py +++ b/lib/galaxy/visualization/data_providers/phyloviz/baseparser.py @@ -1,6 +1,4 @@ -import pkg_resources -pkg_resources.require("simplejson") -import simplejson +import json class Node(object): """Node class of PhyloTree, which represents a CLAUDE in a phylogenetic tree""" @@ -118,7 +116,7 @@ class Base_Parser(object): def toJson(self, jsonDict): """Convenience method to get a json string from a python json dict""" - return simplejson.dumps(jsonDict) + return json.dumps(jsonDict) def _writeJsonToFile(self, filepath, json): """Writes the file out to the system""" diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py index 7a019b7efcd..4432d743924 100644 --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -2,6 +2,7 @@ Galaxy web application framework """ +import hashlib import inspect import os import pkg_resources @@ -9,27 +10,24 @@ import random import socket import string import time -import hashlib -from Cookie import CookieError +from functools import wraps +from Cookie import CookieError pkg_resources.require( "Cheetah" ) from Cheetah.Template import Template + +#TODO: Relative imports to be removed import base -from functools import wraps -from galaxy import util -from galaxy.exceptions import MessageException -from galaxy.util.json import to_json_string, from_json_string -from galaxy.util.backports.importlib import import_module -from galaxy.util.sanitize_html import sanitize_html -from galaxy.util import safe_str_cmp - -pkg_resources.require( "simplejson" ) -import simplejson - import helpers +from galaxy import util +from galaxy.exceptions import MessageException from galaxy.util import asbool +from galaxy.util import safe_str_cmp +from galaxy.util.backports.importlib import import_module +from galaxy.util.json import from_json_string, to_json_string +from galaxy.util.sanitize_html import sanitize_html import paste.httpexceptions @@ -76,7 +74,7 @@ def json( func ): @wraps(func) def decorator( self, trans, *args, **kwargs ): trans.response.set_content_type( "text/javascript" ) - return simplejson.dumps( func( self, trans, *args, **kwargs ) ) + return to_json_string( func( self, trans, *args, **kwargs ) ) if not hasattr(func, '_orig'): decorator._orig = func decorator.exposed = True @@ -86,7 +84,7 @@ def json_pretty( func ): @wraps(func) def decorator( self, trans, *args, **kwargs ): trans.response.set_content_type( "text/javascript" ) - return simplejson.dumps( func( self, trans, *args, **kwargs ), indent=4, sort_keys=True ) + return to_json_string( func( self, trans, *args, **kwargs ), indent=4, sort_keys=True ) if not hasattr(func, '_orig'): decorator._orig = func decorator.exposed = True @@ -158,7 +156,7 @@ def expose_api( func, to_json=True, user_required=True ): for k, v in payload.iteritems(): if isinstance(v, (str, unicode)): try: - payload[k] = simplejson.loads(v) + payload[k] = from_json_string(v) except: # may not actually be json, just continue pass @@ -167,7 +165,7 @@ def expose_api( func, to_json=True, user_required=True ): # Assume application/json content type and parse request body manually, since wsgi won't do it. However, the order of this check # should ideally be in reverse, with the if clause being a check for application/json and the else clause assuming a standard encoding # such as multipart/form-data. Leaving it as is for backward compatibility, just in case. - payload = util.recursively_stringify_dictionary_keys( simplejson.loads( trans.request.body ) ) + payload = util.recursively_stringify_dictionary_keys( from_json_string( trans.request.body ) ) return payload try: kwargs['payload'] = extract_payload_from_request(trans, func, kwargs) @@ -198,9 +196,9 @@ def expose_api( func, to_json=True, user_required=True ): try: rval = func( self, trans, *args, **kwargs) if to_json and trans.debug: - rval = simplejson.dumps( rval, indent=4, sort_keys=True ) + rval = to_json_string( rval, indent=4, sort_keys=True ) elif to_json: - rval = simplejson.dumps( rval ) + rval = to_json_string( rval ) return rval except paste.httpexceptions.HTTPException: raise # handled diff --git a/lib/galaxy/webapps/demo_sequencer/framework/__init__.py b/lib/galaxy/webapps/demo_sequencer/framework/__init__.py index 54cc7e121cf..b2d563fca00 100644 --- a/lib/galaxy/webapps/demo_sequencer/framework/__init__.py +++ b/lib/galaxy/webapps/demo_sequencer/framework/__init__.py @@ -2,9 +2,15 @@ Demo sequencer web application framework """ +import json +import os import pkg_resources +import random +import socket +import string +import sys +import time -import os, sys, time, socket, random, string pkg_resources.require( "Cheetah" ) from Cheetah.Template import Template @@ -19,9 +25,6 @@ import galaxy.web.framework.base from galaxy.util import asbool -pkg_resources.require( "simplejson" ) -import simplejson - pkg_resources.require( "Mako" ) import mako.template import mako.lookup diff --git a/lib/galaxy/webapps/galaxy/controllers/data_admin.py b/lib/galaxy/webapps/galaxy/controllers/data_admin.py index 82762f1d068..59f36f2c808 100644 --- a/lib/galaxy/webapps/galaxy/controllers/data_admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/data_admin.py @@ -1,14 +1,13 @@ -import sys, ftplib +import ftplib +import json +import sys from galaxy import model, util from galaxy.jobs import transfer_manager -from galaxy.web.base.controller import * -from galaxy.web.framework.helpers import time_ago, iff, grids from galaxy.model.orm import * +from galaxy.web.base.controller import * +from galaxy.web.framework.helpers import grids, iff, time_ago from library_common import get_comptypes, lucene_search, whoosh_search -import pkg_resources -pkg_resources.require("simplejson") -import simplejson # Older py compatibility try: @@ -161,7 +160,7 @@ class DataAdmin( BaseUIController ): gname = deferred.params[ 'intname' ] indexers = ', '.join( deferred.params[ 'indexes' ] ) jobs = self._get_jobs( deferred, trans ) - jsonjobs = simplejson.dumps( jobs ) + jsonjobs = json.dumps( jobs ) return trans.fill_template( '/admin/data_admin/download_status.mako', name=gname, indexers=indexers, mainjob=jobid, jobs=jobs, jsonjobs=jsonjobs ) @web.expose @@ -173,7 +172,7 @@ class DataAdmin( BaseUIController ): jobid = params.get( 'jobid', '' ) job = sa_session.query( model.DeferredJob ).filter_by( id=jobid ).first() jobs = self._get_jobs( job, trans ) - return trans.fill_template( '/admin/data_admin/ajax_status.mako', json=simplejson.dumps( jobs ) ) + return trans.fill_template( '/admin/data_admin/ajax_status.mako', json=json.dumps( jobs ) ) def _get_job( self, jobid, jobtype, trans ): sa = trans.app.model.context.current @@ -297,4 +296,4 @@ def build_param_dict( params, trans ): params = dict( status='ok', dbkey=dbkey, datatype='fasta', url=url, user=trans.user.id, liftover=newlift, longname=longname, indexers=indexers ) - return params \ No newline at end of file + return params diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index 3a4ac83bc45..78468b9f33b 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -508,7 +508,7 @@ class RootController( BaseUIController, UsesHistoryMixin, UsesHistoryDatasetAsso Attempts to parse values passed as boolean, float, then int. Defaults to string. Non-recursive (will not parse lists). """ - #TODO: use simplejson or json + #TODO: use json rval = {} for k in kwd: rval[ k ] = kwd[k] diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index c4477535f7e..14b1ab0345a 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -1,13 +1,12 @@ import pkg_resources -pkg_resources.require( "simplejson" ) pkg_resources.require( "SVGFig" ) import base64 import httplib +import json import math import os import sgmllib -import simplejson import svgfig import urllib2 @@ -20,17 +19,17 @@ from galaxy import util from galaxy import web from galaxy.datatypes.data import Data from galaxy.jobs.actions.post import ActionBox +from galaxy.model.item_attrs import UsesAnnotations, UsesItemRatings from galaxy.model.mapping import desc from galaxy.tools.parameters import RuntimeValue, visit_input_values from galaxy.tools.parameters.basic import DataToolParameter, DrillDownSelectToolParameter, SelectToolParameter, UnvalidatedValue from galaxy.tools.parameters.grouping import Conditional, Repeat -from galaxy.util.odict import odict from galaxy.util.json import to_json_string +from galaxy.util.odict import odict from galaxy.util.sanitize_html import sanitize_html from galaxy.util.topsort import CycleError, topsort, topsort_levels from galaxy.web import error, url_for from galaxy.web.base.controller import BaseUIController, SharableMixin, UsesStoredWorkflowMixin -from galaxy.model.item_attrs import UsesAnnotations, UsesItemRatings from galaxy.web.framework import form from galaxy.web.framework.helpers import grids, time_ago from galaxy.web.framework.helpers import to_unicode @@ -812,7 +811,7 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix # Put parameters in workflow mode trans.workflow_building_mode = True # Convert incoming workflow data from json - data = simplejson.loads( workflow_data ) + data = json.loads( workflow_data ) # Create new workflow from incoming data workflow = model.Workflow() # Just keep the last name (user can rename later) @@ -920,7 +919,7 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix # # Create workflow content JSON. - workflow_content = simplejson.dumps( workflow_dict, indent=4, sort_keys=True ) + workflow_content = json.dumps( workflow_dict, indent=4, sort_keys=True ) # Create myExperiment request. request_raw = trans.fill_template( "workflow/myexp_export.mako", \ @@ -1073,7 +1072,7 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix if workflow_data: # Convert incoming workflow data from json try: - data = simplejson.loads( workflow_data ) + data = json.loads( workflow_data ) except Exception, e: data = None message = "The data content does not appear to be a Galaxy workflow.
Exception: %s" % str( e ) @@ -1294,7 +1293,7 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix # It is possible for a workflow to have 0 steps if len( workflow.steps ) == 0: error( "Workflow cannot be run because it does not have any steps" ) - #workflow = Workflow.from_simple( simplejson.loads( stored.encoded_value ), trans.app ) + #workflow = Workflow.from_simple( json.loads( stored.encoded_value ), trans.app ) if workflow.has_cycles: error( "Workflow cannot be run because it contains cycles" ) if workflow.has_errors: diff --git a/lib/tool_shed/scripts/api/common.py b/lib/tool_shed/scripts/api/common.py index 56f42de4298..9ed77d7c5aa 100644 --- a/lib/tool_shed/scripts/api/common.py +++ b/lib/tool_shed/scripts/api/common.py @@ -1,4 +1,8 @@ -import os, sys, urllib, urllib2 +import json +import os +import sys +import urllib +import urllib2 new_path = [ os.path.join( os.path.dirname( __file__ ), '..', '..', '..', '..', 'lib' ) ] new_path.extend( sys.path[ 1: ] ) @@ -7,9 +11,6 @@ sys.path = new_path from galaxy import eggs import pkg_resources -pkg_resources.require( "simplejson" ) -import simplejson - pkg_resources.require( "pycrypto" ) from Crypto.Cipher import Blowfish from Crypto.Util.randpool import RandomPool @@ -29,9 +30,9 @@ def delete( api_key, url, data, return_formatted=True ): # Sends an API DELETE request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. try: url = make_url( api_key, url ) - req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data )) req.get_method = lambda: 'DELETE' - r = simplejson.loads( urllib2.urlopen( req ).read() ) + r = json.loads( urllib2.urlopen( req ).read() ) except urllib2.HTTPError, e: if return_formatted: print e @@ -86,8 +87,8 @@ def get( url, api_key=None ): # Do the actual GET. url = make_url( url, api_key=api_key ) try: - return simplejson.loads( urllib2.urlopen( url ).read() ) - except simplejson.decoder.JSONDecodeError, e: + return json.loads( urllib2.urlopen( url ).read() ) + except ValueError, e: print "URL did not return JSON data" sys.exit(1) @@ -106,15 +107,15 @@ def make_url( url, api_key=None, args=None ): def post( url, data, api_key=None ): # Do the actual POST. url = make_url( url, api_key=api_key ) - req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data ) ) - return simplejson.loads( urllib2.urlopen( req ).read() ) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data ) ) + return json.loads( urllib2.urlopen( req ).read() ) def put( url, data, api_key=None ): # Do the actual PUT. url = make_url( url, api_key=api_key ) - req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data )) req.get_method = lambda: 'PUT' - return simplejson.loads( urllib2.urlopen( req ).read() ) + return json.loads( urllib2.urlopen( req ).read() ) def submit( url, data, api_key=None, return_formatted=True ): # Sends an API POST request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. diff --git a/lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py b/lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py index 16a38bd65b2..322027aaa02 100755 --- a/lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py +++ b/lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py @@ -5,17 +5,16 @@ PUT/update script to update appropriate values in a repository_metadata table re usage: tool_shed_repository_revision_update.py key url key1=value1 key2=value2 ... """ -import os, sys +import json +import os +import sys + sys.path.insert( 0, os.path.dirname( __file__ ) ) from common import update -import pkg_resources -pkg_resources.require( "simplejson" ) -import simplejson - -to_json_string = simplejson.dumps -from_json_string = simplejson.loads +to_json_string = json.dumps +from_json_string = json.loads data = {} for key, value in [ kwarg.split( '=', 1 ) for kwarg in sys.argv[ 3: ] ]: diff --git a/lib/tool_shed/util/encoding_util.py b/lib/tool_shed/util/encoding_util.py index 6b7c12c1ff9..92e53a21e0e 100644 --- a/lib/tool_shed/util/encoding_util.py +++ b/lib/tool_shed/util/encoding_util.py @@ -1,14 +1,10 @@ import binascii +import json import logging from galaxy import eggs from galaxy.util.hash_util import hmac_new from galaxy.util.json import json_fix -import pkg_resources - -pkg_resources.require( "simplejson" ) -import simplejson - log = logging.getLogger( __name__ ) encoding_sep = '__esep__' @@ -23,7 +19,7 @@ def tool_shed_decode( value ): # Restore from string values = None try: - values = simplejson.loads( value ) + values = json.loads( value ) except Exception, e: #log.debug( "Decoding json value from tool shed for value '%s' threw exception: %s" % ( str( value ), str( e ) ) ) pass @@ -39,7 +35,7 @@ def tool_shed_decode( value ): def tool_shed_encode( val ): if isinstance( val, dict ): - value = simplejson.dumps( val ) + value = json.dumps( val ) else: value = val a = hmac_new( 'ToolShedAndGalaxyMustHaveThisSameKey', value ) diff --git a/scripts/api/common.py b/scripts/api/common.py index b39a0b136a2..fd240a5d0cf 100644 --- a/scripts/api/common.py +++ b/scripts/api/common.py @@ -1,3 +1,4 @@ +import json import logging import os import sys @@ -10,9 +11,6 @@ sys.path = new_path from galaxy import eggs import pkg_resources -pkg_resources.require( "simplejson" ) -import simplejson - pkg_resources.require( "pycrypto" ) from Crypto.Cipher import Blowfish from Crypto.Util.randpool import RandomPool @@ -35,30 +33,30 @@ def get( api_key, url ): # Do the actual GET. url = make_url( api_key, url ) try: - return simplejson.loads( urllib2.urlopen( url ).read() ) - except simplejson.decoder.JSONDecodeError, e: + return json.loads( urllib2.urlopen( url ).read() ) + except json.decoder.JSONDecodeError, e: print "URL did not return JSON data" sys.exit(1) def post( api_key, url, data ): # Do the actual POST. url = make_url( api_key, url ) - req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data ) ) - return simplejson.loads( urllib2.urlopen( req ).read() ) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data ) ) + return json.loads( urllib2.urlopen( req ).read() ) def put( api_key, url, data ): # Do the actual PUT url = make_url( api_key, url ) - req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data )) req.get_method = lambda: 'PUT' - return simplejson.loads( urllib2.urlopen( req ).read() ) + return json.loads( urllib2.urlopen( req ).read() ) def __del( api_key, url, data ): # Do the actual DELETE url = make_url( api_key, url ) - req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data )) req.get_method = lambda: 'DELETE' - return simplejson.loads( urllib2.urlopen( req ).read() ) + return json.loads( urllib2.urlopen( req ).read() ) def display( api_key, url, return_formatted=True ): diff --git a/scripts/api/workflow_import_from_file_rpark.py b/scripts/api/workflow_import_from_file_rpark.py index 3443d3989fa..d1c8ad0af6a 100644 --- a/scripts/api/workflow_import_from_file_rpark.py +++ b/scripts/api/workflow_import_from_file_rpark.py @@ -11,11 +11,11 @@ sys.path.insert( 0, os.path.dirname( __file__ ) ) from common import submit ### Rpark edit ### -import simplejson +import json def openWorkflow(in_file): with open(in_file) as f: - temp_data = simplejson.load(f) + temp_data = json.load(f) return temp_data; diff --git a/scripts/drmaa_external_killer.py b/scripts/drmaa_external_killer.py index 98320ce2540..37c7ebe004f 100755 --- a/scripts/drmaa_external_killer.py +++ b/scripts/drmaa_external_killer.py @@ -4,10 +4,11 @@ Terminates a DRMAA job if given a job id and (appropriate) user id. """ -import os -import sys import errno +import json +import os import pwd +import sys #import drmaa new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path @@ -15,8 +16,6 @@ sys.path = new_path from galaxy import eggs import pkg_resources -pkg_resources.require("simplejson") -import simplejson as json pkg_resources.require("drmaa") import drmaa diff --git a/scripts/drmaa_external_runner.py b/scripts/drmaa_external_runner.py index bead083fdf5..7653654a1c1 100755 --- a/scripts/drmaa_external_runner.py +++ b/scripts/drmaa_external_runner.py @@ -10,8 +10,8 @@ import os import sys import errno import pwd +import json -#import simplejson as json #import drmaa new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path @@ -19,8 +19,6 @@ sys.path = new_path from galaxy import eggs import pkg_resources -pkg_resources.require("simplejson") -import simplejson as json pkg_resources.require("drmaa") import drmaa diff --git a/scripts/external_chown_script.py b/scripts/external_chown_script.py index ce0e268a454..b53c97ea457 100755 --- a/scripts/external_chown_script.py +++ b/scripts/external_chown_script.py @@ -1,17 +1,17 @@ #!/usr/bin/env python -import os -import sys import errno +import json +import os import pwd +import sys #import drmaa + new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path from galaxy import eggs import pkg_resources -pkg_resources.require("simplejson") -import simplejson as json pkg_resources.require("drmaa") import drmaa diff --git a/scripts/extract_dataset_part.py b/scripts/extract_dataset_part.py index 862cb8f40cb..05c57cf5ef6 100644 --- a/scripts/extract_dataset_part.py +++ b/scripts/extract_dataset_part.py @@ -6,9 +6,10 @@ Used by jobs that split large files into pieces to be processed concurrently on a gid in a scatter-gather mode. This does part of the scatter. """ +import json +import logging import os import sys -import logging logging.basicConfig() log = logging.getLogger( __name__ ) @@ -16,11 +17,6 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path -from galaxy import eggs -import pkg_resources -pkg_resources.require("simplejson") -import simplejson - # This junk is here to prevent loading errors import galaxy.model.mapping #need to load this before we unpickle, in order to setup properties assigned by the mappers galaxy.model.Job() #this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here @@ -33,7 +29,7 @@ def __main__(): if not os.path.isfile(file_path): #Nothing to do - some splitters don't write a JSON file sys.exit(0) - data = simplejson.load(open(file_path, 'r')) + data = json.load(open(file_path, 'r')) try: class_name_parts = data['class_name'].split('.') module_name = '.'.join(class_name_parts[:-1]) diff --git a/scripts/galaxy_messaging/server/data_transfer.py b/scripts/galaxy_messaging/server/data_transfer.py index 619100a676d..6eb90b95348 100755 --- a/scripts/galaxy_messaging/server/data_transfer.py +++ b/scripts/galaxy_messaging/server/data_transfer.py @@ -13,12 +13,21 @@ python data_transfer.py """ import ConfigParser -import sys, os, time, traceback +import cookielib +import datetime +import logging import optparse -import urllib,urllib2, cookielib, shutil -import logging, time, datetime +import os +import shutil +import sys +import time +import time +import traceback +import urllib +import urllib2 import xml.dom.minidom + from xml_helper import get_value, get_value_index log = logging.getLogger( "datatx_" + str( os.getpid() ) ) @@ -39,14 +48,11 @@ new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path from galaxy import eggs -from galaxy.util.json import from_json_string, to_json_string from galaxy.model import SampleDataset from galaxy.web.api.samples import SamplesAPIController import pkg_resources pkg_resources.require( "pexpect" ) import pexpect -pkg_resources.require( "simplejson" ) -import simplejson log.debug(str(dir(api))) diff --git a/scripts/set_metadata.py b/scripts/set_metadata.py index fdeb2b0da09..711b02f3de5 100644 --- a/scripts/set_metadata.py +++ b/scripts/set_metadata.py @@ -10,9 +10,11 @@ import logging logging.basicConfig() log = logging.getLogger( __name__ ) +import cPickle +import json import os import sys -import cPickle + # ensure supported version from check_python import check_python try: @@ -26,8 +28,6 @@ sys.path = new_path from galaxy import eggs import pkg_resources -pkg_resources.require("simplejson") -import simplejson import galaxy.model.mapping # need to load this before we unpickle, in order to setup properties assigned by the mappers galaxy.model.Job() # this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here from galaxy.util import stringify_dictionary_keys @@ -107,17 +107,17 @@ def __main__(): dataset.extension = ext_override[ dataset.dataset.id ] # Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles if override_metadata: - override_metadata = simplejson.load( open( override_metadata ) ) + override_metadata = json.load( open( override_metadata ) ) for metadata_name, metadata_file_override in override_metadata: if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value( metadata_file_override ): metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON( metadata_file_override ) setattr( dataset.metadata, metadata_name, metadata_file_override ) - kwds = stringify_dictionary_keys( simplejson.load( open( filename_kwds ) ) ) # load kwds; need to ensure our keywords are not unicode + kwds = stringify_dictionary_keys( json.load( open( filename_kwds ) ) ) # load kwds; need to ensure our keywords are not unicode dataset.datatype.set_meta( dataset, **kwds ) dataset.metadata.to_JSON_dict( filename_out ) # write out results of set_meta - simplejson.dump( ( True, 'Metadata has been set successfully' ), open( filename_results_code, 'wb+' ) ) # setting metadata has succeeded + json.dump( ( True, 'Metadata has been set successfully' ), open( filename_results_code, 'wb+' ) ) # setting metadata has succeeded except Exception, e: - simplejson.dump( ( False, str( e ) ), open( filename_results_code, 'wb+' ) ) # setting metadata has failed somehow + json.dump( ( False, str( e ) ), open( filename_results_code, 'wb+' ) ) # setting metadata has failed somehow clear_mappers() # Shut down any additional threads that might have been created via the ObjectStore object_store.shutdown() diff --git a/test/base/interactor.py b/test/base/interactor.py index 9e8b9acebb8..95b3ca20656 100644 --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -5,7 +5,7 @@ from galaxy.util.odict import odict import galaxy.model from galaxy.model.orm import and_, desc from base.test_db_util import sa_session -from simplejson import dumps, loads +from json import dumps, loads from logging import getLogger log = getLogger( __name__ ) diff --git a/test/functional/test_workflow.py b/test/functional/test_workflow.py index 93ce0e02afe..db2c88d292b 100644 --- a/test/functional/test_workflow.py +++ b/test/functional/test_workflow.py @@ -5,7 +5,7 @@ from base.interactor import GalaxyInteractorApi, stage_data_in_history from galaxy.util import parse_xml from galaxy.tools.test import parse_param_elem, require_file, test_data_iter, parse_output_elems -from simplejson import load, dumps +from json import load, dumps from logging import getLogger log = getLogger( __name__ ) diff --git a/test/tool_shed/base/twilltestcase.py b/test/tool_shed/base/twilltestcase.py index f73419b23a8..4e282e6b215 100644 --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -3,7 +3,6 @@ import string import os import re import test_db_util -import simplejson import shutil import logging import time diff --git a/tools/filters/join.py b/tools/filters/join.py index f2fa82cd076..db29cdca8c9 100644 --- a/tools/filters/join.py +++ b/tools/filters/join.py @@ -8,20 +8,15 @@ User can also opt to have have non-joining rows of file1 echoed. """ -import optparse, os, sys, tempfile, struct +import json +import optparse +import os import psyco_full - -try: - simple_json_exception = None - from galaxy import eggs - from galaxy.util.bunch import Bunch - from galaxy.util import stringify_dictionary_keys - import pkg_resources - pkg_resources.require("simplejson") - import simplejson -except Exception, e: - simplejson_exception = e - simplejson = None +import struct +import sys +import tempfile +from galaxy.util.bunch import Bunch +from galaxy.util import stringify_dictionary_keys class OffsetList: @@ -337,11 +332,9 @@ def main(): fill_options = None if options.fill_options_file is not None: try: - if simplejson is None: - raise simplejson_exception - fill_options = Bunch( **stringify_dictionary_keys( simplejson.load( open( options.fill_options_file ) ) ) ) #simplejson.load( open( options.fill_options_file ) ) + fill_options = Bunch( **stringify_dictionary_keys( json.load( open( options.fill_options_file ) ) ) ) #json.load( open( options.fill_options_file ) ) except Exception, e: - print "Warning: Ignoring fill options due to simplejson error (%s)." % e + print "Warning: Ignoring fill options due to json error (%s)." % e if fill_options is None: fill_options = Bunch() if 'fill_unjoined_only' not in fill_options: diff --git a/tools/filters/joiner.xml b/tools/filters/joiner.xml index 757965118c0..953145f4594 100644 --- a/tools/filters/joiner.xml +++ b/tools/filters/joiner.xml @@ -51,7 +51,7 @@ <% -import simplejson +import json %> #set $__fill_options = {} #if $fill_empty_columns['fill_empty_columns_switch'] == 'fill_empty': @@ -72,7 +72,7 @@ import simplejson #end for #end if #end if -${simplejson.dumps( __fill_options )} +${json.dumps( __fill_options )} diff --git a/tools/genomespace/genomespace_exporter.py b/tools/genomespace/genomespace_exporter.py index 2ebebcf5a61..fc2a2a09ab1 100644 --- a/tools/genomespace/genomespace_exporter.py +++ b/tools/genomespace/genomespace_exporter.py @@ -1,12 +1,16 @@ #Dan Blankenberg -import optparse, os, urllib2, urllib, cookielib, hashlib, base64, cgi, binascii, logging - -from galaxy import eggs -import pkg_resources - -pkg_resources.require( "simplejson" ) -import simplejson +import base64 +import binascii +import cgi +import cookielib +import hashlib +import json +import logging +import optparse +import os +import urllib +import urllib2 log = logging.getLogger( "tools.genomespace.genomespace_exporter" )#( __name__ ) @@ -58,7 +62,7 @@ def get_directory( url_opener, dm_url, path ): dir_request = urllib2.Request( url, headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' } ) dir_request.get_method = lambda: 'GET' try: - dir_dict = simplejson.loads( url_opener.open( dir_request ).read() ) + dir_dict = json.loads( url_opener.open( dir_request ).read() ) except urllib2.HTTPError, e: #print "e", e, url #punting, assuming lack of permissions at this low of a level... continue @@ -81,16 +85,16 @@ def create_directory( url_opener, directory_dict, new_dir, dm_url ): if dir_slice in ( '', '/', None ): continue url = '/'.join( ( directory_dict['url'], urllib.quote( dir_slice.replace( '/', '_' ), safe='' ) ) ) - new_dir_request = urllib2.Request( url, headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' }, data = simplejson.dumps( payload ) ) + new_dir_request = urllib2.Request( url, headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' }, data = json.dumps( payload ) ) new_dir_request.get_method = lambda: 'PUT' - directory_dict = simplejson.loads( url_opener.open( new_dir_request ).read() ) + directory_dict = json.loads( url_opener.open( new_dir_request ).read() ) return directory_dict def get_genome_space_launch_apps( atm_url, url_opener, file_url, file_type ): gs_request = urllib2.Request( "%s/%s/webtool/descriptor" % ( atm_url, GENOMESPACE_API_VERSION_STRING ) ) gs_request.get_method = lambda: 'GET' opened_gs_request = url_opener.open( gs_request ) - webtool_descriptors = simplejson.loads( opened_gs_request.read() ) + webtool_descriptors = json.loads( opened_gs_request.read() ) webtools = [] for webtool in webtool_descriptors: webtool_name = webtool.get( 'name' ) @@ -125,7 +129,7 @@ def galaxy_code_get_genomespace_folders( genomespace_site='prod', trans=None, va except urllib2.HTTPError, e: log.debug( 'GenomeSpace export tool failed reading a directory "%s": %s' % ( url, e ) ) return #bad url, go to next - cur_directory = simplejson.loads( cur_directory ) + cur_directory = json.loads( cur_directory ) directory = cur_directory.get( 'directory', {} ) contents = cur_directory.get( 'contents', [] ) if directory.get( 'isDirectory', False ): diff --git a/tools/genomespace/genomespace_file_browser.py b/tools/genomespace/genomespace_file_browser.py index 002b72d90b9..7b1ce406ef0 100644 --- a/tools/genomespace/genomespace_file_browser.py +++ b/tools/genomespace/genomespace_file_browser.py @@ -1,12 +1,13 @@ #Dan Blankenberg -import optparse, os, urllib, urllib2, urlparse, cookielib +import cookielib +import json +import optparse +import os +import urllib +import urllib2 +import urlparse -from galaxy import eggs -import pkg_resources - -pkg_resources.require( "simplejson" ) -import simplejson GENOMESPACE_API_VERSION_STRING = "v1.0" GENOMESPACE_SERVER_URL_PROPERTIES = "https://dm.genomespace.org/config/%s/serverurl.properties" % ( GENOMESPACE_API_VERSION_STRING ) @@ -87,12 +88,12 @@ def set_genomespace_format_identifiers( url_opener, dm_site ): gs_request = urllib2.Request( "%s/%s/dataformat/list" % ( dm_site, GENOMESPACE_API_VERSION_STRING ) ) gs_request.get_method = lambda: 'GET' opened_gs_request = url_opener.open( gs_request ) - genomespace_formats = simplejson.loads( opened_gs_request.read() ) + genomespace_formats = json.loads( opened_gs_request.read() ) for format in genomespace_formats: GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT[ format['url'] ] = format['name'] def download_from_genomespace_file_browser( json_parameter_file, genomespace_site ): - json_params = simplejson.loads( open( json_parameter_file, 'r' ).read() ) + json_params = json.loads( open( json_parameter_file, 'r' ).read() ) datasource_params = json_params.get( 'param_dict' ) username = datasource_params.get( "gs-username", None ) token = datasource_params.get( "gs-token", None ) @@ -150,14 +151,14 @@ def download_from_genomespace_file_browser( json_parameter_file, genomespace_sit filename = "-%s" % filename used_filenames.append( filename ) output_filename = os.path.join( datasource_params['__new_file_path__'], 'primary_%i_%s_visible_%s' % ( hda_id, filename, galaxy_ext ) ) - metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'new_primary_dataset', + metadata_parameter_file.write( "%s\n" % json.dumps( dict( type = 'new_primary_dataset', base_dataset_id = dataset_id, ext = galaxy_ext, filename = output_filename, name = "GenomeSpace import on %s" % ( original_filename ) ) ) ) else: if dataset_id is not None: - metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'dataset', + metadata_parameter_file.write( "%s\n" % json.dumps( dict( type = 'dataset', dataset_id = dataset_id, ext = galaxy_ext, name = "GenomeSpace import on %s" % ( filename ) ) ) ) diff --git a/tools/genomespace/genomespace_importer.py b/tools/genomespace/genomespace_importer.py index 75977fc7363..ef467ff2469 100644 --- a/tools/genomespace/genomespace_importer.py +++ b/tools/genomespace/genomespace_importer.py @@ -1,12 +1,14 @@ #Dan Blankenberg -import optparse, os, urllib2, urllib, cookielib, urlparse, tempfile, shutil - -from galaxy import eggs -import pkg_resources - -pkg_resources.require( "simplejson" ) -import simplejson +import cookielib +import json +import optparse +import os +import shutil +import tempfile +import urllib +import urllib2 +import urlparse import galaxy.model # need to import model before sniff to resolve a circular import dependency from galaxy.datatypes import sniff @@ -91,12 +93,12 @@ def set_genomespace_format_identifiers( url_opener, dm_site ): gs_request = urllib2.Request( "%s/%s/dataformat/list" % ( dm_site, GENOMESPACE_API_VERSION_STRING ) ) gs_request.get_method = lambda: 'GET' opened_gs_request = url_opener.open( gs_request ) - genomespace_formats = simplejson.loads( opened_gs_request.read() ) + genomespace_formats = json.loads( opened_gs_request.read() ) for format in genomespace_formats: GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT[ format['url'] ] = format['name'] def download_from_genomespace_importer( username, token, json_parameter_file, genomespace_site ): - json_params = simplejson.loads( open( json_parameter_file, 'r' ).read() ) + json_params = json.loads( open( json_parameter_file, 'r' ).read() ) datasource_params = json_params.get( 'param_dict' ) assert None not in [ username, token ], "Missing GenomeSpace username or token." output_filename = datasource_params.get( "output_file1", None ) @@ -152,7 +154,7 @@ def download_from_genomespace_importer( username, token, json_parameter_file, ge metadata_request = urllib2.Request( "%s/%s/filemetadata/%s" % ( genomespace_site_dict['dmServer'], GENOMESPACE_API_VERSION_STRING, download_file_path ) ) metadata_request.get_method = lambda: 'GET' metadata_url = url_opener.open( metadata_request ) - file_metadata_dict = simplejson.loads( metadata_url.read() ) + file_metadata_dict = json.loads( metadata_url.read() ) metadata_url.close() file_type = file_metadata_dict.get( 'dataFormat', None ) if file_type and file_type.get( 'url' ): @@ -176,7 +178,7 @@ def download_from_genomespace_importer( username, token, json_parameter_file, ge #save json info for single primary dataset if dataset_id is not None: - metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'dataset', + metadata_parameter_file.write( "%s\n" % json.dumps( dict( type = 'dataset', dataset_id = dataset_id, ext = file_type, name = "GenomeSpace importer on %s" % ( filename ) ) ) ) @@ -189,7 +191,7 @@ def download_from_genomespace_importer( username, token, json_parameter_file, ge used_filenames.append( filename ) target_output_filename = os.path.join( datasource_params['__new_file_path__'], 'primary_%i_%s_visible_%s' % ( hda_id, filename, file_type ) ) shutil.move( output_filename, target_output_filename ) - metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'new_primary_dataset', + metadata_parameter_file.write( "%s\n" % json.dumps( dict( type = 'new_primary_dataset', base_dataset_id = base_dataset_id, ext = file_type, filename = target_output_filename, diff --git a/tools/new_operations/column_join.py b/tools/new_operations/column_join.py index e7e025c8866..aa1624e06fd 100644 --- a/tools/new_operations/column_join.py +++ b/tools/new_operations/column_join.py @@ -13,19 +13,16 @@ usage: %prog -o output -1 input1 -2 input2 -c column1[,column2[,column3[,...]]] other_inputs: the other input files to join """ -import optparse, os, re, struct, sys, tempfile +import json +import optparse +import os +import re +import struct +import sys +import tempfile -try: - simple_json_exception = None - from galaxy import eggs - from galaxy.util.bunch import Bunch - from galaxy.util import stringify_dictionary_keys - import pkg_resources - pkg_resources.require("simplejson") - import simplejson -except Exception, e: - simplejson_exception = e - simplejson = None +from galaxy.util.bunch import Bunch +from galaxy.util import stringify_dictionary_keys def stop_err( msg ): sys.stderr.write( msg ) @@ -162,11 +159,9 @@ def __main__(): fill_options = None if options.fill_options_file != 'None' and options.fill_options_file is not None: try: - if simplejson is None: - raise simplejson_exception - fill_options = Bunch( **stringify_dictionary_keys( simplejson.load( open( options.fill_options_file ) ) ) ) + fill_options = Bunch( **stringify_dictionary_keys( json.load( open( options.fill_options_file ) ) ) ) except Exception, e: - print 'Warning: Ignoring fill options due to simplejson error (%s).' % e + print 'Warning: Ignoring fill options due to json error (%s).' % e if fill_options is None: fill_options = Bunch() if 'file1_columns' not in fill_options: diff --git a/tools/new_operations/column_join.xml b/tools/new_operations/column_join.xml index 79c890b111e..c194babfbf3 100644 --- a/tools/new_operations/column_join.xml +++ b/tools/new_operations/column_join.xml @@ -49,7 +49,7 @@ <% -import simplejson +import json %> #set $__fill_options = {} #if $fill_empty_columns['fill_empty_columns_switch'] == 'fill_empty': @@ -65,7 +65,7 @@ import simplejson #end for #end if #end if -${simplejson.dumps( __fill_options )} +${json.dumps( __fill_options )} diff --git a/tools/peak_calling/macs_wrapper.py b/tools/peak_calling/macs_wrapper.py index 0487239eff7..710f573ad06 100644 --- a/tools/peak_calling/macs_wrapper.py +++ b/tools/peak_calling/macs_wrapper.py @@ -1,8 +1,12 @@ -import sys, subprocess, tempfile, shutil, glob, os, os.path, gzip -from galaxy import eggs -import pkg_resources -pkg_resources.require( "simplejson" ) -import simplejson +import glob +import gzip +import json +import os +import os.path +import shutil +import subprocess +import sys +import tempfile CHUNK_SIZE = 1024 @@ -42,7 +46,7 @@ def xls_to_interval( xls_file, interval_file, header = None ): out.close() def main(): - options = simplejson.load( open( sys.argv[1] ) ) + options = json.load( open( sys.argv[1] ) ) output_bed = sys.argv[2] output_extra_html = sys.argv[3] output_extra_path = sys.argv[4] diff --git a/tools/peak_calling/macs_wrapper.xml b/tools/peak_calling/macs_wrapper.xml index 6601102b306..504c7bef60d 100644 --- a/tools/peak_calling/macs_wrapper.xml +++ b/tools/peak_calling/macs_wrapper.xml @@ -93,7 +93,7 @@ <% -import simplejson +import json %> #set $__options = { 'experiment_name':str( $experiment_name ), 'gsize':int( float( str( $gsize ) ) ), 'tsize':str( $tsize ), 'bw':str( $bw ), 'pvalue':str( $pvalue ), 'mfold':str( $mfold ), 'nolambda':str( $nolambda ), 'lambdaset': str( $lambdaset ), 'futurefdr':str( $futurefdr ) } #if str( $xls_to_interval ) == 'create': @@ -135,7 +135,7 @@ import simplejson #if $diag_type['diag_type_selector'] == 'diag': #set $__options['diag'] = { 'fe-min':str( $diag_type['fe-min'] ), 'fe-max':str( $diag_type['fe-max'] ), 'fe-step':str( $diag_type['fe-step'] ) } #end if -${ simplejson.dumps( __options ) } +${ json.dumps( __options ) }