From 9be7118d6a8608c265e0a1ea5023732771cb0197 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 8 Sep 2014 16:41:53 -0400 Subject: [PATCH] First pass removing to_json_string/from_json_string. --- .../bedgraph_to_array_tree_converter.py | 2 +- .../datatypes/converters/fasta_to_len.py | 2 +- .../converters/fasta_to_tabular_converter.py | 2 +- .../fastqsolexa_to_fasta_converter.py | 2 +- .../converters/interval_to_coverage.py | 2 +- .../wiggle_to_array_tree_converter.py | 2 +- lib/galaxy/datatypes/tabular.py | 4 +- lib/galaxy/datatypes/util/__init__.py | 2 +- .../result_handlers/basic.py | 6 +- lib/galaxy/jobs/__init__.py | 6 +- lib/galaxy/jobs/actions/__init__.py | 2 +- lib/galaxy/jobs/actions/post.py | 4 +- .../pacific_biosciences_smrt_portal.py | 2 +- lib/galaxy/jobs/transfer_manager.py | 2 +- lib/galaxy/managers/context.py | 4 +- .../versions/0052_sample_dataset_table.py | 4 +- .../versions/0054_visualization_dbkey.py | 4 +- .../migrate/versions/0057_request_notify.py | 6 +- .../versions/0059_sample_dataset_file_path.py | 2 +- ...0065_add_name_to_form_fields_and_values.py | 24 ++-- .../versions/0067_populate_sequencer_table.py | 16 +-- ...8_rename_sequencer_to_external_services.py | 2 +- .../0076_fix_form_values_data_corruption.py | 8 +- .../versions/0091_add_tool_version_tables.py | 4 +- .../0115_longer_user_password_field.py | 2 +- .../versions/0117_add_user_activation.py | 2 +- .../model/orm/logging_connection_proxy.py | 2 +- lib/galaxy/model/search.py | 4 +- .../0001_add_tool_shed_repository_table.py | 52 +++++++- ..._add_tool_shed_repository_table_columns.py | 80 +++++++++++- .../versions/0003_tool_id_guid_map_table.py | 53 +++++++- ..._add_installed_changeset_revison_column.py | 65 ++++++++- ..._add_tool_shed_repository_table_columns.py | 64 ++++++++- .../versions/0006_add_tool_version_tables.py | 123 +++++++++++++++++- .../versions/0007_add_migrate_tools_table.py | 51 +++++++- .../versions/0008_add_ctx_rev_column.py | 45 ++++++- .../0009_add_tool_dependency_table.py | 52 +++++++- ...er_tool_dependency_table_version_column.py | 55 +++++++- ...rop_installed_changeset_revision_column.py | 42 +++++- ...0012_add_tool_dependency_status_columns.py | 72 +++++++++- ...add_tool_shed_repository_status_columns.py | 70 +++++++++- .../0014_add_repository_dependency_tables.py | 61 ++++++++- .../0015_update_migrate_tools_table.py | 39 +++++- .../0016_update_migrate_tools_table_again.py | 41 +++++- ..._available_col_add_tool_shed_status_col.py | 78 ++++++++++- lib/galaxy/openid/__init__.py | 2 +- lib/galaxy/tools/actions/__init__.py | 4 +- lib/galaxy/tools/actions/metadata.py | 4 +- lib/galaxy/tools/actions/upload_common.py | 6 +- lib/galaxy/tools/imp_exp/__init__.py | 32 ++--- lib/galaxy/tools/imp_exp/export_history.py | 6 +- lib/galaxy/tools/parameters/__init__.py | 4 +- lib/galaxy/tools/util/__init__.py | 2 +- lib/galaxy/util/__init__.py | 4 +- lib/galaxy/util/backports/__init__.py | 2 +- .../util/backports/importlib/__init__.py | 2 +- lib/galaxy/util/dbkeys.py | 8 +- lib/galaxy/util/json.py | 8 +- lib/galaxy/util/log/__init__.py | 2 +- lib/galaxy/util/pastescript/serve.py | 2 +- .../visualization/data_providers/__init__.py | 2 +- .../visualization/data_providers/basic.py | 4 +- .../visualization/data_providers/genome.py | 12 +- .../data_providers/phyloviz/nexusparser.py | 2 +- lib/galaxy/visualization/genome/__init__.py | 2 +- lib/galaxy/visualization/genomes.py | 10 +- lib/galaxy/visualization/registry.py | 2 +- lib/galaxy/visualization/tracks/__init__.py | 2 +- lib/galaxy/web/base/controller.py | 4 +- lib/galaxy/web/form_builder.py | 4 +- lib/galaxy/web/framework/decorators.py | 18 +-- lib/galaxy/web/framework/helpers/__init__.py | 2 +- lib/galaxy/web/framework/helpers/grids.py | 22 ++-- .../web/framework/middleware/__init__.py | 2 +- lib/galaxy/web/framework/middleware/error.py | 2 +- .../web/framework/middleware/profile.py | 2 +- lib/galaxy/web/framework/middleware/sentry.py | 2 +- lib/galaxy/webapps/__init__.py | 2 +- .../demo_sequencer/controllers/__init__.py | 2 +- .../demo_sequencer/controllers/common.py | 8 +- .../demo_sequencer/framework/__init__.py | 4 +- .../galaxy/api/tool_shed_repositories.py | 8 +- lib/galaxy/webapps/galaxy/api/tools.py | 4 +- .../webapps/galaxy/api/visualizations.py | 2 +- .../webapps/galaxy/controllers/__init__.py | 2 +- .../galaxy/controllers/admin_toolshed.py | 14 +- .../webapps/galaxy/controllers/cloudlaunch.py | 6 +- .../galaxy/controllers/data_manager.py | 4 +- .../webapps/galaxy/controllers/dataset.py | 4 +- .../webapps/galaxy/controllers/error.py | 2 +- .../galaxy/controllers/library_common.py | 10 +- lib/galaxy/webapps/galaxy/controllers/page.py | 4 +- lib/galaxy/webapps/galaxy/controllers/root.py | 2 +- lib/galaxy/webapps/galaxy/controllers/user.py | 10 +- .../galaxy/controllers/visualization.py | 8 +- .../webapps/reports/controllers/__init__.py | 2 +- .../webapps/tool_shed/controllers/__init__.py | 2 +- .../tool_shed/controllers/repository.py | 22 ++-- .../webapps/tool_shed/framework/__init__.py | 2 +- .../framework/middleware/__init__.py | 2 +- lib/galaxy/workflow/modules.py | 12 +- lib/tool_shed/grids/repository_grids.py | 2 +- .../api/get_filtered_repository_revisions.py | 2 +- .../tool_shed_repository_revision_update.py | 6 +- .../bootstrap_tool_shed/bootstrap_util.py | 2 +- lib/tool_shed/util/readme_util.py | 2 +- lib/tool_shed/util/workflow_util.py | 2 +- 107 files changed, 1257 insertions(+), 246 deletions(-) mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0001_add_tool_shed_repository_table.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0002_add_tool_shed_repository_table_columns.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0003_tool_id_guid_map_table.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0004_add_installed_changeset_revison_column.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0005_add_tool_shed_repository_table_columns.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0006_add_tool_version_tables.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0007_add_migrate_tools_table.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0008_add_ctx_rev_column.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0009_add_tool_dependency_table.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0010_alter_tool_dependency_table_version_column.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0011_drop_installed_changeset_revision_column.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0012_add_tool_dependency_status_columns.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0013_add_tool_shed_repository_status_columns.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0014_add_repository_dependency_tables.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0015_update_migrate_tools_table.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0016_update_migrate_tools_table_again.py mode change 120000 => 100755 lib/galaxy/model/tool_shed_install/migrate/versions/0017_drop_update_available_col_add_tool_shed_status_col.py diff --git a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py index b20a1d889a5..c2bb35760a5 100644 --- a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py +++ b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py @@ -52,4 +52,4 @@ def main(): FileArrayTreeDict.dict_to_file( d, open( out_fname, "w" ) ) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/lib/galaxy/datatypes/converters/fasta_to_len.py b/lib/galaxy/datatypes/converters/fasta_to_len.py index 278a3449e5d..e4327e03574 100644 --- a/lib/galaxy/datatypes/converters/fasta_to_len.py +++ b/lib/galaxy/datatypes/converters/fasta_to_len.py @@ -49,4 +49,4 @@ def compute_fasta_length( fasta_file, out_file, keep_first_char, keep_first_word out.close() if __name__ == "__main__" : - compute_fasta_length( sys.argv[1], sys.argv[2], sys.argv[3], True ) \ No newline at end of file + compute_fasta_length( sys.argv[1], sys.argv[2], sys.argv[3], True ) diff --git a/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py b/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py index bbe05e3a7ca..13453672c52 100644 --- a/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py +++ b/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py @@ -40,4 +40,4 @@ def __main__(): print >> out, "%s\t%s" %( fasta_title, sequence ) out.close() -if __name__ == "__main__" : __main__() \ No newline at end of file +if __name__ == "__main__" : __main__() diff --git a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py index b5d82ba81fb..61269171817 100644 --- a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py +++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py @@ -52,4 +52,4 @@ def __main__(): outfile.close() -if __name__ == "__main__": __main__() \ No newline at end of file +if __name__ == "__main__": __main__() diff --git a/lib/galaxy/datatypes/converters/interval_to_coverage.py b/lib/galaxy/datatypes/converters/interval_to_coverage.py index 07093be0adc..64868ff019a 100644 --- a/lib/galaxy/datatypes/converters/interval_to_coverage.py +++ b/lib/galaxy/datatypes/converters/interval_to_coverage.py @@ -149,4 +149,4 @@ if __name__ == "__main__": fix_strand=True ) main( interval, coverage ) temp_file.close() - coverage.close() \ No newline at end of file + coverage.close() diff --git a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py index de13538cdcb..336c0171371 100644 --- a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py +++ b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py @@ -26,4 +26,4 @@ def main(): FileArrayTreeDict.dict_to_file( d, open( out_fname, "w" ) ) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index b8a766e3d8e..5c2c3dce5f8 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -14,7 +14,7 @@ from galaxy.datatypes import metadata from galaxy.datatypes.checkers import is_gzip from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.sniff import get_headers, get_test_fname -from galaxy.util.json import to_json_string +from galaxy.util.json import dumps import dataproviders log = logging.getLogger(__name__) @@ -272,7 +272,7 @@ class Tabular( data.Text ): while cursor and ck_data[-1] != '\n': ck_data += cursor cursor = f.read(1) - return to_json_string( { 'ck_data': util.unicodify( ck_data ), 'ck_index': ck_index + 1 } ) + return dumps( { 'ck_data': util.unicodify( ck_data ), 'ck_index': ck_index + 1 } ) def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None, **kwd): preview = util.string_as_bool( preview ) diff --git a/lib/galaxy/datatypes/util/__init__.py b/lib/galaxy/datatypes/util/__init__.py index 871a0debf19..e498fa06602 100644 --- a/lib/galaxy/datatypes/util/__init__.py +++ b/lib/galaxy/datatypes/util/__init__.py @@ -1,3 +1,3 @@ """ Utilities for Galaxy datatypes. -""" \ No newline at end of file +""" diff --git a/lib/galaxy/external_services/result_handlers/basic.py b/lib/galaxy/external_services/result_handlers/basic.py index 7bda4857835..bab5c6d359f 100644 --- a/lib/galaxy/external_services/result_handlers/basic.py +++ b/lib/galaxy/external_services/result_handlers/basic.py @@ -1,4 +1,4 @@ -from galaxy.util.json import to_json_string, from_json_string +from galaxy.util.json import dumps, loads from galaxy.util.template import fill_template import logging @@ -44,7 +44,7 @@ class ExternalServiceActionJSONResultHandler( ExternalServiceActionResultHandler type = 'json_display' def handle_result( self, result, param_dict, trans ): - rval = from_json_string( result.content ) + rval = loads( result.content ) return trans.fill_template( '/external_services/generic_json.mako', result = rval, param_dict = param_dict, action=self.parent ) class ExternalServiceActionJQueryGridResultHandler( ExternalServiceActionResultHandler ): @@ -53,7 +53,7 @@ class ExternalServiceActionJQueryGridResultHandler( ExternalServiceActionResultH type = 'jquery_grid' def handle_result( self, result, param_dict, trans ): - rval = from_json_string( result.content ) + rval = loads( result.content ) return trans.fill_template( '/external_services/generic_jquery_grid.mako', result = rval, param_dict = param_dict, action=self.parent ) result_type_to_class = {} diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index d6f6402d5d9..a84bf27dd8e 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -25,7 +25,7 @@ from galaxy.jobs.mapper import JobRunnerMapper from galaxy.jobs.runners import BaseJobRunner, JobState from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext -from galaxy.util.json import from_json_string +from galaxy.util.json import loads from galaxy.util import unicodify from .output_checker import check_output @@ -723,7 +723,7 @@ class JobWrapper( object ): self.job_runner_mapper = JobRunnerMapper( self, queue.dispatcher.url_to_destination, self.app.job_config ) self.params = None if job.params: - self.params = from_json_string( job.params ) + self.params = loads( job.params ) if use_persisted_destination: self.job_runner_mapper.cached_job_destination = JobDestination( from_job=job ) @@ -1390,7 +1390,7 @@ class JobWrapper( object ): if os.path.exists( meta_file ): for line in open( meta_file, 'r' ): try: - line = from_json_string( line ) + line = loads( line ) assert 'type' in line except: log.exception( '(%s) Got JSON data from tool, but data is improperly formatted or no "type" key in data' % self.job_id ) diff --git a/lib/galaxy/jobs/actions/__init__.py b/lib/galaxy/jobs/actions/__init__.py index 9750741c2a6..afde1ac53e3 100644 --- a/lib/galaxy/jobs/actions/__init__.py +++ b/lib/galaxy/jobs/actions/__init__.py @@ -1,4 +1,4 @@ """ This package contains job action classes. -""" \ No newline at end of file +""" diff --git a/lib/galaxy/jobs/actions/post.py b/lib/galaxy/jobs/actions/post.py index d223b06d509..ac02112c909 100644 --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -5,7 +5,7 @@ Actions to be run at job completion. Currently only used in workflows. import datetime import logging from galaxy.util import send_mail -from galaxy.util.json import to_json_string +from galaxy.util.json import dumps log = logging.getLogger( __name__ ) @@ -461,7 +461,7 @@ class ActionBox(object): else: # Not pja stuff. pass - return to_json_string(npd) + return dumps(npd) @classmethod def get_add_list(cls): diff --git a/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py b/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py index d64b2219cff..ead0fb2575d 100644 --- a/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py +++ b/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py @@ -83,7 +83,7 @@ class SMRTPortalPlugin( DataTransfer ): return self.job_states.INVALID url = 'http://' + job.params[ 'smrt_host' ] + self.api_path + '/Jobs/' + job.params[ 'smrt_job_id' ] + '/Status' r = urllib2.urlopen( url ) - status = json.from_json_string( r.read() ) + status = json.loads( r.read() ) # TODO: error handling: unexpected json or bad response, bad url, etc. if status[ 'Code' ] == 'Completed': log.debug( "SMRT Portal job '%s' is Completed. Initiating transfer." % job.params[ 'smrt_job_id' ] ) diff --git a/lib/galaxy/jobs/transfer_manager.py b/lib/galaxy/jobs/transfer_manager.py index 030e911ae42..3e3a4af4b7c 100644 --- a/lib/galaxy/jobs/transfer_manager.py +++ b/lib/galaxy/jobs/transfer_manager.py @@ -82,7 +82,7 @@ class TransferManager( object ): sock = socket.socket( socket.AF_INET, socket.SOCK_STREAM ) sock.settimeout( 5 ) sock.connect( ( 'localhost', tj.socket ) ) - sock.send( json.to_json_string( request ) ) + sock.send( json.dumps( request ) ) response = sock.recv( 8192 ) valid, response = json.validate_jsonrpc_response( response, id=request['id'] ) if not valid: diff --git a/lib/galaxy/managers/context.py b/lib/galaxy/managers/context.py index 00f583f2925..39471fae855 100644 --- a/lib/galaxy/managers/context.py +++ b/lib/galaxy/managers/context.py @@ -4,7 +4,7 @@ Mixins for transaction-like objects. import os -from galaxy.util.json import to_json_string +from galaxy.util.json import dumps from galaxy.util import bunch class ProvidesAppContext( object ): @@ -19,7 +19,7 @@ class ProvidesAppContext( object ): Application-level logging of user actions. """ if self.app.config.log_actions: - action = self.app.model.UserAction(action=action, context=context, params=unicode( to_json_string( params ) ) ) + action = self.app.model.UserAction(action=action, context=context, params=unicode( dumps( params ) ) ) try: if user: action.user = user diff --git a/lib/galaxy/model/migrate/versions/0052_sample_dataset_table.py b/lib/galaxy/model/migrate/versions/0052_sample_dataset_table.py index 38d5dc8a469..33a18b1520a 100644 --- a/lib/galaxy/model/migrate/versions/0052_sample_dataset_table.py +++ b/lib/galaxy/model/migrate/versions/0052_sample_dataset_table.py @@ -10,7 +10,7 @@ from migrate.changeset import * from sqlalchemy.exc import * from galaxy.model.custom_types import * -from galaxy.util.json import from_json_string, to_json_string +from galaxy.util.json import loads, dumps import datetime now = datetime.datetime.utcnow @@ -62,7 +62,7 @@ def upgrade(migrate_engine): for r in result: sample_id = r[0] if r[1]: - dataset_files = from_json_string(r[1]) + dataset_files = loads(r[1]) for df in dataset_files: if type(df) == type(dict()): cmd = "INSERT INTO sample_dataset VALUES (%s, %s, %s, %s, '%s', '%s', '%s', '%s', '%s')" diff --git a/lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py b/lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py index e879d141e7a..9cac2d61df5 100644 --- a/lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py +++ b/lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py @@ -6,7 +6,7 @@ from sqlalchemy import * from sqlalchemy.orm import * from migrate import * from migrate.changeset import * -from galaxy.util.json import from_json_string +from galaxy.util.json import loads import logging log = logging.getLogger( __name__ ) @@ -41,7 +41,7 @@ def upgrade(migrate_engine): viz_id = viz['viz_id'] viz_rev_id = viz['viz_rev_id'] if viz[Visualization_revision_table.c.config]: - dbkey = from_json_string(viz[Visualization_revision_table.c.config]).get('dbkey', "").replace("'", "\\'") + dbkey = loads(viz[Visualization_revision_table.c.config]).get('dbkey', "").replace("'", "\\'") migrate_engine.execute("UPDATE visualization_revision SET dbkey='%s' WHERE id=%s" % (dbkey, viz_rev_id)) migrate_engine.execute("UPDATE visualization SET dbkey='%s' WHERE id=%s" % (dbkey, viz_id)) diff --git a/lib/galaxy/model/migrate/versions/0057_request_notify.py b/lib/galaxy/model/migrate/versions/0057_request_notify.py index bb38ae22ae6..e763f524d96 100644 --- a/lib/galaxy/model/migrate/versions/0057_request_notify.py +++ b/lib/galaxy/model/migrate/versions/0057_request_notify.py @@ -10,7 +10,7 @@ from migrate.changeset import * from sqlalchemy.exc import * from galaxy.model.custom_types import * -from galaxy.util.json import from_json_string, to_json_string +from galaxy.util.json import loads, dumps import datetime now = datetime.datetime.utcnow @@ -46,13 +46,13 @@ def upgrade(migrate_engine): id = int(r[0]) notify_old = r[1] notify_new = dict(email=[], sample_states=[], body='', subject='') - cmd = "update request set notification='%s' where id=%i" % (to_json_string(notify_new), id) + cmd = "update request set notification='%s' where id=%i" % (dumps(notify_new), id) migrate_engine.execute( cmd ) cmd = "SELECT id, notification FROM request" result = migrate_engine.execute( cmd ) for r in result: - rr = from_json_string(str(r[1])) + rr = loads(str(r[1])) # remove the 'notify' column for non-sqlite databases. if migrate_engine.name != 'sqlite': diff --git a/lib/galaxy/model/migrate/versions/0059_sample_dataset_file_path.py b/lib/galaxy/model/migrate/versions/0059_sample_dataset_file_path.py index 3ffca0455eb..53ee2ef8840 100644 --- a/lib/galaxy/model/migrate/versions/0059_sample_dataset_file_path.py +++ b/lib/galaxy/model/migrate/versions/0059_sample_dataset_file_path.py @@ -10,7 +10,7 @@ from migrate.changeset import * from sqlalchemy.exc import * from galaxy.model.custom_types import * -from galaxy.util.json import from_json_string, to_json_string +from galaxy.util.json import loads, dumps import datetime now = datetime.datetime.utcnow diff --git a/lib/galaxy/model/migrate/versions/0065_add_name_to_form_fields_and_values.py b/lib/galaxy/model/migrate/versions/0065_add_name_to_form_fields_and_values.py index 1da427152ca..2776d859fd1 100644 --- a/lib/galaxy/model/migrate/versions/0065_add_name_to_form_fields_and_values.py +++ b/lib/galaxy/model/migrate/versions/0065_add_name_to_form_fields_and_values.py @@ -9,7 +9,7 @@ from sqlalchemy.orm import * from migrate import * from migrate.changeset import * from sqlalchemy.exc import * -from galaxy.util.json import from_json_string, to_json_string +from galaxy.util.json import loads, dumps from galaxy.model.custom_types import _sniffnfix_pg9_hex import datetime @@ -53,13 +53,13 @@ def upgrade(migrate_engine): fields = str( row[1] ) if not fields.strip(): continue - fields_list = from_json_string( _sniffnfix_pg9_hex( fields ) ) + fields_list = loads( _sniffnfix_pg9_hex( fields ) ) if len( fields_list ): for index, field in enumerate( fields_list ): field[ 'name' ] = 'field_%i' % index field[ 'helptext' ] = field[ 'helptext' ].replace("'", "''").replace('"', "") field[ 'label' ] = field[ 'label' ].replace("'", "''") - fields_json = to_json_string( fields_list ) + fields_json = dumps( fields_list ) if migrate_engine.name == 'mysql': cmd = "UPDATE form_definition AS f SET f.fields='%s' WHERE f.id=%i" %( fields_json, form_definition_id ) else: @@ -76,16 +76,16 @@ def upgrade(migrate_engine): if not str( row[1] ).strip(): continue row1 = str(row[1]).replace('\n', '').replace('\r', '') - values_list = from_json_string( str( row1 ).strip() ) + values_list = loads( str( row1 ).strip() ) if not str( row[2] ).strip(): continue - fields_list = from_json_string( str( row[2] ).strip() ) + fields_list = loads( str( row[2] ).strip() ) if fields_list and type(values_list) == type(list()): values_dict = {} for field_index, field in enumerate( fields_list ): field_name = field[ 'name' ] values_dict[ field_name ] = get_value(values_list, field_index ) - cmd = "UPDATE form_values SET content='%s' WHERE id=%i" %( to_json_string( values_dict ), form_values_id ) + cmd = "UPDATE form_values SET content='%s' WHERE id=%i" %( dumps( values_dict ), form_values_id ) migrate_engine.execute( cmd ) def downgrade(migrate_engine): @@ -110,17 +110,17 @@ def downgrade(migrate_engine): form_values_id = int( row[0] ) if not str( row[1] ).strip(): continue - values_dict = from_json_string( str( row[1] ) ) + values_dict = loads( str( row[1] ) ) if not str( row[2] ).strip(): continue - fields_list = from_json_string( str( row[2] ) ) + fields_list = loads( str( row[2] ) ) if fields_list: values_list = [] for field_index, field in enumerate( fields_list ): field_name = field[ 'name' ] field_value = values_dict[ field_name ] values_list.append( field_value ) - cmd = "UPDATE form_values SET content='%s' WHERE id=%i" %( to_json_string( values_list ), form_values_id ) + cmd = "UPDATE form_values SET content='%s' WHERE id=%i" %( dumps( values_list ), form_values_id ) migrate_engine.execute( cmd ) # remove name attribute from the field column of the form_definition table cmd = "SELECT f.id, f.fields FROM form_definition AS f" @@ -130,13 +130,13 @@ def downgrade(migrate_engine): fields = str( row[1] ) if not fields.strip(): continue - fields_list = from_json_string( _sniffnfix_pg9_hex( fields ) ) + fields_list = loads( _sniffnfix_pg9_hex( fields ) ) if len( fields_list ): for index, field in enumerate( fields_list ): if field.has_key( 'name' ): del field[ 'name' ] if migrate_engine.name == 'mysql': - cmd = "UPDATE form_definition AS f SET f.fields='%s' WHERE f.id=%i" %( to_json_string( fields_list ), form_definition_id ) + cmd = "UPDATE form_definition AS f SET f.fields='%s' WHERE f.id=%i" %( dumps( fields_list ), form_definition_id ) else: - cmd = "UPDATE form_definition SET fields='%s' WHERE id=%i" %( to_json_string( fields_list ), form_definition_id ) + cmd = "UPDATE form_definition SET fields='%s' WHERE id=%i" %( dumps( fields_list ), form_definition_id ) migrate_engine.execute( cmd ) diff --git a/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py b/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py index ff0d7513f12..3c790330a24 100644 --- a/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py +++ b/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py @@ -13,7 +13,7 @@ from sqlalchemy.exc import * from galaxy.model.custom_types import * -from galaxy.util.json import from_json_string, to_json_string +from galaxy.util.json import loads, dumps import datetime now = datetime.datetime.utcnow @@ -104,7 +104,7 @@ def create_sequencer_form_definition(): 'layout': 'none', 'default': '' } ) form_definition_type = 'Sequencer Information Form' - form_definition_layout = to_json_string('[]') + form_definition_layout = dumps('[]') cmd = "INSERT INTO form_definition VALUES ( %s, %s, %s, '%s', '%s', %s, '%s', '%s', '%s' )" cmd = cmd % ( nextval( 'form_definition' ), localtimestamp(), @@ -112,7 +112,7 @@ def create_sequencer_form_definition(): form_definition_name, form_definition_desc, form_definition_current_id, - to_json_string( form_definition_fields ), + dumps( form_definition_fields ), form_definition_type, form_definition_layout ) migrate_engine.execute( cmd ) @@ -134,7 +134,7 @@ def get_sequencer_id( sequencer_info ): values = str( row[1] ) if not values.strip(): continue - values = from_json_string( values ) + values = loads( values ) # proceed only if sequencer_info is a valid list if values and type( values ) == type( dict() ): if sequencer_info.get( 'host', '' ) == values.get( 'field_0', '' ) \ @@ -148,7 +148,7 @@ def get_sequencer_id( sequencer_info ): def add_sequencer( sequencer_index, sequencer_form_definition_id, sequencer_info ): '''Adds a new sequencer to the sequencer table along with its form values.''' # Create a new form values record with the supplied sequencer information - values = to_json_string( { 'field_0': sequencer_info.get( 'host', '' ), + values = dumps( { 'field_0': sequencer_info.get( 'host', '' ), 'field_1': sequencer_info.get( 'username', '' ), 'field_2': sequencer_info.get( 'password', '' ), 'field_3': sequencer_info.get( 'data_dir', '' ), @@ -233,7 +233,7 @@ def upgrade(migrate_engine): # skip if sequencer_info is empty if not sequencer_info.strip() or sequencer_info in ['None', 'null']: continue - sequencer_info = from_json_string( sequencer_info.strip() ) + sequencer_info = loads( sequencer_info.strip() ) # proceed only if sequencer_info is a valid dict if sequencer_info and type( sequencer_info ) == type( dict() ): # check if this sequencer has already been added to the sequencer table @@ -277,9 +277,9 @@ def downgrade(migrate_engine): result = migrate_engine.execute( cmd ) for row in result: request_type_id = row[0] - seq_values = from_json_string( str( row[1] ) ) + seq_values = loads( str( row[1] ) ) # create the datatx_info json dict - datatx_info = to_json_string( dict( host = seq_values.get( 'field_0', '' ), + datatx_info = dumps( dict( host = seq_values.get( 'field_0', '' ), username = seq_values.get( 'field_1', '' ), password = seq_values.get( 'field_2', '' ), data_dir = seq_values.get( 'field_3', '' ), diff --git a/lib/galaxy/model/migrate/versions/0068_rename_sequencer_to_external_services.py b/lib/galaxy/model/migrate/versions/0068_rename_sequencer_to_external_services.py index 282b4ea8854..ec05e516239 100644 --- a/lib/galaxy/model/migrate/versions/0068_rename_sequencer_to_external_services.py +++ b/lib/galaxy/model/migrate/versions/0068_rename_sequencer_to_external_services.py @@ -15,7 +15,7 @@ from sqlalchemy.exc import * from galaxy.model.custom_types import * -from galaxy.util.json import from_json_string, to_json_string +from galaxy.util.json import loads, dumps import datetime now = datetime.datetime.utcnow diff --git a/lib/galaxy/model/migrate/versions/0076_fix_form_values_data_corruption.py b/lib/galaxy/model/migrate/versions/0076_fix_form_values_data_corruption.py index 622781195eb..84202ca2a4e 100644 --- a/lib/galaxy/model/migrate/versions/0076_fix_form_values_data_corruption.py +++ b/lib/galaxy/model/migrate/versions/0076_fix_form_values_data_corruption.py @@ -9,7 +9,7 @@ from migrate.changeset import * from sqlalchemy.exc import * import binascii -from galaxy.util.json import from_json_string, to_json_string +from galaxy.util.json import loads, dumps import logging log = logging.getLogger( __name__ ) @@ -42,11 +42,11 @@ def upgrade(migrate_engine): # first check if loading the dict from the json succeeds # if that fails, it means that the content field is corrupted. try: - field_values_dict = from_json_string( _sniffnfix_pg9_hex( str( row['field_values'] ) ) ) + field_values_dict = loads( _sniffnfix_pg9_hex( str( row['field_values'] ) ) ) except Exception, e: corrupted_rows = corrupted_rows + 1 # content field is corrupted - fields_list = from_json_string( _sniffnfix_pg9_hex( str( row['fdfields'] ) ) ) + fields_list = loads( _sniffnfix_pg9_hex( str( row['fdfields'] ) ) ) field_values_str = _sniffnfix_pg9_hex( str( row['field_values'] ) ) try: #Encoding errors? Just to be safe. @@ -85,7 +85,7 @@ def upgrade(migrate_engine): # add to the new values dict field_values_dict[ field['name'] ] = value # update the db - json_values = to_json_string(field_values_dict) + json_values = dumps(field_values_dict) cmd = "UPDATE form_values SET content='%s' WHERE id=%i" %( json_values, int( row['id'] ) ) migrate_engine.execute( cmd ) try: diff --git a/lib/galaxy/model/migrate/versions/0091_add_tool_version_tables.py b/lib/galaxy/model/migrate/versions/0091_add_tool_version_tables.py index 434955a32d2..5d43fdc509a 100644 --- a/lib/galaxy/model/migrate/versions/0091_add_tool_version_tables.py +++ b/lib/galaxy/model/migrate/versions/0091_add_tool_version_tables.py @@ -12,7 +12,7 @@ now = datetime.datetime.utcnow # Need our custom types, but don't import anything else from model from galaxy.model.custom_types import * from galaxy.model.custom_types import _sniffnfix_pg9_hex -from galaxy.util.json import from_json_string, to_json_string +from galaxy.util.json import loads, dumps import sys, logging log = logging.getLogger( __name__ ) @@ -77,7 +77,7 @@ def upgrade(migrate_engine): for row in result: if row[1]: tool_shed_repository_id = row[0] - repository_metadata = from_json_string( _sniffnfix_pg9_hex( str( row[1] ) ) ) + repository_metadata = loads( _sniffnfix_pg9_hex( str( row[1] ) ) ) # Create a new row in the tool table for each tool included in repository. We will NOT # handle tool_version_associaions because we do not have the information we need to do so. tools = repository_metadata.get( 'tools', [] ) diff --git a/lib/galaxy/model/migrate/versions/0115_longer_user_password_field.py b/lib/galaxy/model/migrate/versions/0115_longer_user_password_field.py index 9d868398804..0efb653eb7c 100644 --- a/lib/galaxy/model/migrate/versions/0115_longer_user_password_field.py +++ b/lib/galaxy/model/migrate/versions/0115_longer_user_password_field.py @@ -22,4 +22,4 @@ def downgrade(migrate_engine): try: user.c.password.alter(type=String(40)) except: - log.exception( "Altering password column failed" ) \ No newline at end of file + log.exception( "Altering password column failed" ) diff --git a/lib/galaxy/model/migrate/versions/0117_add_user_activation.py b/lib/galaxy/model/migrate/versions/0117_add_user_activation.py index 6cbfc8fa250..eefd3600aef 100644 --- a/lib/galaxy/model/migrate/versions/0117_add_user_activation.py +++ b/lib/galaxy/model/migrate/versions/0117_add_user_activation.py @@ -54,4 +54,4 @@ def downgrade(migrate_engine): user_active.drop() user_activation_token.drop() except Exception, e: - log.debug( "Dropping 'active' and 'activation_token' columns from galaxy_user table failed: %s" % ( str( e ) ) ) \ No newline at end of file + log.debug( "Dropping 'active' and 'activation_token' columns from galaxy_user table failed: %s" % ( str( e ) ) ) diff --git a/lib/galaxy/model/orm/logging_connection_proxy.py b/lib/galaxy/model/orm/logging_connection_proxy.py index 5ed981f0868..4356721c7e9 100644 --- a/lib/galaxy/model/orm/logging_connection_proxy.py +++ b/lib/galaxy/model/orm/logging_connection_proxy.py @@ -45,4 +45,4 @@ class TraceLoggerProxy(ConnectionProxy): self.trace_logger.log( "sqlalchemy_query", message="Query executed", statement=statement, parameters=parameters, executemany=executemany, duration=duration ) - return rval \ No newline at end of file + return rval diff --git a/lib/galaxy/model/search.py b/lib/galaxy/model/search.py index 5933252c458..896c6b29b36 100644 --- a/lib/galaxy/model/search.py +++ b/lib/galaxy/model/search.py @@ -39,7 +39,7 @@ JobToInputLibraryDatasetAssociation, JobToInputDatasetAssociation, JobToOutputDa Page, PageRevision) from galaxy.model.tool_shed_install import ToolVersion -from galaxy.util.json import to_json_string +from galaxy.util.json import dumps from sqlalchemy import and_ from sqlalchemy.orm import aliased @@ -429,7 +429,7 @@ def job_param_filter(view, left, operator, right): and_( Job.id == alias.job_id, alias.name == param_name, - alias.value == to_json_string(right) + alias.value == dumps(right) ) ) diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0001_add_tool_shed_repository_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0001_add_tool_shed_repository_table.py deleted file mode 120000 index e0b183100f2..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0001_add_tool_shed_repository_table.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0082_add_tool_shed_repository_table.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0001_add_tool_shed_repository_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0001_add_tool_shed_repository_table.py new file mode 100755 index 00000000000..a6a0db00c47 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0001_add_tool_shed_repository_table.py @@ -0,0 +1,51 @@ +""" +Migration script to add the tool_shed_repository table. +""" +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +import sys, logging +from galaxy.model.custom_types import * +from sqlalchemy.exc import * +import datetime +now = datetime.datetime.utcnow + +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() + +# New table to store information about cloned tool shed repositories. +ToolShedRepository_table = Table( "tool_shed_repository", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed", TrimmedString( 255 ), index=True ), + Column( "name", TrimmedString( 255 ), index=True ), + Column( "description" , TEXT ), + Column( "owner", TrimmedString( 255 ), index=True ), + Column( "changeset_revision", TrimmedString( 255 ), index=True ), + Column( "deleted", Boolean, index=True, default=False ) ) + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + try: + ToolShedRepository_table.create() + except Exception, e: + log.debug( "Creating tool_shed_repository table failed: %s" % str( e ) ) + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + try: + ToolShedRepository_table.drop() + except Exception, e: + log.debug( "Dropping tool_shed_repository table failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0002_add_tool_shed_repository_table_columns.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0002_add_tool_shed_repository_table_columns.py deleted file mode 120000 index 7cacf92509e..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0002_add_tool_shed_repository_table_columns.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0086_add_tool_shed_repository_table_columns.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0002_add_tool_shed_repository_table_columns.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0002_add_tool_shed_repository_table_columns.py new file mode 100755 index 00000000000..828bacd910b --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0002_add_tool_shed_repository_table_columns.py @@ -0,0 +1,79 @@ +""" +Migration script to add the metadata, update_available and includes_datatypes columns to the tool_shed_repository table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() + +def get_default_false(migrate_engine): + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + return "0" + elif migrate_engine.name in ['postgresql', 'postgres']: + return "false" + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + c = Column( "metadata", JSONType(), nullable=True ) + try: + c.create( ToolShedRepository_table ) + assert c is ToolShedRepository_table.c.metadata + except Exception, e: + print "Adding metadata column to the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Adding metadata column to the tool_shed_repository table failed: %s" % str( e ) ) + c = Column( "includes_datatypes", Boolean, index=True, default=False ) + try: + c.create( ToolShedRepository_table, index_name="ix_tool_shed_repository_includes_datatypes") + assert c is ToolShedRepository_table.c.includes_datatypes + migrate_engine.execute( "UPDATE tool_shed_repository SET includes_datatypes=%s" % get_default_false(migrate_engine)) + except Exception, e: + print "Adding includes_datatypes column to the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Adding includes_datatypes column to the tool_shed_repository table failed: %s" % str( e ) ) + c = Column( "update_available", Boolean, default=False ) + try: + c.create( ToolShedRepository_table ) + assert c is ToolShedRepository_table.c.update_available + migrate_engine.execute( "UPDATE tool_shed_repository SET update_available=%s" % get_default_false(migrate_engine)) + except Exception, e: + print "Adding update_available column to the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Adding update_available column to the tool_shed_repository table failed: %s" % str( e ) ) + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + try: + ToolShedRepository_table.c.metadata.drop() + except Exception, e: + print "Dropping column metadata from the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Dropping column metadata from the tool_shed_repository table failed: %s" % str( e ) ) + try: + ToolShedRepository_table.c.includes_datatypes.drop() + except Exception, e: + print "Dropping column includes_datatypes from the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Dropping column includes_datatypes from the tool_shed_repository table failed: %s" % str( e ) ) + try: + ToolShedRepository_table.c.update_available.drop() + except Exception, e: + print "Dropping column update_available from the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Dropping column update_available from the tool_shed_repository table failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0003_tool_id_guid_map_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0003_tool_id_guid_map_table.py deleted file mode 120000 index 3e9dea9c20a..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0003_tool_id_guid_map_table.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0087_tool_id_guid_map_table.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0003_tool_id_guid_map_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0003_tool_id_guid_map_table.py new file mode 100755 index 00000000000..caa3b4eb74c --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0003_tool_id_guid_map_table.py @@ -0,0 +1,52 @@ +""" +Migration script to create the tool_id_guid_map table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() + +ToolIdGuidMap_table = Table( "tool_id_guid_map", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_id", String( 255 ) ), + Column( "tool_version", TEXT ), + Column( "tool_shed", TrimmedString( 255 ) ), + Column( "repository_owner", TrimmedString( 255 ) ), + Column( "repository_name", TrimmedString( 255 ) ), + Column( "guid", TEXT, index=True, unique=True ) ) + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + try: + ToolIdGuidMap_table.create() + except Exception, e: + log.debug( "Creating tool_id_guid_map table failed: %s" % str( e ) ) + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + try: + ToolIdGuidMap_table.drop() + except Exception, e: + log.debug( "Dropping tool_id_guid_map table failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0004_add_installed_changeset_revison_column.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0004_add_installed_changeset_revison_column.py deleted file mode 120000 index 4e57eb188b8..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0004_add_installed_changeset_revison_column.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0088_add_installed_changeset_revison_column.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0004_add_installed_changeset_revison_column.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0004_add_installed_changeset_revison_column.py new file mode 100755 index 00000000000..eff95ca0076 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0004_add_installed_changeset_revison_column.py @@ -0,0 +1,64 @@ +""" +Migration script to add the installed_changeset_revision column to the tool_shed_repository table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + col = Column( "installed_changeset_revision", TrimmedString( 255 ) ) + try: + col.create( ToolShedRepository_table ) + assert col is ToolShedRepository_table.c.installed_changeset_revision + except Exception, e: + print "Adding installed_changeset_revision column to the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Adding installed_changeset_revision column to the tool_shed_repository table failed: %s" % str( e ) ) + # Update each row by setting the value of installed_changeset_revison to be the value of changeset_revision. + # This will be problematic if the value of changeset_revision was updated to something other than the value + # that it was when the repository was installed (because the install path determined in real time will attempt to + # find the repository using the updated changeset_revison instead of the required installed_changeset_revision), + # but at the time this script was written, this scenario is extremely unlikely. + cmd = "SELECT id AS id, " \ + + "installed_changeset_revision AS installed_changeset_revision, " \ + + "changeset_revision AS changeset_revision " \ + + "FROM tool_shed_repository;" + tool_shed_repositories = migrate_engine.execute( cmd ).fetchall() + update_count = 0 + for row in tool_shed_repositories: + cmd = "UPDATE tool_shed_repository " \ + + "SET installed_changeset_revision = '%s' " % row.changeset_revision \ + + "WHERE changeset_revision = '%s';" % row.changeset_revision + migrate_engine.execute( cmd ) + update_count += 1 + print "Updated the installed_changeset_revision column for ", update_count, " rows in the tool_shed_repository table. " +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + try: + ToolShedRepository_table.c.installed_changeset_revision.drop() + except Exception, e: + print "Dropping column installed_changeset_revision from the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Dropping column installed_changeset_revision from the tool_shed_repository table failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0005_add_tool_shed_repository_table_columns.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0005_add_tool_shed_repository_table_columns.py deleted file mode 120000 index 6359c3b9bbb..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0005_add_tool_shed_repository_table_columns.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0090_add_tool_shed_repository_table_columns.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0005_add_tool_shed_repository_table_columns.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0005_add_tool_shed_repository_table_columns.py new file mode 100755 index 00000000000..5f412cb5a40 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0005_add_tool_shed_repository_table_columns.py @@ -0,0 +1,63 @@ +""" +Migration script to add the uninstalled and dist_to_shed columns to the tool_shed_repository table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() + +def default_false(migrate_engine): + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + return "0" + elif migrate_engine.name in ['postgresql', 'postgres']: + return "false" + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + c = Column( "uninstalled", Boolean, default=False ) + try: + c.create( ToolShedRepository_table ) + assert c is ToolShedRepository_table.c.uninstalled + migrate_engine.execute( "UPDATE tool_shed_repository SET uninstalled=%s" % default_false(migrate_engine) ) + except Exception, e: + print "Adding uninstalled column to the tool_shed_repository table failed: %s" % str( e ) + c = Column( "dist_to_shed", Boolean, default=False ) + try: + c.create( ToolShedRepository_table ) + assert c is ToolShedRepository_table.c.dist_to_shed + migrate_engine.execute( "UPDATE tool_shed_repository SET dist_to_shed=%s" % default_false(migrate_engine) ) + except Exception, e: + print "Adding dist_to_shed column to the tool_shed_repository table failed: %s" % str( e ) + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + try: + ToolShedRepository_table.c.uninstalled.drop() + except Exception, e: + print "Dropping column uninstalled from the tool_shed_repository table failed: %s" % str( e ) + try: + ToolShedRepository_table.c.dist_to_shed.drop() + except Exception, e: + print "Dropping column dist_to_shed from the tool_shed_repository table failed: %s" % str( e ) diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0006_add_tool_version_tables.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0006_add_tool_version_tables.py deleted file mode 120000 index 9a4526d9b96..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0006_add_tool_version_tables.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0091_add_tool_version_tables.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0006_add_tool_version_tables.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0006_add_tool_version_tables.py new file mode 100755 index 00000000000..5d43fdc509a --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0006_add_tool_version_tables.py @@ -0,0 +1,122 @@ +""" +Migration script to create the tool_version and tool_version_association tables and drop the tool_id_guid_map table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * +from galaxy.model.custom_types import _sniffnfix_pg9_hex +from galaxy.util.json import loads, dumps + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() +#migrate_engine = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def nextval( table, col='id' ): + if migrate_engine.name == 'postgres': + return "nextval('%s_%s_seq')" % ( table, col ) + elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + return "null" + else: + raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) + +def localtimestamp(): + if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql': + return "LOCALTIMESTAMP" + elif migrate_engine.name == 'sqlite': + return "current_date || ' ' || current_time" + else: + raise Exception( 'Unable to convert data for unknown database type: %s' % db ) + +ToolVersion_table = Table( "tool_version", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_id", String( 255 ) ), + Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=True ) ) + +ToolVersionAssociation_table = Table( "tool_version_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "tool_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ), + Column( "parent_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ) ) + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + + ToolIdGuidMap_table = Table( "tool_id_guid_map", metadata, autoload=True ) + + metadata.reflect() + # Create the tables. + try: + ToolVersion_table.create() + except Exception, e: + log.debug( "Creating tool_version table failed: %s" % str( e ) ) + try: + ToolVersionAssociation_table.create() + except Exception, e: + log.debug( "Creating tool_version_association table failed: %s" % str( e ) ) + # Populate the tool table with tools included in installed tool shed repositories. + cmd = "SELECT id, metadata FROM tool_shed_repository" + result = migrate_engine.execute( cmd ) + count = 0 + for row in result: + if row[1]: + tool_shed_repository_id = row[0] + repository_metadata = loads( _sniffnfix_pg9_hex( str( row[1] ) ) ) + # Create a new row in the tool table for each tool included in repository. We will NOT + # handle tool_version_associaions because we do not have the information we need to do so. + tools = repository_metadata.get( 'tools', [] ) + for tool_dict in tools: + cmd = "INSERT INTO tool_version VALUES (%s, %s, %s, '%s', %s)" % \ + ( nextval( 'tool_version' ), localtimestamp(), localtimestamp(), tool_dict[ 'guid' ], tool_shed_repository_id ) + migrate_engine.execute( cmd ) + count += 1 + print "Added %d rows to the new tool_version table." % count + # Drop the tool_id_guid_map table since the 2 new tables render it unnecessary. + try: + ToolIdGuidMap_table.drop() + except Exception, e: + log.debug( "Dropping tool_id_guid_map table failed: %s" % str( e ) ) + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + + ToolIdGuidMap_table = Table( "tool_id_guid_map", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_id", String( 255 ) ), + Column( "tool_version", TEXT ), + Column( "tool_shed", TrimmedString( 255 ) ), + Column( "repository_owner", TrimmedString( 255 ) ), + Column( "repository_name", TrimmedString( 255 ) ), + Column( "guid", TEXT, index=True, unique=True ) ) + + metadata.reflect() + try: + ToolVersionAssociation_table.drop() + except Exception, e: + log.debug( "Dropping tool_version_association table failed: %s" % str( e ) ) + try: + ToolVersion_table.drop() + except Exception, e: + log.debug( "Dropping tool_version table failed: %s" % str( e ) ) + try: + ToolIdGuidMap_table.create() + except Exception, e: + log.debug( "Creating tool_id_guid_map table failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0007_add_migrate_tools_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0007_add_migrate_tools_table.py deleted file mode 120000 index 9fbb816a5cf..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0007_add_migrate_tools_table.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0092_add_migrate_tools_table.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0007_add_migrate_tools_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0007_add_migrate_tools_table.py new file mode 100755 index 00000000000..f323e2823fa --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0007_add_migrate_tools_table.py @@ -0,0 +1,50 @@ +""" +Migration script to create the migrate_tools table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() + +MigrateTools_table = Table( "migrate_tools", metadata, + Column( "repository_id", TrimmedString( 255 ) ), + Column( "repository_path", TEXT ), + Column( "version", Integer ) ) + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + + metadata.reflect() + # Create the table. + try: + MigrateTools_table.create() + cmd = "INSERT INTO migrate_tools VALUES ('GalaxyTools', 'lib/galaxy/tool_shed/migrate', %d)" % 1 + migrate_engine.execute( cmd ) + except Exception, e: + log.debug( "Creating migrate_tools table failed: %s" % str( e ) ) + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + try: + MigrateTools_table.drop() + except Exception, e: + log.debug( "Dropping migrate_tools table failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0008_add_ctx_rev_column.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0008_add_ctx_rev_column.py deleted file mode 120000 index a1268f1ff03..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0008_add_ctx_rev_column.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0097_add_ctx_rev_column.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0008_add_ctx_rev_column.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0008_add_ctx_rev_column.py new file mode 100755 index 00000000000..4d2acc5551a --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0008_add_ctx_rev_column.py @@ -0,0 +1,44 @@ +""" +Migration script to add the ctx_rev column to the tool_shed_repository table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + col = Column( "ctx_rev", TrimmedString( 10 ) ) + try: + col.create( ToolShedRepository_table ) + assert col is ToolShedRepository_table.c.ctx_rev + except Exception, e: + print "Adding ctx_rev column to the tool_shed_repository table failed: %s" % str( e ) +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + try: + ToolShedRepository_table.c.ctx_rev.drop() + except Exception, e: + print "Dropping column ctx_rev from the tool_shed_repository table failed: %s" % str( e ) diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0009_add_tool_dependency_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0009_add_tool_dependency_table.py deleted file mode 120000 index a119ddd378f..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0009_add_tool_dependency_table.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0099_add_tool_dependency_table.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0009_add_tool_dependency_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0009_add_tool_dependency_table.py new file mode 100755 index 00000000000..60c2581ba40 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0009_add_tool_dependency_table.py @@ -0,0 +1,51 @@ +""" +Migration script to add the tool_dependency table. +""" +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +import sys, logging +from galaxy.model.custom_types import * +from sqlalchemy.exc import * +import datetime +now = datetime.datetime.utcnow + +log = logging.getLogger( __name__ ) +log.setLevel( logging.DEBUG ) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() + +# New table to store information about cloned tool shed repositories. +ToolDependency_table = Table( "tool_dependency", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=False ), + Column( "installed_changeset_revision", TrimmedString( 255 ) ), + Column( "name", TrimmedString( 255 ) ), + Column( "version", TrimmedString( 40 ) ), + Column( "type", TrimmedString( 40 ) ), + Column( "uninstalled", Boolean, default=False ) ) + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + try: + ToolDependency_table.create() + except Exception, e: + log.debug( "Creating tool_dependency table failed: %s" % str( e ) ) + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + try: + ToolDependency_table.drop() + except Exception, e: + log.debug( "Dropping tool_dependency table failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0010_alter_tool_dependency_table_version_column.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0010_alter_tool_dependency_table_version_column.py deleted file mode 120000 index 1a39c5872a2..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0010_alter_tool_dependency_table_version_column.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0100_alter_tool_dependency_table_version_column.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0010_alter_tool_dependency_table_version_column.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0010_alter_tool_dependency_table_version_column.py new file mode 100755 index 00000000000..95554ea2791 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0010_alter_tool_dependency_table_version_column.py @@ -0,0 +1,54 @@ +""" +Migration script to alter the type of the tool_dependency.version column from TrimmedString(40) to Text. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + ToolDependency_table = Table( "tool_dependency", metadata, autoload=True ) + # Change the tool_dependency table's version column from TrimmedString to Text. + if migrate_engine.name in ['postgresql', 'postgres']: + cmd = "ALTER TABLE tool_dependency ALTER COLUMN version TYPE Text;" + elif migrate_engine.name == 'mysql': + cmd = "ALTER TABLE tool_dependency MODIFY COLUMN version Text;" + else: + # We don't have to do anything for sqlite tables. From the sqlite documentation at http://sqlite.org/datatype3.html: + # 1.0 Storage Classes and Datatypes + # Each value stored in an SQLite database (or manipulated by the database engine) has one of the following storage classes: + # NULL. The value is a NULL value. + # INTEGER. The value is a signed integer, stored in 1, 2, 3, 4, 6, or 8 bytes depending on the magnitude of the value. + # REAL. The value is a floating point value, stored as an 8-byte IEEE floating point number. + # TEXT. The value is a text string, stored using the database encoding (UTF-8, UTF-16BE or UTF-16LE). + # BLOB. The value is a blob of data, stored exactly as it was input. + cmd = None + if cmd: + try: + migrate_engine.execute( cmd ) + except Exception, e: + log.debug( "Altering tool_dependency.version column from TrimmedString(40) to Text failed: %s" % str( e ) ) +def downgrade(migrate_engine): + metadata.bind = migrate_engine + # Not necessary to change column type Text to TrimmedString(40). + pass diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0011_drop_installed_changeset_revision_column.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0011_drop_installed_changeset_revision_column.py deleted file mode 120000 index eed3a389c27..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0011_drop_installed_changeset_revision_column.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0101_drop_installed_changeset_revision_column.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0011_drop_installed_changeset_revision_column.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0011_drop_installed_changeset_revision_column.py new file mode 100755 index 00000000000..9fc373d4560 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0011_drop_installed_changeset_revision_column.py @@ -0,0 +1,41 @@ +""" +Migration script to drop the installed_changeset_revision column from the tool_dependency table. +""" +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +import sys, logging +from galaxy.model.custom_types import * +from sqlalchemy.exc import * +import datetime +now = datetime.datetime.utcnow + +log = logging.getLogger( __name__ ) +log.setLevel( logging.DEBUG ) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + try: + ToolDependency_table = Table( "tool_dependency", metadata, autoload=True ) + except NoSuchTableError: + ToolDependency_table = None + log.debug( "Failed loading table tool_dependency" ) + if ToolDependency_table is not None: + try: + col = ToolDependency_table.c.installed_changeset_revision + col.drop() + except Exception, e: + log.debug( "Dropping column 'installed_changeset_revision' from tool_dependency table failed: %s" % ( str( e ) ) ) +def downgrade(migrate_engine): + metadata.bind = migrate_engine + pass diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0012_add_tool_dependency_status_columns.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0012_add_tool_dependency_status_columns.py deleted file mode 120000 index c5759e0e88d..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0012_add_tool_dependency_status_columns.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0102_add_tool_dependency_status_columns.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0012_add_tool_dependency_status_columns.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0012_add_tool_dependency_status_columns.py new file mode 100755 index 00000000000..54eb23d2fb0 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0012_add_tool_dependency_status_columns.py @@ -0,0 +1,71 @@ +""" +Migration script to add status and error_message columns to the tool_dependency table and drop the uninstalled column from the tool_dependency table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + ToolDependency_table = Table( "tool_dependency", metadata, autoload=True ) + if migrate_engine.name == 'sqlite': + col = Column( "status", TrimmedString( 255 )) + else: + col = Column( "status", TrimmedString( 255 ), nullable=False) + try: + col.create( ToolDependency_table ) + assert col is ToolDependency_table.c.status + except Exception, e: + print "Adding status column to the tool_dependency table failed: %s" % str( e ) + col = Column( "error_message", TEXT ) + try: + col.create( ToolDependency_table ) + assert col is ToolDependency_table.c.error_message + except Exception, e: + print "Adding error_message column to the tool_dependency table failed: %s" % str( e ) + + if migrate_engine.name != 'sqlite': + #This breaks in sqlite due to failure to drop check constraint. + # TODO move to alembic. + try: + ToolDependency_table.c.uninstalled.drop() + except Exception, e: + print "Dropping uninstalled column from the tool_dependency table failed: %s" % str( e ) +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + ToolDependency_table = Table( "tool_dependency", metadata, autoload=True ) + try: + ToolDependency_table.c.status.drop() + except Exception, e: + print "Dropping column status from the tool_dependency table failed: %s" % str( e ) + try: + ToolDependency_table.c.error_message.drop() + except Exception, e: + print "Dropping column error_message from the tool_dependency table failed: %s" % str( e ) + col = Column( "uninstalled", Boolean, default=False ) + try: + col.create( ToolDependency_table ) + assert col is ToolDependency_table.c.uninstalled + except Exception, e: + print "Adding uninstalled column to the tool_dependency table failed: %s" % str( e ) diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0013_add_tool_shed_repository_status_columns.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0013_add_tool_shed_repository_status_columns.py deleted file mode 120000 index c17fdea9421..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0013_add_tool_shed_repository_status_columns.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0103_add_tool_shed_repository_status_columns.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0013_add_tool_shed_repository_status_columns.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0013_add_tool_shed_repository_status_columns.py new file mode 100755 index 00000000000..ff45a2426b7 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0013_add_tool_shed_repository_status_columns.py @@ -0,0 +1,69 @@ +"""Migration script to add status and error_message columns to the tool_shed_repository table.""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +metadata = MetaData() + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + # Add the status column to the tool_shed_repository table. + col = Column( "status", TrimmedString( 255 ) ) + try: + col.create( ToolShedRepository_table ) + assert col is ToolShedRepository_table.c.status + except Exception, e: + print "Adding status column to the tool_shed_repository table failed: %s" % str( e ) + # Add the error_message column to the tool_shed_repository table. + col = Column( "error_message", TEXT ) + try: + col.create( ToolShedRepository_table ) + assert col is ToolShedRepository_table.c.error_message + except Exception, e: + print "Adding error_message column to the tool_shed_repository table failed: %s" % str( e ) + # Update the status column value for tool_shed_repositories to the default value 'Installed'. + cmd = "UPDATE tool_shed_repository SET status = 'Installed';" + try: + migrate_engine.execute( cmd ) + except Exception, e: + print "Exception executing sql command: " + print cmd + print str( e ) + # Update the status column for tool_shed_repositories that have been uninstalled. + cmd = "UPDATE tool_shed_repository SET status = 'Uninstalled' WHERE uninstalled;" + try: + migrate_engine.execute( cmd ) + except Exception, e: + print "Exception executing sql command: " + print cmd + print str( e ) + # Update the status column for tool_shed_repositories that have been deactivated. + cmd = "UPDATE tool_shed_repository SET status = 'Deactivated' where deleted and not uninstalled;" + try: + migrate_engine.execute( cmd ) + except Exception, e: + print "Exception executing sql command: " + print cmd + print str( e ) +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + try: + ToolShedRepository_table.c.status.drop() + except Exception, e: + print "Dropping column status from the tool_shed_repository table failed: %s" % str( e ) + try: + ToolShedRepository_table.c.error_message.drop() + except Exception, e: + print "Dropping column error_message from the tool_shed_repository table failed: %s" % str( e ) diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0014_add_repository_dependency_tables.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0014_add_repository_dependency_tables.py deleted file mode 120000 index 5f07fdf3a63..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0014_add_repository_dependency_tables.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0109_add_repository_dependency_tables.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0014_add_repository_dependency_tables.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0014_add_repository_dependency_tables.py new file mode 100755 index 00000000000..25a020024ca --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0014_add_repository_dependency_tables.py @@ -0,0 +1,60 @@ +""" +Migration script to add the repository_dependency and repository_repository_dependency_association tables. +""" +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +import sys, logging +from galaxy.model.custom_types import * +from sqlalchemy.exc import * +import datetime +now = datetime.datetime.utcnow + +log = logging.getLogger( __name__ ) +log.setLevel( logging.DEBUG ) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() + +RepositoryDependency_table = Table( "repository_dependency", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=False ) ) + +RepositoryRepositoryDependencyAssociation_table = Table( "repository_repository_dependency_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True ), + Column( "repository_dependency_id", Integer, ForeignKey( "repository_dependency.id" ), index=True ) ) + +def upgrade(migrate_engine): + print __doc__ + metadata.bind = migrate_engine + metadata.reflect() + try: + RepositoryDependency_table.create() + except Exception, e: + log.debug( "Creating repository_dependency table failed: %s" % str( e ) ) + try: + RepositoryRepositoryDependencyAssociation_table.create() + except Exception, e: + log.debug( "Creating repository_repository_dependency_association table failed: %s" % str( e ) ) + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + try: + RepositoryRepositoryDependencyAssociation_table.drop() + except Exception, e: + log.debug( "Dropping repository_repository_dependency_association table failed: %s" % str( e ) ) + try: + RepositoryDependency_table.drop() + except Exception, e: + log.debug( "Dropping repository_dependency table failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0015_update_migrate_tools_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0015_update_migrate_tools_table.py deleted file mode 120000 index 55c1ced6cd5..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0015_update_migrate_tools_table.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0113_update_migrate_tools_table.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0015_update_migrate_tools_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0015_update_migrate_tools_table.py new file mode 100755 index 00000000000..bc423a0a93c --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0015_update_migrate_tools_table.py @@ -0,0 +1,38 @@ +""" +Migration script to update the migrate_tools.repository_path column to point to the new location lib/tool_shed/galaxy_install/migrate. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +def upgrade(migrate_engine): + print __doc__ + # Create the table. + try: + cmd = "UPDATE migrate_tools set repository_path='lib/galaxy/tool_shed/migrate';" + migrate_engine.execute( cmd ) + except Exception, e: + log.debug( "Updating migrate_tools.repository_path column to point to the new location lib/tool_shed/galaxy_install/migrate failed: %s" % str( e ) ) + +def downgrade(migrate_engine): + try: + cmd = "UPDATE migrate_tools set repository_path='lib/galaxy/tool_shed/migrate';" + migrate_engine.execute( cmd ) + except Exception, e: + log.debug( "Updating migrate_tools.repository_path column to point to the old location lib/galaxy/tool_shed/migrate failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0016_update_migrate_tools_table_again.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0016_update_migrate_tools_table_again.py deleted file mode 120000 index 7af8a2a1890..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0016_update_migrate_tools_table_again.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0114_update_migrate_tools_table_again.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0016_update_migrate_tools_table_again.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0016_update_migrate_tools_table_again.py new file mode 100755 index 00000000000..aab8bbfc55d --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0016_update_migrate_tools_table_again.py @@ -0,0 +1,40 @@ +""" +Migration script to update the migrate_tools.repository_path column to point to the new location lib/tool_shed/galaxy_install/migrate. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + + +def upgrade(migrate_engine): + print __doc__ + # Create the table. + try: + cmd = "UPDATE migrate_tools set repository_path='lib/tool_shed/galaxy_install/migrate';" + migrate_engine.execute( cmd ) + except Exception, e: + log.debug( "Updating migrate_tools.repository_path column to point to the new location lib/tool_shed/galaxy_install/migrate failed: %s" % str( e ) ) + +def downgrade(migrate_engine): + try: + cmd = "UPDATE migrate_tools set repository_path='lib/galaxy/tool_shed/migrate';" + migrate_engine.execute( cmd ) + except Exception, e: + log.debug( "Updating migrate_tools.repository_path column to point to the old location lib/galaxy/tool_shed/migrate failed: %s" % str( e ) ) + diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0017_drop_update_available_col_add_tool_shed_status_col.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0017_drop_update_available_col_add_tool_shed_status_col.py deleted file mode 120000 index d874f486173..00000000000 --- a/lib/galaxy/model/tool_shed_install/migrate/versions/0017_drop_update_available_col_add_tool_shed_status_col.py +++ /dev/null @@ -1 +0,0 @@ -../../../migrate/versions/0116_drop_update_available_col_add_tool_shed_status_col.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0017_drop_update_available_col_add_tool_shed_status_col.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0017_drop_update_available_col_add_tool_shed_status_col.py new file mode 100755 index 00000000000..5cec56f23cd --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0017_drop_update_available_col_add_tool_shed_status_col.py @@ -0,0 +1,77 @@ +""" +Migration script to drop the update_available Boolean column and replace it with the tool_shed_status JSONType column in the tool_shed_repository table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +import sys, logging +from galaxy.model.custom_types import * +from sqlalchemy.exc import * +import datetime +now = datetime.datetime.utcnow + +log = logging.getLogger( __name__ ) +log.setLevel( logging.DEBUG ) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() + +def default_false( migrate_engine ): + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + return "0" + elif migrate_engine.name in [ 'postgresql', 'postgres' ]: + return "false" + +def upgrade( migrate_engine ): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + try: + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + except NoSuchTableError: + ToolShedRepository_table = None + log.debug( "Failed loading table tool_shed_repository" ) + if ToolShedRepository_table is not None: + # For some unknown reason it is no longer possible to drop a column in a migration script if using the sqlite database. + if migrate_engine.name != 'sqlite': + try: + col = ToolShedRepository_table.c.update_available + col.drop() + except Exception, e: + print "Dropping column update_available from the tool_shed_repository table failed: %s" % str( e ) + c = Column( "tool_shed_status", JSONType, nullable=True ) + try: + c.create( ToolShedRepository_table ) + assert c is ToolShedRepository_table.c.tool_shed_status + except Exception, e: + print "Adding tool_shed_status column to the tool_shed_repository table failed: %s" % str( e ) + +def downgrade( migrate_engine ): + metadata.bind = migrate_engine + metadata.reflect() + try: + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + except NoSuchTableError: + ToolShedRepository_table = None + log.debug( "Failed loading table tool_shed_repository" ) + if ToolShedRepository_table is not None: + # For some unknown reason it is no longer possible to drop a column in a migration script if using the sqlite database. + if migrate_engine.name != 'sqlite': + try: + col = ToolShedRepository_table.c.tool_shed_status + col.drop() + except Exception, e: + print "Dropping column tool_shed_status from the tool_shed_repository table failed: %s" % str( e ) + c = Column( "update_available", Boolean, default=False ) + try: + c.create( ToolShedRepository_table ) + assert c is ToolShedRepository_table.c.update_available + migrate_engine.execute( "UPDATE tool_shed_repository SET update_available=%s" % default_false( migrate_engine ) ) + except Exception, e: + print "Adding column update_available to the tool_shed_repository table failed: %s" % str( e ) diff --git a/lib/galaxy/openid/__init__.py b/lib/galaxy/openid/__init__.py index fcd83f5093c..d196c842445 100644 --- a/lib/galaxy/openid/__init__.py +++ b/lib/galaxy/openid/__init__.py @@ -1,3 +1,3 @@ """ OpenID functionality -""" \ No newline at end of file +""" diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index c531141e094..9df392aa603 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -4,7 +4,7 @@ from galaxy import model from galaxy.tools.parameters import DataToolParameter from galaxy.tools.parameters import DataCollectionToolParameter from galaxy.tools.parameters.wrapped import WrappedParameters -from galaxy.util.json import to_json_string +from galaxy.util.json import dumps from galaxy.util.none_like import NoneDataset from galaxy.util.odict import odict from galaxy.util.template import fill_template @@ -324,7 +324,7 @@ class DefaultToolAction( object ): job.add_output_dataset( name, dataset ) job.object_store_id = object_store_populator.object_store_id if job_params: - job.params = to_json_string( job_params ) + job.params = dumps( job_params ) job.set_handler(tool.get_job_handler(job_params)) trans.sa_session.add( job ) # Now that we have a job id, we can remap any outputs if this is a rerun and the user chose to continue dependent jobs diff --git a/lib/galaxy/tools/actions/metadata.py b/lib/galaxy/tools/actions/metadata.py index f450c19a2de..4fc4558ac43 100644 --- a/lib/galaxy/tools/actions/metadata.py +++ b/lib/galaxy/tools/actions/metadata.py @@ -1,7 +1,7 @@ from __init__ import ToolAction from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper from galaxy.util.odict import odict -from galaxy.util.json import to_json_string +from galaxy.util.json import dumps import logging log = logging.getLogger( __name__ ) @@ -50,7 +50,7 @@ class SetMetadataToolAction( ToolAction ): if user: job.user_id = user.id if job_params: - job.params = to_json_string( job_params ) + job.params = dumps( job_params ) start_job_state = job.state #should be job.states.NEW try: # For backward compatibility, some tools may not have versions yet. diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 2bc47dc886b..0f3ff1a1c24 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -7,7 +7,7 @@ from cgi import FieldStorage from galaxy import datatypes, util from galaxy.util.odict import odict from galaxy.datatypes import sniff -from galaxy.util.json import to_json_string +from galaxy.util.json import dumps from galaxy.model.orm import eagerload_all from galaxy.exceptions import ObjectInvalid @@ -342,7 +342,7 @@ def create_paramfile( trans, uploaded_datasets ): # user cannot remove it unless the parent directory is writable. if link_data_only == 'copy_files' and trans.app.config.external_chown_script: _chown( uploaded_dataset.path ) - json_file.write( to_json_string( json ) + '\n' ) + json_file.write( dumps( json ) + '\n' ) json_file.close() if trans.app.config.external_chown_script: _chown( json_file_path ) @@ -375,7 +375,7 @@ def create_job( trans, params, tool, json_file_path, data_list, folder=None, his for name, value in tool.params_to_strings( params, trans.app ).iteritems(): job.add_parameter( name, value ) - job.add_parameter( 'paramfile', to_json_string( json_file_path ) ) + job.add_parameter( 'paramfile', dumps( json_file_path ) ) object_store_id = None for i, dataset in enumerate( data_list ): if folder: diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index fbf225c6ce0..1983bb2f58b 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -8,7 +8,7 @@ from galaxy import model from galaxy.tools.parameters.basic import UnvalidatedValue from galaxy.web.framework.helpers import to_unicode from galaxy.model.item_attrs import UsesAnnotations -from galaxy.util.json import from_json_string, to_json_string +from galaxy.util.json import loads, dumps from galaxy.web.base.controller import UsesHistoryMixin log = logging.getLogger(__name__) @@ -108,7 +108,7 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations # history_attr_file_name = os.path.join( archive_dir, 'history_attrs.txt') history_attr_str = read_file_contents( history_attr_file_name ) - history_attrs = from_json_string( history_attr_str ) + history_attrs = loads( history_attr_str ) # Create history. new_history = model.History( name='imported from archive: %s' % history_attrs['name'].encode( 'utf-8' ), \ @@ -134,12 +134,12 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations # datasets_attrs_file_name = os.path.join( archive_dir, 'datasets_attrs.txt') datasets_attr_str = read_file_contents( datasets_attrs_file_name ) - datasets_attrs = from_json_string( datasets_attr_str ) - + datasets_attrs = loads( datasets_attr_str ) + if os.path.exists( datasets_attrs_file_name + ".provenance" ): provenance_attr_str = read_file_contents( datasets_attrs_file_name + ".provenance" ) - provenance_attrs = from_json_string( provenance_attr_str ) - datasets_attrs += provenance_attrs + provenance_attrs = loads( provenance_attr_str ) + datasets_attrs += provenance_attrs # Get counts of how often each dataset file is used; a file can # be linked to multiple dataset objects (HDAs). @@ -230,7 +230,7 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations return self.sa_session.query( model.HistoryDatasetAssociation ) \ .filter_by( history=new_history, hid=obj_dct['hid'] ).first() return obj_dct - jobs_attrs = from_json_string( jobs_attr_str, object_hook=as_hda ) + jobs_attrs = loads( jobs_attr_str, object_hook=as_hda ) # Create each job. for job_attrs in jobs_attrs: @@ -279,7 +279,7 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations .filter_by( history=new_history, hid=value.hid ).first() value = input_hda.id #print "added parameter %s-->%s to job %i" % ( name, value, imported_job.id ) - imported_job.add_parameter( name, to_json_string( value, cls=HistoryDatasetAssociationIDEncoder ) ) + imported_job.add_parameter( name, dumps( value, cls=HistoryDatasetAssociationIDEncoder ) ) # TODO: Connect jobs to input datasets. @@ -292,14 +292,14 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations imported_job.add_output_dataset( output_hda.name, output_hda ) # Connect jobs to input datasets. - if 'input_mapping' in job_attrs: + if 'input_mapping' in job_attrs: for input_name, input_hid in job_attrs[ 'input_mapping' ].items(): #print "%s job has input dataset %i" % (imported_job.id, input_hid) input_hda = self.sa_session.query( model.HistoryDatasetAssociation ) \ .filter_by( history=new_history, hid=input_hid ).first() if input_hda: imported_job.add_input_dataset( input_name, input_hda ) - + self.sa_session.flush() @@ -409,7 +409,7 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations } history_attrs_filename = tempfile.NamedTemporaryFile( dir=temp_output_dir ).name history_attrs_out = open( history_attrs_filename, 'w' ) - history_attrs_out.write( to_json_string( history_attrs ) ) + history_attrs_out.write( dumps( history_attrs ) ) history_attrs_out.close() jeha.history_attrs_filename = history_attrs_filename @@ -427,12 +427,12 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations included_datasets.append( dataset ) datasets_attrs_filename = tempfile.NamedTemporaryFile( dir=temp_output_dir ).name datasets_attrs_out = open( datasets_attrs_filename, 'w' ) - datasets_attrs_out.write( to_json_string( datasets_attrs, cls=HistoryDatasetAssociationEncoder ) ) + datasets_attrs_out.write( dumps( datasets_attrs, cls=HistoryDatasetAssociationEncoder ) ) datasets_attrs_out.close() jeha.datasets_attrs_filename = datasets_attrs_filename - + provenance_attrs_out = open( datasets_attrs_filename + ".provenance", 'w' ) - provenance_attrs_out.write( to_json_string( provenance_attrs, cls=HistoryDatasetAssociationEncoder ) ) + provenance_attrs_out.write( dumps( provenance_attrs, cls=HistoryDatasetAssociationEncoder ) ) provenance_attrs_out.close() # @@ -477,7 +477,7 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations job_attrs[ 'exit_code' ] = job.exit_code job_attrs[ 'create_time' ] = job.create_time.isoformat() job_attrs[ 'update_time' ] = job.update_time.isoformat() - + # Get the job's parameters try: @@ -509,7 +509,7 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations jobs_attrs_filename = tempfile.NamedTemporaryFile( dir=temp_output_dir ).name jobs_attrs_out = open( jobs_attrs_filename, 'w' ) - jobs_attrs_out.write( to_json_string( jobs_attrs, cls=HistoryDatasetAssociationEncoder ) ) + jobs_attrs_out.write( dumps( jobs_attrs, cls=HistoryDatasetAssociationEncoder ) ) jobs_attrs_out.close() jeha.jobs_attrs_filename = jobs_attrs_filename diff --git a/lib/galaxy/tools/imp_exp/export_history.py b/lib/galaxy/tools/imp_exp/export_history.py index 5b9a9470302..cb7cbd50273 100644 --- a/lib/galaxy/tools/imp_exp/export_history.py +++ b/lib/galaxy/tools/imp_exp/export_history.py @@ -39,7 +39,7 @@ def create_archive( history_attrs_file, datasets_attrs_file, jobs_attrs_file, ou except OverflowError: pass datasets_attr_in.close() - datasets_attrs = from_json_string( datasets_attr_str ) + datasets_attrs = loads( datasets_attr_str ) # Add datasets to archive and update dataset attributes. # TODO: security check to ensure that files added are in Galaxy dataset directory? @@ -54,7 +54,7 @@ def create_archive( history_attrs_file, datasets_attrs_file, jobs_attrs_file, ou # Rewrite dataset attributes file. datasets_attrs_out = open( datasets_attrs_file, 'w' ) - datasets_attrs_out.write( to_json_string( datasets_attrs ) ) + datasets_attrs_out.write( dumps( datasets_attrs ) ) datasets_attrs_out.close() # Finish archive. @@ -80,4 +80,4 @@ if __name__ == "__main__": # Create archive. status = create_archive( history_attrs, dataset_attrs, job_attrs, out_file, gzip ) - print status \ No newline at end of file + print status diff --git a/lib/galaxy/tools/parameters/__init__.py b/lib/galaxy/tools/parameters/__init__.py index ae71aed85a3..f1fdf1ba372 100644 --- a/lib/galaxy/tools/parameters/__init__.py +++ b/lib/galaxy/tools/parameters/__init__.py @@ -85,7 +85,7 @@ def params_to_strings( params, param_values, app ): for key, value in param_values.iteritems(): if key in params: value = params[ key ].value_to_basic( value, app ) - rval[ key ] = str( to_json_string( value ) ) + rval[ key ] = str( dumps( value ) ) return rval @@ -98,7 +98,7 @@ def params_from_strings( params, param_values, app, ignore_errors=False ): """ rval = dict() for key, value in param_values.iteritems(): - value = json_fix( from_json_string( value ) ) + value = json_fix( loads( value ) ) if key in params: value = params[key].value_from_basic( value, app, ignore_errors ) rval[ key ] = value diff --git a/lib/galaxy/tools/util/__init__.py b/lib/galaxy/tools/util/__init__.py index 1e1919d9b4b..77298be29f5 100644 --- a/lib/galaxy/tools/util/__init__.py +++ b/lib/galaxy/tools/util/__init__.py @@ -3,4 +3,4 @@ Utilities used by various Galaxy tools FIXME: These are used by tool scripts, not the framework, and should not live in this package. -""" \ No newline at end of file +""" diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index b4b1aa5e1d0..6e957f2b07a 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -338,8 +338,8 @@ def shrink_string_by_size( value, size, join_by="..", left_larger=True, beginnin def pretty_print_json(json_data, is_json_string=False): if is_json_string: - json_data = json.from_json_string(json_data) - return json.to_json_string(json_data, sort_keys=True, indent=4) + json_data = json.loads(json_data) + return json.dumps(json_data, sort_keys=True, indent=4) # characters that are valid valid_chars = set(string.letters + string.digits + " -=_.()/+*^,:?!") diff --git a/lib/galaxy/util/backports/__init__.py b/lib/galaxy/util/backports/__init__.py index c53fb7e1a81..18f846f2f09 100644 --- a/lib/galaxy/util/backports/__init__.py +++ b/lib/galaxy/util/backports/__init__.py @@ -1,3 +1,3 @@ """ Modules for providing backward compatibility with future versions of Python -""" \ No newline at end of file +""" diff --git a/lib/galaxy/util/backports/importlib/__init__.py b/lib/galaxy/util/backports/importlib/__init__.py index 53a807affae..e885ac50bbf 100644 --- a/lib/galaxy/util/backports/importlib/__init__.py +++ b/lib/galaxy/util/backports/importlib/__init__.py @@ -39,4 +39,4 @@ def import_module(name, package=None): ## Note: this was copied from ## http://svn.python.org/projects/python/trunk/Lib/importlib/__init__.py - ## on 24 September 2012 \ No newline at end of file + ## on 24 September 2012 diff --git a/lib/galaxy/util/dbkeys.py b/lib/galaxy/util/dbkeys.py index 884640a7484..2c36902b805 100644 --- a/lib/galaxy/util/dbkeys.py +++ b/lib/galaxy/util/dbkeys.py @@ -3,7 +3,7 @@ Functionality for dealing with dbkeys. """ #dbkeys read from disk using builds.txt from galaxy.util import read_dbnames -from galaxy.util.json import from_json_string +from galaxy.util.json import loads import os.path @@ -38,7 +38,7 @@ class GenomeBuilds( object ): rval.append( ( dataset.dbkey, "%s (%s) [History]" % ( dataset.name, dataset.dbkey ) ) ) user = trans.user if user and 'dbkeys' in user.preferences: - user_keys = from_json_string( user.preferences['dbkeys'] ) + user_keys = loads( user.preferences['dbkeys'] ) for key, chrom_dict in user_keys.iteritems(): rval.append( ( key, "%s (%s) [Custom]" % ( chrom_dict['name'], key ) ) ) # Load old builds.txt static keys @@ -61,8 +61,8 @@ class GenomeBuilds( object ): chrom_info = db_dataset.file_name else: # Do Custom Build handling - if trans.user and ( 'dbkeys' in trans.user.preferences ) and ( dbkey in from_json_string( trans.user.preferences[ 'dbkeys' ] ) ): - custom_build_dict = from_json_string( trans.user.preferences[ 'dbkeys' ] )[ dbkey ] + if trans.user and ( 'dbkeys' in trans.user.preferences ) and ( dbkey in loads( trans.user.preferences[ 'dbkeys' ] ) ): + custom_build_dict = loads( trans.user.preferences[ 'dbkeys' ] )[ dbkey ] # HACK: the attempt to get chrom_info below will trigger the # fasta-to-len converter if the dataset is not available or, # which will in turn create a recursive loop when diff --git a/lib/galaxy/util/json.py b/lib/galaxy/util/json.py index 871992294bd..254d965399d 100644 --- a/lib/galaxy/util/json.py +++ b/lib/galaxy/util/json.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -__all__ = [ "to_json_string", "from_json_string", "json_fix", "validate_jsonrpc_request", "validate_jsonrpc_response", "jsonrpc_request", "jsonrpc_response" ] +__all__ = [ "dumps", "loads", "json_fix", "validate_jsonrpc_request", "validate_jsonrpc_response", "jsonrpc_request", "jsonrpc_response" ] import json import logging @@ -9,6 +9,8 @@ import string to_json_string = json.dumps from_json_string = json.loads +dumps = json.dumps +loads = json.loads log = logging.getLogger( __name__ ) @@ -28,7 +30,7 @@ def json_fix( val ): def validate_jsonrpc_request( request, regular_methods, notification_methods ): try: - request = from_json_string( request ) + request = loads( request ) except Exception, e: return False, request, jsonrpc_response( id=None, error=dict( code=-32700, @@ -65,7 +67,7 @@ def validate_jsonrpc_request( request, regular_methods, notification_methods ): def validate_jsonrpc_response( response, id=None ): try: - response = from_json_string( response ) + response = loads( response ) except Exception, e: log.error( 'Response was not valid JSON: %s' % str( e ) ) log.debug( 'Response was: %s' % response ) diff --git a/lib/galaxy/util/log/__init__.py b/lib/galaxy/util/log/__init__.py index 7f12ad27adf..7ba27d27d10 100644 --- a/lib/galaxy/util/log/__init__.py +++ b/lib/galaxy/util/log/__init__.py @@ -2,4 +2,4 @@ class TraceLogger( object ): def __init__( self, name ): self.name = name def log( **kwargs ): - raise TypeError( "Abstract Method" ) \ No newline at end of file + raise TypeError( "Abstract Method" ) diff --git a/lib/galaxy/util/pastescript/serve.py b/lib/galaxy/util/pastescript/serve.py index 1fc07759ee2..c8006713c3c 100644 --- a/lib/galaxy/util/pastescript/serve.py +++ b/lib/galaxy/util/pastescript/serve.py @@ -1056,4 +1056,4 @@ def invoke(command, command_name, options, args): except BadCommand, e: print e.message exit_code = e.exit_code - sys.exit(exit_code) \ No newline at end of file + sys.exit(exit_code) diff --git a/lib/galaxy/visualization/data_providers/__init__.py b/lib/galaxy/visualization/data_providers/__init__.py index 6dd520aaa91..679ff01ad84 100644 --- a/lib/galaxy/visualization/data_providers/__init__.py +++ b/lib/galaxy/visualization/data_providers/__init__.py @@ -1,3 +1,3 @@ """ Galaxy visualization/visual analysis data providers. -""" \ No newline at end of file +""" diff --git a/lib/galaxy/visualization/data_providers/basic.py b/lib/galaxy/visualization/data_providers/basic.py index 7023c4e497e..3108fb26d84 100644 --- a/lib/galaxy/visualization/data_providers/basic.py +++ b/lib/galaxy/visualization/data_providers/basic.py @@ -1,6 +1,6 @@ import sys from galaxy.datatypes.tabular import Tabular -from galaxy.util.json import from_json_string +from galaxy.util.json import loads class BaseDataProvider( object ): """ @@ -96,7 +96,7 @@ class ColumnDataProvider( BaseDataProvider ): start_val = int( self.original_dataset.metadata.comment_lines ) # columns is an array of ints for now (should handle column names later) - columns = from_json_string( columns ) + columns = loads( columns ) for column in columns: assert( ( column < self.original_dataset.metadata.columns ) and ( column >= 0 ) ),( diff --git a/lib/galaxy/visualization/data_providers/genome.py b/lib/galaxy/visualization/data_providers/genome.py index 2dc616f12a0..66e102e22b8 100644 --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -17,7 +17,7 @@ from bx.bbi.bigbed_file import BigBedFile from pysam import csamtools, ctabix from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GFFFeature, GFFInterval, GFFReaderWrapper, parse_gff_attributes -from galaxy.util.json import from_json_string +from galaxy.util.json import loads from galaxy.visualization.data_providers.basic import BaseDataProvider from galaxy.visualization.data_providers.cigar import get_ref_based_read_seq_and_cigar from galaxy.datatypes.interval import Bed, Gff, Gtf @@ -409,7 +409,7 @@ class IntervalDataProvider( GenomeDataProvider ): # # First three entries are mandatory, others are optional. # - filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) + filter_cols = loads( kwargs.get( "filter_cols", "[]" ) ) no_detail = ( "no_detail" in kwargs ) rval = [] message = None @@ -490,7 +490,7 @@ class BedDataProvider( GenomeDataProvider ): # # First three entries are mandatory, others are optional. # - filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) + filter_cols = loads( kwargs.get( "filter_cols", "[]" ) ) no_detail = ( "no_detail" in kwargs ) rval = [] message = None @@ -1336,7 +1336,7 @@ class IntervalIndexDataProvider( FilterableMixin, GenomeDataProvider ): # # First three entries are mandatory, others are optional. # - filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) + filter_cols = loads( kwargs.get( "filter_cols", "[]" ) ) no_detail = ( "no_detail" in kwargs ) for count, val in enumerate( iterator ): start, end, offset = val[0], val[1], val[2] @@ -1404,7 +1404,7 @@ class RawGFFDataProvider( GenomeDataProvider ): """ Process data from an iterator to a format that can be provided to client. """ - filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) + filter_cols = loads( kwargs.get( "filter_cols", "[]" ) ) no_detail = ( "no_detail" in kwargs ) results = [] message = None @@ -1446,7 +1446,7 @@ class GtfTabixDataProvider( TabixDataProvider ): feature.append( GFFInterval( None, line.split( '\t') ) ) # Process data. - filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) + filter_cols = loads( kwargs.get( "filter_cols", "[]" ) ) no_detail = ( "no_detail" in kwargs ) results = [] message = None diff --git a/lib/galaxy/visualization/data_providers/phyloviz/nexusparser.py b/lib/galaxy/visualization/data_providers/phyloviz/nexusparser.py index d4b5b34ad99..d34a3697c39 100644 --- a/lib/galaxy/visualization/data_providers/phyloviz/nexusparser.py +++ b/lib/galaxy/visualization/data_providers/phyloviz/nexusparser.py @@ -105,4 +105,4 @@ class Nexus_Parser(Newick_Parser): else: self.inCommentBlock = False return "Nextline" # need to move on to the nextline after getting out of comment - return "" \ No newline at end of file + return "" diff --git a/lib/galaxy/visualization/genome/__init__.py b/lib/galaxy/visualization/genome/__init__.py index 6c374815e4f..39810ccbf2c 100644 --- a/lib/galaxy/visualization/genome/__init__.py +++ b/lib/galaxy/visualization/genome/__init__.py @@ -1,3 +1,3 @@ """ Code for Galaxy genome visualizations. -""" \ No newline at end of file +""" diff --git a/lib/galaxy/visualization/genomes.py b/lib/galaxy/visualization/genomes.py index 197e9724bf9..36d856fd4ad 100644 --- a/lib/galaxy/visualization/genomes.py +++ b/lib/galaxy/visualization/genomes.py @@ -1,6 +1,6 @@ import os, re, sys, glob, logging from bx.seq.twobit import TwoBitFile -from galaxy.util.json import from_json_string +from galaxy.util.json import loads from galaxy import model, util from galaxy.util.bunch import Bunch @@ -221,7 +221,7 @@ class Genomes( object ): user = trans.get_user() if user: if 'dbkeys' in user.preferences: - user_keys_dict = from_json_string( user.preferences[ 'dbkeys' ] ) + user_keys_dict = loads( user.preferences[ 'dbkeys' ] ) dbkeys.extend( [ (attributes[ 'name' ], key ) for key, attributes in user_keys_dict.items() ] ) # Add app keys to dbkeys. @@ -257,7 +257,7 @@ class Genomes( object ): # Look first in user's custom builds. if dbkey_user and 'dbkeys' in dbkey_user.preferences: - user_keys = from_json_string( dbkey_user.preferences['dbkeys'] ) + user_keys = loads( dbkey_user.preferences['dbkeys'] ) if dbkey in user_keys: dbkey_attributes = user_keys[ dbkey ] dbkey_name = dbkey_attributes[ 'name' ] @@ -310,7 +310,7 @@ class Genomes( object ): # Look for key in owner's custom builds. if dbkey_owner and 'dbkeys' in dbkey_owner.preferences: - user_keys = from_json_string( dbkey_owner.preferences[ 'dbkeys' ] ) + user_keys = loads( dbkey_owner.preferences[ 'dbkeys' ] ) if dbkey in user_keys: dbkey_attributes = user_keys[ dbkey ] if 'fasta' in dbkey_attributes: @@ -342,7 +342,7 @@ class Genomes( object ): # Built-in twobit. twobit_file_name = self.genomes[dbkey].twobit_file else: - user_keys = from_json_string( dbkey_user.preferences['dbkeys'] ) + user_keys = loads( dbkey_user.preferences['dbkeys'] ) dbkey_attributes = user_keys[ dbkey ] fasta_dataset = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dbkey_attributes[ 'fasta' ] ) msg = fasta_dataset.convert_dataset( trans, 'twobit' ) diff --git a/lib/galaxy/visualization/registry.py b/lib/galaxy/visualization/registry.py index c18b0f81bed..ba04c02befc 100644 --- a/lib/galaxy/visualization/registry.py +++ b/lib/galaxy/visualization/registry.py @@ -791,7 +791,7 @@ class ResourceParser( object ): 'int' : lambda param: int( param ), 'float' : lambda param: float( param ), #'date' : lambda param: , - 'json' : ( lambda param: galaxy.util.json.from_json_string( + 'json' : ( lambda param: galaxy.util.json.loads( galaxy.util.sanitize_html.sanitize_html( param ) ) ), } diff --git a/lib/galaxy/visualization/tracks/__init__.py b/lib/galaxy/visualization/tracks/__init__.py index 6ac8d65d52c..0e006ee2d53 100644 --- a/lib/galaxy/visualization/tracks/__init__.py +++ b/lib/galaxy/visualization/tracks/__init__.py @@ -1 +1 @@ -""" Summary.py required to be in this module due to pickling. """ \ No newline at end of file +""" Summary.py required to be in this module due to pickling. """ diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index 5216e3e350a..e56981af6b3 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -42,7 +42,7 @@ from galaxy.datatypes.metadata import FileParameter from galaxy.tools.parameters import RuntimeValue, visit_input_values from galaxy.tools.parameters.basic import DataToolParameter from galaxy.tools.parameters.basic import DataCollectionToolParameter -from galaxy.util.json import to_json_string +from galaxy.util.json import dumps from galaxy.workflow.modules import ToolModule from galaxy.workflow.steps import attach_ordered_steps from galaxy.util import validation @@ -1706,7 +1706,7 @@ class UsesStoredWorkflowMixin( SharableItemSecurityMixin, UsesAnnotations ): missing_tool_tups.append( missing_tool_tup ) # Save the entire step_dict in the unused config field, be parsed later # when we do have the tool - step.config = to_json_string(step_dict) + step.config = dumps(step_dict) if step.tool_errors: workflow.has_errors = True # Stick this in the step temporarily diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index ec48a5f1b04..c97255d3561 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -7,7 +7,7 @@ import logging, sys, os, time from operator import itemgetter from cgi import escape from galaxy.util import restore_text, relpath, nice_size, unicodify -from galaxy.util.json import to_json_string +from galaxy.util.json import dumps from galaxy.web import url_for from binascii import hexlify @@ -563,7 +563,7 @@ class SwitchingSelectField(BaseField): html += '' % self.default_field options = [] for name, delegate_field in self.delegate_fields.items(): - field = to_json_string( delegate_field.to_dict() ) + field = dumps( delegate_field.to_dict() ) option = " '%s': %s" % ( name, field ) options.append( option ) html += '