From a22af161ade4f2bccf81f9b009947a8b6c7ec579 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Sun, 14 Feb 2016 13:51:49 +0100 Subject: [PATCH 1/9] For python2/3 compatibility, replace basestring with six's string_types, unicode with six's text_type. --- lib/galaxy/config.py | 4 +- lib/galaxy/datatypes/data.py | 16 +++---- .../display_applications/application.py | 8 ++-- lib/galaxy/datatypes/sniff.py | 7 +-- lib/galaxy/managers/datasets.py | 4 +- lib/galaxy/managers/taggable.py | 6 ++- lib/galaxy/managers/tags.py | 7 ++- lib/galaxy/model/__init__.py | 21 +++++---- lib/galaxy/model/metadata.py | 3 +- lib/galaxy/tools/__init__.py | 11 +++-- lib/galaxy/tools/actions/__init__.py | 3 +- lib/galaxy/tools/data_manager/manager.py | 3 +- lib/galaxy/tools/evaluation.py | 3 +- lib/galaxy/tools/parameters/basic.py | 37 +++++++-------- lib/galaxy/tools/parameters/grouping.py | 8 ++-- lib/galaxy/tools/parameters/sanitize.py | 6 ++- lib/galaxy/tools/parameters/validation.py | 4 +- lib/galaxy/tools/test.py | 4 +- lib/galaxy/util/__init__.py | 40 ++++++++-------- lib/galaxy/util/sanitize_html.py | 12 ++--- .../visualization/data_providers/registry.py | 4 +- .../visualization/plugins/config_parser.py | 4 +- lib/galaxy/web/base/controller.py | 11 +++-- lib/galaxy/web/base/controllers/admin.py | 3 +- lib/galaxy/web/form_builder.py | 17 +++---- lib/galaxy/web/framework/base.py | 3 +- lib/galaxy/web/framework/decorators.py | 3 +- lib/galaxy/web/framework/helpers/__init__.py | 9 ++-- lib/galaxy/web/framework/helpers/grids.py | 46 ++++++++++--------- .../web/framework/middleware/translogger.py | 5 +- lib/galaxy/web/framework/webapp.py | 3 +- lib/galaxy/webapps/galaxy/api/datasets.py | 4 +- lib/galaxy/webapps/galaxy/api/jobs.py | 5 +- lib/galaxy/webapps/galaxy/api/lda_datasets.py | 1 + .../webapps/galaxy/api/visualizations.py | 11 +++-- .../galaxy/controllers/admin_toolshed.py | 3 +- .../galaxy/controllers/data_manager.py | 3 +- .../webapps/galaxy/controllers/dataset.py | 5 +- .../webapps/galaxy/controllers/history.py | 3 +- .../galaxy/controllers/library_common.py | 3 +- lib/galaxy/webapps/galaxy/controllers/page.py | 3 +- .../galaxy/controllers/requests_admin.py | 3 +- .../galaxy/controllers/requests_common.py | 7 +-- lib/galaxy/webapps/galaxy/controllers/tag.py | 11 +++-- .../galaxy/controllers/visualization.py | 9 ++-- .../webapps/galaxy/controllers/workflow.py | 3 +- .../tool_shed/framework/middleware/hg.py | 11 +++-- lib/galaxy_utils/sequence/fasta.py | 3 +- lib/galaxy_utils/sequence/fastq.py | 5 +- lib/pulsar/client/manager.py | 7 +-- .../galaxy_install/install_manager.py | 3 +- .../recipe/env_file_builder.py | 3 +- lib/tool_shed/util/basic_util.py | 3 +- lib/tool_shed/util/common_util.py | 3 +- lib/tool_shed/util/tool_util.py | 3 +- 55 files changed, 247 insertions(+), 183 deletions(-) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index c397d7a9206..c8c757807a9 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -16,6 +16,8 @@ import sys import tempfile import threading from datetime import timedelta +from six import string_types + from galaxy.exceptions import ConfigurationError from galaxy.util import listify from galaxy.util import string_as_bool @@ -125,7 +127,7 @@ class Configuration( object ): self.running_functional_tests = string_as_bool( kwargs.get( 'running_functional_tests', False ) ) self.hours_between_check = kwargs.get( 'hours_between_check', 12 ) self.enable_tool_shed_check = string_as_bool( kwargs.get( 'enable_tool_shed_check', False ) ) - if isinstance( self.hours_between_check, basestring ): + if isinstance( self.hours_between_check, string_types ): self.hours_between_check = float( self.hours_between_check ) try: if isinstance( self.hours_between_check, int ): diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index af276616b32..a5694af120e 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -7,6 +7,7 @@ import tempfile import zipfile from cgi import escape from inspect import isclass +from six import string_types, text_type from . import metadata from galaxy import util @@ -201,10 +202,10 @@ class Data( object ): line = line.strip() if not line: continue - if isinstance(line, unicode): + if isinstance(line, text_type): out.append( '%s' % escape( line ) ) else: - out.append( '%s' % escape( unicode( line, 'utf-8' ) ) ) + out.append( '%s' % escape( text_type( line, 'utf-8' ) ) ) out.append( '' ) out = "".join( out ) except Exception as exc: @@ -327,7 +328,7 @@ class Data( object ): # Prevent IE8 from sniffing content type since we're explicit about it. This prevents intentionally text/plain # content from being rendered in the browser trans.response.headers['X-Content-Type-Options'] = 'nosniff' - if isinstance( data, basestring ): + if isinstance( data, string_types ): return data if filename and filename != "index": # For files in extra_files_path @@ -385,10 +386,10 @@ class Data( object ): def display_name(self, dataset): """Returns formatted html of dataset name""" try: - if isinstance(dataset.name, unicode): + if isinstance(dataset.name, text_type): return escape( dataset.name ) else: - return escape( unicode( dataset.name, 'utf-8 ') ) + return escape( text_type( dataset.name, 'utf-8 ') ) except: return "name unavailable" @@ -404,9 +405,8 @@ class Data( object ): if info.find( '\n' ) >= 0: info = info.replace( '\n', '
' ) - # Convert to unicode to display non-ascii characters. - if not isinstance(info, unicode): - info = unicode( info, 'utf-8') + if not isinstance(info, text_type): + info = text_type( info, 'utf-8') return info except: diff --git a/lib/galaxy/datatypes/display_applications/application.py b/lib/galaxy/datatypes/display_applications/application.py index 94b4e0a1089..65308d59a88 100644 --- a/lib/galaxy/datatypes/display_applications/application.py +++ b/lib/galaxy/datatypes/display_applications/application.py @@ -1,14 +1,16 @@ # Contains objects for using external display applications import logging import urllib +from six import string_types +from urllib import quote_plus +from copy import deepcopy + from galaxy.util import parse_xml, string_as_bool from galaxy.util.odict import odict from galaxy.util.template import fill_template from galaxy.web import url_for from parameters import DisplayApplicationParameter, DisplayApplicationDataParameter, DEFAULT_DATASET_NAME -from urllib import quote_plus from util import encode_dataset_user -from copy import deepcopy log = logging.getLogger( __name__ ) @@ -157,7 +159,7 @@ class DynamicDisplayApplicationBuilder( object ): display_application.add_data_table_watch( data_table.name, version ) links = [] for line in data_iter: - if isinstance( line, basestring ): + if isinstance( line, string_types ): if not skip_startswith or not line.startswith( skip_startswith ): line = line.rstrip( '\n\r' ) if not line: diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 2dbebd9ff18..8f514017fcb 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -13,6 +13,7 @@ import tempfile import zipfile from encodings import search_function as encodings_search_function +from six import text_type from galaxy import util from galaxy.util import multi_byte @@ -51,7 +52,7 @@ def stream_to_open_named_file( stream, fd, filename, source_encoding=None, sourc is_compressed = True else: try: - if unicode( chunk[:2] ) == unicode( util.gzip_magic ): + if text_type( chunk[:2] ) == text_type( util.gzip_magic ): is_compressed = True except: pass @@ -63,7 +64,7 @@ def stream_to_open_named_file( stream, fd, filename, source_encoding=None, sourc is_binary = util.is_binary( chunk ) data_checked = True if not is_compressed and not is_binary: - if not isinstance( chunk, unicode ): + if not isinstance( chunk, text_type ): chunk = chunk.decode( source_encoding, source_error ) os.write( fd, chunk.encode( target_encoding, target_error ) ) else: @@ -200,7 +201,7 @@ def get_headers( fname, sep, count=60, is_multi_byte=False ): line = line.rstrip('\n\r') if is_multi_byte: # TODO: fix this - sep is never found in line - line = unicode( line, 'utf-8' ) + line = text_type( line, 'utf-8' ) sep = sep.encode( 'utf-8' ) headers.append( line.split(sep) ) if idx == count: diff --git a/lib/galaxy/managers/datasets.py b/lib/galaxy/managers/datasets.py index 39f46ea8ad9..07c1750a9ea 100644 --- a/lib/galaxy/managers/datasets.py +++ b/lib/galaxy/managers/datasets.py @@ -1,6 +1,8 @@ """ Manager and Serializer for Datasets. """ +from six import string_types + from galaxy import model from galaxy import exceptions import galaxy.datatypes.metadata @@ -379,7 +381,7 @@ class _UnflattenedMetadataDatasetAssociationSerializer( base.ModelSerializer, # common to lddas and hdas - from mapping.py 'copied_from_history_dataset_association_id' : self.serialize_id, 'copied_from_library_dataset_dataset_association_id': self.serialize_id, - 'info' : lambda i, k, **c: i.info.strip() if isinstance( i.info, basestring ) else i.info, + 'info' : lambda i, k, **c: i.info.strip() if isinstance( i.info, string_types ) else i.info, 'blurb' : lambda i, k, **c: i.blurb, 'peek' : lambda i, k, **c: i.display_peek() if i.peek and i.peek != 'no peek' else None, diff --git a/lib/galaxy/managers/taggable.py b/lib/galaxy/managers/taggable.py index 6ae9fd741d4..26629915deb 100644 --- a/lib/galaxy/managers/taggable.py +++ b/lib/galaxy/managers/taggable.py @@ -5,6 +5,8 @@ Mixins for Taggable model managers and serializers. # from galaxy import exceptions as galaxy_exceptions import logging +from six import text_type + log = logging.getLogger( __name__ ) @@ -34,7 +36,9 @@ def _tags_from_strings( item, tag_handler, new_tags_list, user=None ): # TODO: duped from tags manager - de-dupe when moved to taggable mixin tag_handler.delete_item_tags( user, item ) new_tags_str = ','.join( new_tags_list ) - tag_handler.apply_item_tags( user, item, unicode( new_tags_str.encode( 'utf-8' ), 'utf-8' ) ) + if not isinstance( new_tags_str, text_type): + new_tags_str = text_type( new_tags_str, 'utf-8' ) + tag_handler.apply_item_tags( user, item, new_tags_str ) # TODO:!! does the creation of new_tags_list mean there are now more and more unused tag rows in the db? diff --git a/lib/galaxy/managers/tags.py b/lib/galaxy/managers/tags.py index 304e198552e..17c31695160 100644 --- a/lib/galaxy/managers/tags.py +++ b/lib/galaxy/managers/tags.py @@ -1,6 +1,7 @@ import logging import re +from six import string_types, text_type from sqlalchemy.sql import select from sqlalchemy.sql.expression import func @@ -41,7 +42,9 @@ class TagManager( object ): self.delete_item_tags( user, item ) new_tags_str = ','.join( new_tags_list ) - self.apply_item_tags( user, item, unicode( new_tags_str.encode( 'utf-8' ), 'utf-8' ) ) + if not isinstance( new_tags_str, text_type): + new_tags_str = text_type(new_tags_str, 'utf-8') + self.apply_item_tags( user, item, new_tags_str) self.app.model.context.flush() return item.tags @@ -114,7 +117,7 @@ class TagManager( object ): def item_has_tag( self, user, item, tag ): """Returns true if item is has a given tag.""" # Get tag name. - if isinstance( tag, basestring ): + if isinstance( tag, string_types ): tag_name = tag elif isinstance( tag, self.app.model.Tag ): tag_name = tag.name diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index a101e7a9a38..cfe6c3ae4b1 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -17,6 +17,7 @@ from datetime import datetime, timedelta from itertools import ifilter, imap from string import Template from uuid import UUID, uuid4 +from six import string_types, text_type from sqlalchemy import and_, func, not_, or_, true, join, select from sqlalchemy.orm import joinedload, object_session, aliased @@ -100,8 +101,8 @@ class HasName: object. If string, convert to unicode object assuming 'utf-8' format. """ name = self.name - if isinstance(name, str): - name = unicode(name, 'utf-8') + if not isinstance( name, text_type ): + name = text_type( name, 'utf-8' ) return name @@ -112,18 +113,18 @@ class JobLike: self.numeric_metrics = [] def add_metric( self, plugin, metric_name, metric_value ): - if isinstance( plugin, str ): - plugin = unicode( plugin, 'utf-8' ) + if isinstance( plugin, string_types ) and not isinstance( plugin, text_type ): + plugin = text_type( plugin, 'utf-8' ) - if isinstance( metric_name, str ): - metric_name = unicode( metric_name, 'utf-8' ) + if isinstance( metric_name, string_types ) and not isinstance( metric_name, text_type ): + metric_name = text_type( metric_name, 'utf-8' ) if isinstance( metric_value, numbers.Number ): metric = self._numeric_metric( plugin, metric_name, metric_value ) self.numeric_metrics.append( metric ) else: - if isinstance( metric_value, str ): - metric_value = unicode( metric_value, 'utf-8' ) + if isinstance( metric_value, string_types ) and not isinstance( metric_value, text_type ): + metric_value = text_type( metric_value, 'utf-8' ) if len( metric_value ) > 1022: # Truncate these values - not needed with sqlite # but other backends must need it. @@ -2178,7 +2179,7 @@ class DatasetInstance( object ): data_source = source_list else: # Convert. - if isinstance( source_list, str ): + if isinstance( source_list, string_types ): source_list = [ source_list ] # Loop through sources until viable one is found. @@ -2413,7 +2414,7 @@ class HistoryDatasetAssociation( DatasetInstance, Dictifiable, UsesAnnotations, update_time=hda.update_time.isoformat(), data_type=hda.datatype.__class__.__module__ + '.' + hda.datatype.__class__.__name__, genome_build=hda.dbkey, - misc_info=hda.info.strip() if isinstance( hda.info, basestring ) else hda.info, + misc_info=hda.info.strip() if isinstance( hda.info, string_types ) else hda.info, misc_blurb=hda.blurb ) # add tags string list diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py index 49f8f79b457..b081a08bd76 100644 --- a/lib/galaxy/model/metadata.py +++ b/lib/galaxy/model/metadata.py @@ -14,6 +14,7 @@ import weakref from os.path import abspath +from six import string_types from sqlalchemy.orm import object_session import galaxy.model @@ -151,7 +152,7 @@ class MetadataCollection( object ): JSONified_dict = json.load( open( filename ) ) elif json_dict is not None: log.debug( 'loading metadata from dict for: %s %s' % ( dataset.__class__.__name__, dataset.id ) ) - if isinstance( json_dict, basestring ): + if isinstance( json_dict, string_types ): JSONified_dict = json.loads( json_dict ) elif isinstance( json_dict, dict ): JSONified_dict = json_dict diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index f636bd24d5a..0ffa16f77dc 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -18,6 +18,7 @@ from cgi import FieldStorage from xml.etree import ElementTree from mako.template import Template from paste import httpexceptions +from six import string_types, text_type from galaxy import model from galaxy.managers import histories @@ -48,6 +49,7 @@ from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext from galaxy.util.hash_util import hmac_new from galaxy.util.odict import odict +from galaxy.util import unicodify from galaxy.util.template import fill_template from galaxy.web import url_for from galaxy.web.form_builder import SelectField @@ -1191,7 +1193,7 @@ class Tool( object, Dictifiable ): if isinstance( out_data, odict ): return job, out_data.items() else: - if isinstance( out_data, str ): + if isinstance( out_data, string_types ): message = out_data else: message = 'Failure executing tool (invalid data returned from tool execution)' @@ -1742,7 +1744,7 @@ class Tool( object, Dictifiable ): return 'true' else: return 'false' - elif isinstance(v, basestring) or isnumber: + elif isinstance(v, string_types) or isnumber: return v elif isinstance(v, dict) and hasattr(v, '__class__'): return v @@ -1850,8 +1852,7 @@ class Tool( object, Dictifiable ): tool_help = '' if self.help: tool_help = self.help.render( static_path=url_for( '/static' ), host_url=url_for( '/', qualified=True ) ) - if type( tool_help ) is not unicode: - tool_help = unicode( tool_help, 'utf-8' ) + tool_help = unicodify( tool_help, 'utf-8' ) # create tool versions tool_versions = [] @@ -2367,7 +2368,7 @@ def json_fix( val ): return [ json_fix( v ) for v in val ] elif isinstance( val, dict ): return dict( [ ( json_fix( k ), json_fix( v ) ) for ( k, v ) in val.iteritems() ] ) - elif isinstance( val, unicode ): + elif isinstance( val, text_type ): return val.encode( "utf8" ) else: return val diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index a28543404c6..4f695d5062f 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -1,5 +1,6 @@ import json import re +from six import string_types from galaxy.exceptions import ObjectInvalid from galaxy.model import LibraryDatasetDatasetAssociation @@ -316,7 +317,7 @@ class DefaultToolAction( object ): # or an actual object to copy. metadata_source = output.metadata_source if metadata_source: - if isinstance( metadata_source, basestring ): + if isinstance( metadata_source, string_types ): metadata_source = inp_data[metadata_source] if metadata_source is not None: diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index a2df7de272b..9b4570e1473 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -1,6 +1,7 @@ import errno import json import os +from six import string_types from galaxy import util from galaxy.util.odict import odict @@ -369,7 +370,7 @@ class DataManager( object ): value = kwd.get( column_name ) if data_table_name in self.value_translation_by_data_table_column and column_name in self.value_translation_by_data_table_column[ data_table_name ]: for value_translation in self.value_translation_by_data_table_column[ data_table_name ][ column_name ]: - if isinstance( value_translation, basestring ): + if isinstance( value_translation, string_types ): value = fill_template( value_translation, GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd ) else: value = value_translation( value ) diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index ba638ab9ae7..e6a2314601e 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -1,6 +1,7 @@ import json import os import tempfile +from six import string_types from galaxy import model from galaxy.util.object_wrapper import wrap_with_safe_string @@ -542,7 +543,7 @@ class ToolEvaluator( object ): return None def __build_config_file_text( self, content ): - if isinstance( content, basestring ): + if isinstance( content, string_types ): return content, True content_format = content["format"] diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 10f4ecbbe76..05a20963a94 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -6,7 +6,9 @@ import logging import re import os import os.path +from six import string_types from xml.etree.ElementTree import XML + from galaxy import util from galaxy.web import form_builder from galaxy.util import string_as_bool, sanitize_param, unicodify @@ -30,7 +32,7 @@ WORKFLOW_PARAMETER_REGULAR_EXPRESSION = re.compile( '''\$\{.+?\}''' ) def contains_workflow_parameter(value, search=False): - if not isinstance( value, basestring ): + if not isinstance( value, string_types ): return False if search and WORKFLOW_PARAMETER_REGULAR_EXPRESSION.search(value): return True @@ -141,8 +143,6 @@ class ToolParameter( object, Dictifiable ): def to_string( self, value, app ): """Convert a value to a string representation suitable for persisting""" - if not isinstance( value, basestring ): - value = str( value ) return unicodify( value ) def to_python( self, value, app ): @@ -158,9 +158,6 @@ class ToolParameter( object, Dictifiable ): return self.to_string( value, app ) def value_from_basic( self, value, app, ignore_errors=False ): - # HACK: Some things don't deal with unicode well, psycopg problem? - if type( value ) == unicode: - value = str( value ) # Handle Runtime values (valid for any parameter?) if isinstance( value, dict ) and '__class__' in value and value['__class__'] == "RuntimeValue": return RuntimeValue() @@ -184,7 +181,7 @@ class ToolParameter( object, Dictifiable ): """Called via __str__ when used in the Cheetah template""" if value is None: value = "" - elif not isinstance( value, basestring ): + elif not isinstance( value, string_types ): value = str( value ) if self.tool is None or self.tool.options.sanitize: if self.sanitizer: @@ -555,7 +552,7 @@ class FileToolParameter( ToolParameter ): def to_string( self, value, app ): if value in [ None, '' ]: return None - elif isinstance( value, unicode ) or isinstance( value, str ): + elif isinstance( value, string_types ): return value elif isinstance( value, dict ): # or should we jsonify? @@ -568,7 +565,7 @@ class FileToolParameter( ToolParameter ): def to_python( self, value, app ): if value is None: return None - elif isinstance( value, unicode ) or isinstance( value, str ): + elif isinstance( value, string_types ): return value else: raise Exception( "FileToolParameter cannot be persisted" ) @@ -907,7 +904,7 @@ class SelectToolParameter( ToolParameter ): if value == '': value = None else: - if isinstance( value, basestring ): + if isinstance( value, string_types ): # Split on all whitespace. This not only provides flexibility # in interpreting values but also is needed because many browsers # use \r\n to separate lines. @@ -1172,7 +1169,7 @@ class ColumnListParameter( SelectToolParameter ): """ if self.multiple: # split on newline and , - if isinstance( value, list ) or isinstance( value, basestring ): + if isinstance( value, list ) or isinstance( value, string_types ): column_list = [] if not isinstance( value, list ): value = value.split( '\n' ) @@ -1196,7 +1193,7 @@ class ColumnListParameter( SelectToolParameter ): @staticmethod def _strip_c(column): - if isinstance(column, basestring): + if isinstance(column, string_types): if column.startswith( 'c' ): column = column.strip().lower()[1:] return column @@ -1837,7 +1834,7 @@ class DataToolParameter( BaseDataToolParameter ): raise ValueError( "History does not include a dataset of the required format / build" ) if value in [ None, "None", '' ]: return None - if isinstance( value, str ) and value.find( "," ) > 0: + if isinstance( value, string_types ) and value.find( "," ) > 0: value = [ int( value_part ) for value_part in value.split( "," ) ] if isinstance( value, list ): rval = [] @@ -1900,7 +1897,7 @@ class DataToolParameter( BaseDataToolParameter ): return rval def to_string( self, value, app ): - if value is None or isinstance( value, basestring ): + if value is None or isinstance( value, string_types ): return value elif isinstance( value, int ): return str( value ) @@ -1934,7 +1931,7 @@ class DataToolParameter( BaseDataToolParameter ): else: return app.model.context.query( app.model.HistoryDatasetAssociation ).get( int( value ) ) - if isinstance(value, str) and value.find(",") > -1: + if isinstance(value, string_types) and value.find(",") > -1: values = value.split(",") return [v for v in map( single_to_python, values ) if v not in none_values] else: @@ -2150,7 +2147,7 @@ class DataCollectionToolParameter( BaseDataToolParameter ): raise ValueError( "History does not include a dataset collection of the correct type or containing the correct types of datasets" ) if value in [None, "None"]: return None - if isinstance( value, str ) and value.find( "," ) > 0: + if isinstance( value, string_types ) and value.find( "," ) > 0: value = [ int( value_part ) for value_part in value.split( "," ) ] elif isinstance( value, trans.app.model.HistoryDatasetCollectionAssociation ): rval = value @@ -2168,7 +2165,7 @@ class DataCollectionToolParameter( BaseDataToolParameter ): if isinstance( value, dict ) and 'src' in value and 'id' in value: if value['src'] == 'hdca': rval = trans.sa_session.query( trans.app.model.HistoryDatasetCollectionAssociation ).get( trans.security.decode_id(value['id']) ) - elif isinstance( value, basestring ): + elif isinstance( value, string_types ): if value.startswith( "dce:" ): rval = trans.sa_session.query( trans.app.model.DatasetCollectionElement ).get( value[ len( "dce:"): ] ) elif value.startswith( "hdca:" ): @@ -2182,7 +2179,7 @@ class DataCollectionToolParameter( BaseDataToolParameter ): return rval def to_string( self, value, app ): - if value is None or isinstance( value, basestring ): + if value is None or isinstance( value, string_types ): return value elif isinstance( value, RuntimeValue ): return None @@ -2201,7 +2198,7 @@ class DataCollectionToolParameter( BaseDataToolParameter ): if value is None or value == '' or value == 'None': return value - if not isinstance( value, basestring ): + if not isinstance( value, string_types ): raise ValueError( "Can not convert data collection parameter value to python object - %s" % value ) if value.startswith( "dce:" ): @@ -2358,7 +2355,7 @@ class LibraryDatasetToolParameter( ToolParameter ): encoded_id = None if isinstance(item, dict): encoded_id = item.get('id') - elif isinstance(item, basestring): + elif isinstance(item, string_types): encoded_id = item else: lst = [] diff --git a/lib/galaxy/tools/parameters/grouping.py b/lib/galaxy/tools/parameters/grouping.py index e82685e4dfd..da07012a932 100644 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -8,6 +8,8 @@ log = logging.getLogger( __name__ ) import os import StringIO import unicodedata +from six import text_type + from basic import ToolParameter from galaxy.datatypes import sniff from galaxy.util import inflector @@ -32,7 +34,7 @@ class Group( object, Dictifiable ): def value_to_basic( self, value, app ): """ Convert value to a (possibly nested) representation using only basic - types (dict, list, tuple, str, unicode, int, long, float, bool, None) + types (dict, list, tuple, string_types, int, long, float, bool, None) """ return value @@ -431,7 +433,7 @@ class UploadDataset( Group ): if ftp_files is not None: # Normalize input paths to ensure utf-8 encoding is normal form c. # This allows for comparison when the filesystem uses a different encoding than the browser. - ftp_files = [unicodedata.normalize('NFC', f) for f in ftp_files if isinstance(f, unicode)] + ftp_files = [unicodedata.normalize('NFC', f) for f in ftp_files if isinstance(f, text_type)] if trans.user is None: log.warning( 'Anonymous user passed values in ftp_files: %s' % ftp_files ) ftp_files = [] @@ -443,7 +445,7 @@ class UploadDataset( Group ): path = relpath( os.path.join( dirpath, filename ), user_ftp_dir ) if not os.path.islink( os.path.join( dirpath, filename ) ): # Normalize filesystem paths - if isinstance(path, unicode): + if isinstance(path, text_type): valid_files.append(unicodedata.normalize('NFC', path )) else: valid_files.append(path) diff --git a/lib/galaxy/tools/parameters/sanitize.py b/lib/galaxy/tools/parameters/sanitize.py index 5b70f5742d6..ac22df8ac9d 100644 --- a/lib/galaxy/tools/parameters/sanitize.py +++ b/lib/galaxy/tools/parameters/sanitize.py @@ -4,6 +4,8 @@ Tool Parameter specific sanitizing. import logging import string +from six import string_types + import galaxy.util log = logging.getLogger( __name__ ) @@ -134,7 +136,7 @@ class ToolParameterSanitizer( object ): def restore_param( self, value ): if self.sanitize: - if isinstance( value, basestring ): + if isinstance( value, string_types ): return self.restore_text( value ) elif isinstance( value, list ): return map( self.restore_text, value ) @@ -160,7 +162,7 @@ class ToolParameterSanitizer( object ): """Clean incoming parameters (strings or lists)""" if not self.sanitize: return value - if isinstance( value, basestring ): + if isinstance( value, string_types ): return self.sanitize_text( value ) elif isinstance( value, list ): return map( self.sanitize_text, value ) diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py index d802fd8429e..6451d3d44ba 100644 --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -4,6 +4,8 @@ Classes related to parameter validation. import logging import re +from six import string_types + from galaxy import model from galaxy import util @@ -383,7 +385,7 @@ class MetadataInDataTableColumnValidator( Validator ): self.valid_values = [] self._data_table_content_version = None self._tool_data_table = tool_data_table - if isinstance( metadata_column, basestring ): + if isinstance( metadata_column, string_types ): metadata_column = tool_data_table.columns[ metadata_column ] self._metadata_column = metadata_column self._load_values() diff --git a/lib/galaxy/tools/test.py b/lib/galaxy/tools/test.py index 83b7b50e41a..916cab780b0 100644 --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -1,6 +1,8 @@ import logging import os import os.path +from six import string_types + import galaxy.tools.parameters.basic import galaxy.tools.parameters.grouping from galaxy.util import string_as_bool @@ -117,7 +119,7 @@ class ToolTestBuilder( object ): log.info( msg ) def __split_if_str( self, value ): - split = isinstance(value, str) + split = isinstance(value, string_types) if split: value = value.split(",") return value diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 6be669b0936..6faff5e69e2 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -22,21 +22,22 @@ import sys import time import tempfile import threading -from six.moves.urllib import parse as urlparse -from six import iteritems - -from galaxy.util import json -from datetime import datetime - -from six import PY3 -from six import string_types, text_type -from six.moves import xrange -from six.moves import email_mime_text -from six.moves import zip from os.path import relpath from hashlib import md5 +from six import iteritems +from six import PY3 +from six import string_types, text_type +from six.moves import email_mime_text +from six.moves.urllib import parse as urlparse +from six.moves import xrange +from six.moves import zip +from xml.etree import ElementTree, ElementInclude + +from galaxy.util import json +from datetime import datetime + try: import docutils.core as docutils_core import docutils.writers.html4css1 as docutils_html4css1 @@ -44,8 +45,6 @@ except ImportError: docutils_core = None docutils_html4css1 = None -from xml.etree import ElementTree, ElementInclude - from .inflection import Inflector, English inflector = Inflector(English) @@ -852,13 +851,18 @@ def unicodify( value, encoding=DEFAULT_ENCODING, error='replace', default=None ) """ Returns a unicode string or None """ - - if isinstance( value, text_type ): - return value + if value is None: + return None try: - return text_type( str( value ), encoding, error ) - except: + if not isinstance(value, string_types): + value = str(value) + # At this point value is of type str, which in Python 2 needs to be converted to unicode + if not isinstance(value, text_type): + value = text_type(value, encoding, error) + except Exception: + log.exception("value %s could not be coerced to unicode" % value) return default + return value def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): diff --git a/lib/galaxy/util/sanitize_html.py b/lib/galaxy/util/sanitize_html.py index 5f34e9fdda1..1cc30695d50 100644 --- a/lib/galaxy/util/sanitize_html.py +++ b/lib/galaxy/util/sanitize_html.py @@ -6,7 +6,7 @@ import re import sgmllib from six import unichr -from six import text_type as unicode +from six import text_type # reversable htmlentitydefs mappings for Python 2.2 @@ -87,7 +87,7 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): data = re.sub(r'<([^<>\s]+?)\s*/>', self._shorttag_replace, data) data = data.replace(''', "'") data = data.replace('"', '"') - if self.encoding and isinstance(data, unicode): + if self.encoding and isinstance(data, text_type): data = data.encode(self.encoding) sgmllib.SGMLParser.feed(self, data) sgmllib.SGMLParser.close(self) @@ -112,12 +112,12 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): value = value.replace('>', '>').replace('<', '<').replace('"', '"') value = self.bare_ampersand.sub("&", value) # thanks to Kevin Marks for this breathtaking hack to deal with (valid) high-bit attribute values in UTF-8 feeds - if isinstance(value, unicode): + if isinstance(value, text_type): try: - value = unicode(value, self.encoding) + value = text_type(value, self.encoding) except: - value = unicode(value, 'iso-8859-1') - uattrs.append((unicode(key, self.encoding), value)) + value = text_type(value, 'iso-8859-1') + uattrs.append((text_type(key, self.encoding), value)) strattrs = u''.join([u' %s="%s"' % (key, val) for key, val in uattrs]) if self.encoding: try: diff --git a/lib/galaxy/visualization/data_providers/registry.py b/lib/galaxy/visualization/data_providers/registry.py index 9ab9a980356..4fd655fb6f2 100644 --- a/lib/galaxy/visualization/data_providers/registry.py +++ b/lib/galaxy/visualization/data_providers/registry.py @@ -1,3 +1,5 @@ +from six import string_types + from galaxy.visualization.data_providers.basic import ColumnDataProvider from galaxy.visualization.data_providers import genome from galaxy.model import NoConverterException @@ -97,7 +99,7 @@ class DataProviderRegistry( object ): original_dataset=original_dataset ) else: source_list = data_provider_mapping[ source ] - if isinstance( source_list, str ): + if isinstance( source_list, string_types ): source_list = [ source_list ] # Find a valid data provider in the source list. diff --git a/lib/galaxy/visualization/plugins/config_parser.py b/lib/galaxy/visualization/plugins/config_parser.py index eb79095d362..a0e8d39aca1 100644 --- a/lib/galaxy/visualization/plugins/config_parser.py +++ b/lib/galaxy/visualization/plugins/config_parser.py @@ -1,3 +1,5 @@ +from six import string_types + import galaxy.model from galaxy import util @@ -290,7 +292,7 @@ class DataSourceParser( object ): # TODO: too dangerous - constrain these to some allowed list # TODO: does this err if no test_attr - it should... test_attr = test_elem.get( 'test_attr' ) - test_attr = test_attr.split( self.ATTRIBUTE_SPLIT_CHAR ) if isinstance( test_attr, str ) else [] + test_attr = test_attr.split( self.ATTRIBUTE_SPLIT_CHAR ) if isinstance( test_attr, string_types ) else [] # log.debug( 'test_type: %s, test_attr: %s, test_result: %s', test_type, test_attr, test_result ) # build a lambda function that gets the desired attribute to test diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index 8340b227d0b..68d4753e546 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -5,6 +5,7 @@ import logging import operator import re +from six import string_types, text_type from sqlalchemy import true from paste.httpexceptions import HTTPBadRequest, HTTPInternalServerError @@ -221,7 +222,7 @@ class BaseAPIController( BaseController ): def _parse_serialization_params( self, kwd, default_view ): view = kwd.get( 'view', None ) keys = kwd.get( 'keys' ) - if isinstance( keys, basestring ): + if isinstance( keys, string_types ): keys = keys.split( ',' ) return dict( view=view, keys=keys, default_view=default_view ) @@ -373,11 +374,11 @@ class ExportsHistoryMixin: def queue_history_export( self, trans, history, gzip=True, include_hidden=False, include_deleted=False ): # Convert options to booleans. - if isinstance( gzip, basestring ): + if isinstance( gzip, string_types ): gzip = ( gzip in [ 'True', 'true', 'T', 't' ] ) - if isinstance( include_hidden, basestring ): + if isinstance( include_hidden, string_types ): include_hidden = ( include_hidden in [ 'True', 'true', 'T', 't' ] ) - if isinstance( include_deleted, basestring ): + if isinstance( include_deleted, string_types ): include_deleted = ( include_deleted in [ 'True', 'true', 'T', 't' ] ) # Run job to do export. @@ -1563,7 +1564,7 @@ class UsesFormDefinitionsMixin: else: # Form was submitted via refresh_on_change widget.value = 'new' - elif value == unicode( 'none' ): + elif value == text_type( 'none' ): widget.value = '' else: widget.value = value diff --git a/lib/galaxy/web/base/controllers/admin.py b/lib/galaxy/web/base/controllers/admin.py index 57ffca7b485..4da30f61b79 100644 --- a/lib/galaxy/web/base/controllers/admin.py +++ b/lib/galaxy/web/base/controllers/admin.py @@ -1,6 +1,7 @@ import logging import os from datetime import datetime, timedelta +from six import string_types from string import punctuation as PUNCTUATION from sqlalchemy import and_, false, func, or_ @@ -1129,7 +1130,7 @@ class Admin( object ): # write the configured sanitize_whitelist_file with new whitelist # and update in-memory list. with open(trans.app.config.sanitize_whitelist_file, 'wt') as f: - if isinstance(tools_to_whitelist, basestring): + if isinstance(tools_to_whitelist, string_types): tools_to_whitelist = [tools_to_whitelist] new_whitelist = sorted([tid for tid in tools_to_whitelist if tid in trans.app.toolbox.tools_by_id]) f.write("\n".join(new_whitelist)) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 2fecaf3f5d1..90ddb0c0ce0 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -5,6 +5,7 @@ import os import time import logging +from six import string_types from operator import itemgetter from cgi import escape from galaxy.util import restore_text, relpath, nice_size, unicodify @@ -42,7 +43,7 @@ class TextField(BaseField): def get_html( self, prefix="", disabled=False ): value = self.value - if not isinstance( value, basestring ): + if not isinstance( value, string_types ): value = str( value ) value = unicodify( value ) return unicodify( '' @@ -114,7 +115,7 @@ class CheckboxField(BaseField): def __init__( self, name, checked=None, refresh_on_change=False, refresh_on_change_values=None ): self.name = name - self.checked = ( checked is True ) or ( isinstance( checked, basestring ) and ( checked.lower() in ( "yes", "true", "on" ) ) ) + self.checked = ( checked is True ) or ( isinstance( checked, string_types ) and ( checked.lower() in ( "yes", "true", "on" ) ) ) self.refresh_on_change = refresh_on_change self.refresh_on_change_values = refresh_on_change_values or [] if self.refresh_on_change: @@ -140,7 +141,7 @@ class CheckboxField(BaseField): return isinstance( value, list ) and ( '__CHECKED__' in value or len( value ) == 2 ) def set_checked(self, value): - if isinstance( value, basestring ): + if isinstance( value, string_types ): self.checked = value.lower() in [ "yes", "true", "on" ] else: self.checked = value @@ -328,9 +329,9 @@ class SelectField(BaseField): rval.append( '
' % ( prefix, self.name ) ) # placeholder for the insertion of the Select All/Unselect All buttons for text, value, selected in self.options: style = "" - if not isinstance( value, basestring ): + if not isinstance( value, string_types ): value = str( value ) - if not isinstance( text, basestring ): + if not isinstance( text, string_types ): text = str( text ) text = unicodify( text ) escaped_value = escape( unicodify( value ), quote=True ) @@ -386,13 +387,13 @@ class SelectField(BaseField): if selected: selected_text = " selected" last_selected_value = value - if not isinstance( last_selected_value, basestring ): + if not isinstance( last_selected_value, string_types ): last_selected_value = str( last_selected_value ) else: selected_text = "" - if not isinstance( value, basestring ): + if not isinstance( value, string_types ): value = str( value ) - if not isinstance( text, basestring ): + if not isinstance( text, string_types ): text = str( text ) rval.append( '' % ( escape( unicodify( value ), quote=True ), selected_text, escape( unicodify( text ), quote=True ) ) ) if last_selected_value: diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index 85db7e3df42..c560427dcff 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -12,6 +12,7 @@ import types import routes import webob +from six import string_types from Cookie import SimpleCookie # We will use some very basic HTTP/wsgi utilities from the paste library @@ -209,7 +210,7 @@ class WebApplication( object ): if isinstance( body, ( types.GeneratorType, list, tuple ) ): # Recursively stream the iterable return flatten( body ) - elif isinstance( body, basestring ): + elif isinstance( body, string_types ): # Wrap the string so it can be iterated return [ body ] elif body is None: diff --git a/lib/galaxy/web/framework/decorators.py b/lib/galaxy/web/framework/decorators.py index 9830bfbb21f..65aca60d28d 100644 --- a/lib/galaxy/web/framework/decorators.py +++ b/lib/galaxy/web/framework/decorators.py @@ -1,6 +1,7 @@ import inspect from traceback import format_exc from functools import wraps +from six import string_types import paste.httpexceptions @@ -163,7 +164,7 @@ def __extract_payload_from_request(trans, func, kwargs): for arg in named_args: payload.pop(arg, None) for k, v in payload.iteritems(): - if isinstance(v, (str, unicode)): + if isinstance(v, string_types): try: payload[k] = loads(v) except: diff --git a/lib/galaxy/web/framework/helpers/__init__.py b/lib/galaxy/web/framework/helpers/__init__.py index f84facaaa39..3ba4aa9ca36 100644 --- a/lib/galaxy/web/framework/helpers/__init__.py +++ b/lib/galaxy/web/framework/helpers/__init__.py @@ -4,6 +4,8 @@ Galaxy web framework helpers import time from datetime import datetime, timedelta +from six import string_types, text_type + from galaxy.util import hash_util from galaxy.util.json import safe_dumps as dumps # noqa (used by mako templates) from webhelpers import date @@ -103,11 +105,10 @@ def to_unicode( a_string ): Convert a string to unicode in utf-8 format; if string is already unicode, does nothing because string's encoding cannot be determined by introspection. """ - a_string_type = type( a_string ) - if a_string_type is str: - return unicode( a_string, 'utf-8' ) - elif a_string_type is unicode: + if isinstance( a_string, text_type): return a_string + elif isinstance( a_string, string_types): + return text_type( a_string, 'utf-8' ) def is_true( val ): diff --git a/lib/galaxy/web/framework/helpers/grids.py b/lib/galaxy/web/framework/helpers/grids.py index b0204953a74..ff977178f42 100644 --- a/lib/galaxy/web/framework/helpers/grids.py +++ b/lib/galaxy/web/framework/helpers/grids.py @@ -1,6 +1,11 @@ import logging import math +from markupsafe import escape +from six import text_type +from six import string_types +from sqlalchemy.sql.expression import and_, func, or_, null, false, true + from galaxy.model.item_attrs import RuntimeException, UsesAnnotations, UsesItemRatings from galaxy.util import sanitize_text from galaxy.util.json import loads, dumps @@ -8,9 +13,6 @@ from galaxy.util.odict import odict from galaxy.web.framework import decorators from galaxy.web.framework import url_for from galaxy.web.framework.helpers import iff -from markupsafe import escape - -from sqlalchemy.sql.expression import and_, func, or_, null, false, true log = logging.getLogger( __name__ ) @@ -73,11 +75,11 @@ class Grid( object ): base_filter = self.default_filter.copy() base_sort_key = self.default_sort_key if self.preserve_state: - pref_name = unicode( self.__class__.__name__ + self.cur_filter_pref_name ) + pref_name = text_type( self.__class__.__name__ + self.cur_filter_pref_name ) if pref_name in trans.get_user().preferences: saved_filter = loads( trans.get_user().preferences[pref_name] ) base_filter.update( saved_filter ) - pref_name = unicode( self.__class__.__name__ + self.cur_sort_key_pref_name ) + pref_name = text_type( self.__class__.__name__ + self.cur_sort_key_pref_name ) if pref_name in trans.get_user().preferences: base_sort_key = loads( trans.get_user().preferences[pref_name] ) # Build initial query @@ -113,16 +115,16 @@ class Grid( object ): # Method (1) combines a mix of strings and lists of strings into a single string and (2) attempts to de-jsonify all strings. def loads_recurse(item): decoded_list = [] - if isinstance( item, basestring): + if isinstance( item, string_types): try: # Not clear what we're decoding, so recurse to ensure that we catch everything. decoded_item = loads( item ) if isinstance( decoded_item, list): decoded_list = loads_recurse( decoded_item ) else: - decoded_list = [ unicode( decoded_item ) ] + decoded_list = [ text_type( decoded_item ) ] except ValueError: - decoded_list = [ unicode( item ) ] + decoded_list = [ text_type( item ) ] elif isinstance( item, list): for element in item: a_list = loads_recurse( element ) @@ -136,7 +138,7 @@ class Grid( object ): if len( column_filter ) == 1: column_filter = column_filter[0] # Interpret ',' as a separator for multiple terms. - if isinstance( column_filter, basestring ) and column_filter.find(',') != -1: + if isinstance( column_filter, string_types ) and column_filter.find(',') != -1: column_filter = column_filter.split(',') # Check if filter is empty @@ -145,7 +147,7 @@ class Grid( object ): column_filter = [x for x in column_filter if x != ''] if len(column_filter) == 0: continue - elif isinstance(column_filter, basestring): + elif isinstance(column_filter, string_types): # If filter criterion is empty, do nothing. if column_filter == '': continue @@ -160,13 +162,13 @@ class Grid( object ): if isinstance( column_filter, list ): # Filter is a list; process each item. for filter in column_filter: - if not isinstance( filter, basestring ): - filter = unicode( filter ).encode("utf-8") + if not isinstance( filter, string_types ): + filter = text_type( filter ).encode("utf-8") extra_url_args[ "f-" + column.key ] = dumps( column_filter ) else: # Process singleton filter. - if not isinstance( column_filter, basestring ): - column_filter = unicode(column_filter) + if not isinstance( column_filter, string_types ): + column_filter = text_type(column_filter) extra_url_args[ "f-" + column.key ] = column_filter.encode("utf-8") # Process sort arguments. sort_key = None @@ -231,14 +233,14 @@ class Grid( object ): self.cur_filter_dict = cur_filter_dict # Preserve grid state: save current filter and sort key. if self.preserve_state: - pref_name = unicode( self.__class__.__name__ + self.cur_filter_pref_name ) - trans.get_user().preferences[pref_name] = unicode( dumps( cur_filter_dict ) ) + pref_name = text_type( self.__class__.__name__ + self.cur_filter_pref_name ) + trans.get_user().preferences[pref_name] = text_type( dumps( cur_filter_dict ) ) if sort_key: - pref_name = unicode( self.__class__.__name__ + self.cur_sort_key_pref_name ) - trans.get_user().preferences[pref_name] = unicode( dumps( sort_key ) ) + pref_name = text_type( self.__class__.__name__ + self.cur_sort_key_pref_name ) + trans.get_user().preferences[pref_name] = text_type( dumps( sort_key ) ) trans.sa_session.flush() # Log grid view. - context = unicode( self.__class__.__name__ ) + context = text_type( self.__class__.__name__ ) params = cur_filter_dict.copy() params['sort'] = sort_key params['async'] = ( 'async' in kwargs ) @@ -248,7 +250,7 @@ class Grid( object ): # is effectively 'wiped' out. Nate believes it has something to do with our use of session( autocommit=True ) # in mapping.py. If you change that to False, the log_action doesn't affect the query # Below, I'm rendering the template first (that uses query), then calling log_action, then returning the page - # trans.log_action( trans.get_user(), unicode( "grid.view" ), context, params ) + # trans.log_action( trans.get_user(), text_type( "grid.view" ), context, params ) # Render grid. def url( *args, **kwargs ): @@ -310,7 +312,7 @@ class Grid( object ): # Pass back kwargs so that grid template can set and use args without # grid explicitly having to pass them. kwargs=kwargs ) - trans.log_action( trans.get_user(), unicode( "grid.view" ), context, params ) + trans.log_action( trans.get_user(), text_type( "grid.view" ), context, params ) return page def get_ids( self, **kwargs ): @@ -429,7 +431,7 @@ class TextColumn( GridColumn ): def get_filter( self, trans, user, column_filter ): """ Returns a SQLAlchemy criterion derived from column_filter. """ - if isinstance( column_filter, basestring ): + if isinstance( column_filter, string_types ): return self.get_single_filter( user, column_filter ) elif isinstance( column_filter, list ): clause_list = [] diff --git a/lib/galaxy/web/framework/middleware/translogger.py b/lib/galaxy/web/framework/middleware/translogger.py index 0af2ebbf79b..a112685472b 100644 --- a/lib/galaxy/web/framework/middleware/translogger.py +++ b/lib/galaxy/web/framework/middleware/translogger.py @@ -7,6 +7,7 @@ Middleware for logging requests, using Apache combined log format import logging import time import urllib +from six import string_types class TransLogger(object): @@ -104,9 +105,9 @@ def make_filter( setup_console_handler=True, set_logger_level=logging.DEBUG): from paste.util.converters import asbool - if isinstance(logging_level, basestring): + if isinstance(logging_level, string_types): logging_level = logging._levelNames[logging_level] - if isinstance(set_logger_level, basestring): + if isinstance(set_logger_level, string_types): set_logger_level = logging._levelNames[set_logger_level] return TransLogger( app, diff --git a/lib/galaxy/web/framework/webapp.py b/lib/galaxy/web/framework/webapp.py index ba942a5723c..10424a154b2 100644 --- a/lib/galaxy/web/framework/webapp.py +++ b/lib/galaxy/web/framework/webapp.py @@ -16,6 +16,7 @@ import mako.runtime import mako.lookup from babel.support import Translations from babel import Locale +from six import string_types from sqlalchemy import and_, true from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.orm import joinedload @@ -277,7 +278,7 @@ class GalaxyWebTransaction( base.DefaultWebTransaction, # singular match def matches_allowed_origin( origin, allowed_origin ): - if isinstance( allowed_origin, str ): + if isinstance( allowed_origin, string_types ): return origin == allowed_origin match = allowed_origin.match( origin ) return match and match.group() == origin diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index eae3004ae58..b80c228a804 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -1,6 +1,8 @@ """ API operations on the contents of a history dataset. """ +from six import string_types + from galaxy import model from galaxy import exceptions as galaxy_exceptions from galaxy import web @@ -30,7 +32,7 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ): def _parse_serialization_params( self, kwd, default_view ): view = kwd.get( 'view', None ) keys = kwd.get( 'keys' ) - if isinstance( keys, basestring ): + if isinstance( keys, string_types ): keys = keys.split( ',' ) return dict( view=view, keys=keys, default_view=default_view ) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 64f045f7e96..1a6ac72c7a4 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -7,6 +7,7 @@ API operations on a jobs. import json import logging +from six import string_types from sqlalchemy import and_, false, or_ from sqlalchemy.orm import aliased @@ -73,7 +74,7 @@ class JobController( BaseAPIController, UsesLibraryMixinItems ): def build_and_apply_filters( query, objects, filter_func ): if objects is not None: - if isinstance( objects, basestring ): + if isinstance( objects, string_types ): query = query.filter( filter_func( objects ) ) elif isinstance( objects, list ): t = [] @@ -310,7 +311,7 @@ class JobController( BaseAPIController, UsesLibraryMixinItems ): ) ) else: - if isinstance( payload[ 'state' ], basestring ): + if isinstance( payload[ 'state' ], string_types ): query = query.filter( trans.app.model.Job.state == payload[ 'state' ] ) elif isinstance( payload[ 'state' ], list ): o = [] diff --git a/lib/galaxy/webapps/galaxy/api/lda_datasets.py b/lib/galaxy/webapps/galaxy/api/lda_datasets.py index 6dff54fdf4c..687459afd38 100644 --- a/lib/galaxy/webapps/galaxy/api/lda_datasets.py +++ b/lib/galaxy/webapps/galaxy/api/lda_datasets.py @@ -8,6 +8,7 @@ import string import sys import tempfile import zipfile + from galaxy import exceptions from galaxy import util from galaxy import web diff --git a/lib/galaxy/webapps/galaxy/api/visualizations.py b/lib/galaxy/webapps/galaxy/api/visualizations.py index 17a3afd3f5c..7f40a214d1f 100644 --- a/lib/galaxy/webapps/galaxy/api/visualizations.py +++ b/lib/galaxy/webapps/galaxy/api/visualizations.py @@ -4,6 +4,7 @@ Visualizations resource control over the API. NOTE!: this is a work in progress and functionality and data structures may change often. """ +from six import string_types from galaxy.web.base.controller import BaseAPIController from galaxy.web.base.controller import UsesVisualizationMixin @@ -170,7 +171,7 @@ class VisualizationsController( BaseAPIController, UsesVisualizationMixin, Shara for key, val in payload.items(): # TODO: validate types in VALID_TYPES/registry names at the mixin/model level? if key == 'type': - if not ( isinstance( val, str ) or isinstance( val, unicode ) ): + if not isinstance( val, string_types ): raise ValidationError( '%s must be a string or unicode: %s' % ( key, str( type( val ) ) ) ) val = util.sanitize_html.sanitize_html( val, 'utf-8' ) elif key == 'config': @@ -178,22 +179,22 @@ class VisualizationsController( BaseAPIController, UsesVisualizationMixin, Shara raise ValidationError( '%s must be a dictionary: %s' % ( key, str( type( val ) ) ) ) elif key == 'annotation': - if not ( isinstance( val, str ) or isinstance( val, unicode ) ): + if not isinstance( val, string_types ): raise ValidationError( '%s must be a string or unicode: %s' % ( key, str( type( val ) ) ) ) val = util.sanitize_html.sanitize_html( val, 'utf-8' ) # these are keys that actually only be *updated* at the revision level and not here # (they are still valid for create, tho) elif key == 'title': - if not ( isinstance( val, str ) or isinstance( val, unicode ) ): + if not isinstance( val, string_types ): raise ValidationError( '%s must be a string or unicode: %s' % ( key, str( type( val ) ) ) ) val = util.sanitize_html.sanitize_html( val, 'utf-8' ) elif key == 'slug': - if not ( isinstance( val, str ) or isinstance( val, unicode ) ): + if not isinstance( val, string_types ): raise ValidationError( '%s must be a string: %s' % ( key, str( type( val ) ) ) ) val = util.sanitize_html.sanitize_html( val, 'utf-8' ) elif key == 'dbkey': - if not ( isinstance( val, str ) or isinstance( val, unicode ) ): + if not isinstance( val, string_types ): raise ValidationError( '%s must be a string or unicode: %s' % ( key, str( type( val ) ) ) ) val = util.sanitize_html.sanitize_html( val, 'utf-8' ) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 92ffd7d5ff7..dc671b12ca5 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -3,6 +3,7 @@ import os import shutil from sqlalchemy import false, or_ +from six import string_types import tool_shed.repository_types.util as rt_util from admin import AdminGalaxy @@ -1334,7 +1335,7 @@ class AdminToolshed( AdminGalaxy ): repo_info_dicts = [] repo_info_dict = kwd.get( 'repo_info_dict', None ) if repo_info_dict: - if isinstance( repo_info_dict, basestring ): + if isinstance( repo_info_dict, string_types ): repo_info_dict = encoding_util.tool_shed_decode( repo_info_dict ) else: # Entering this else block occurs only if the tool_shed_repository does not include any valid tools. diff --git a/lib/galaxy/webapps/galaxy/controllers/data_manager.py b/lib/galaxy/webapps/galaxy/controllers/data_manager.py index 9a7337f19bd..6bfbd241ccd 100644 --- a/lib/galaxy/webapps/galaxy/controllers/data_manager.py +++ b/lib/galaxy/webapps/galaxy/controllers/data_manager.py @@ -1,4 +1,5 @@ from markupsafe import escape +from six import string_types import paste.httpexceptions import galaxy.queue_worker @@ -87,7 +88,7 @@ class DataManager( BaseUIController ): @web.expose @web.require_admin def reload_tool_data_tables( self, trans, table_name=None, **kwd ): - if table_name and isinstance( table_name, basestring ): + if table_name and isinstance( table_name, string_types ): table_name = table_name.split( "," ) # Reload the tool data tables table_names = self.app.tool_data_tables.reload_tables( table_names=table_name ) diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index ba531999960..08c1687f847 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -4,6 +4,7 @@ import urllib from markupsafe import escape import paste.httpexceptions +from six import string_types, text_type from sqlalchemy import false, true from galaxy import datatypes, model, util, web @@ -219,7 +220,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesItemRatings, Uses """ Primarily used for the S3ObjectStore - get the status of data transfer if the file is not in cache """ data = self._check_dataset(trans, dataset_id) - if isinstance( data, basestring ): + if isinstance( data, string_types ): return data log.debug( "Checking transfer status for dataset %s..." % data.dataset.id ) @@ -651,7 +652,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesItemRatings, Uses if not dataset: web.httpexceptions.HTTPNotFound() annotation = self.get_item_annotation_str( trans.sa_session, trans.user, dataset ) - if annotation and isinstance( annotation, unicode ): + if annotation and isinstance( annotation, text_type ): annotation = annotation.encode( 'ascii', 'replace' ) # paste needs ascii here return annotation diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index 62974a60b70..e2f54cb5c68 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -2,6 +2,7 @@ import logging import urllib from markupsafe import escape +from six import string_types from sqlalchemy import and_, false, func, null, true from sqlalchemy.orm import eagerload_all @@ -1331,7 +1332,7 @@ class HistoryController( BaseUIController, SharableMixin, UsesAnnotations, UsesI new_name = name[i] # skip if name is empty - if not isinstance( new_name, basestring ) or not new_name.strip(): + if not isinstance( new_name, string_types ) or not new_name.strip(): change_msgs.append( "You must specify a valid name for History: " + cur_name ) continue diff --git a/lib/galaxy/webapps/galaxy/controllers/library_common.py b/lib/galaxy/webapps/galaxy/controllers/library_common.py index cf3d04ac3dd..f9b4f94f33f 100644 --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -12,6 +12,7 @@ import urllib2 import zipfile from markupsafe import escape +from six import text_type from sqlalchemy import and_, false from sqlalchemy.orm import eagerload_all @@ -86,7 +87,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet job_ldda = job_ldda.copied_from_library_dataset_dataset_association rval[id] = { "state": data.state, - "html": unicode( trans.fill_template( "library/common/library_item_info.mako", ldda=data ), 'utf-8' ) + "html": text_type( trans.fill_template( "library/common/library_item_info.mako", ldda=data ), 'utf-8' ) # "force_history_refresh": force_history_refresh } return rval diff --git a/lib/galaxy/webapps/galaxy/controllers/page.py b/lib/galaxy/webapps/galaxy/controllers/page.py index 43050a8b4cf..ab05fe36204 100644 --- a/lib/galaxy/webapps/galaxy/controllers/page.py +++ b/lib/galaxy/webapps/galaxy/controllers/page.py @@ -1,4 +1,5 @@ from markupsafe import escape +from six import text_type from sqlalchemy import and_, desc, false, true from galaxy import managers, model, util, web @@ -625,7 +626,7 @@ class PageController( BaseUIController, SharableMixin, ave_item_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, page ) # Output is string, so convert to unicode for display. - page_content = unicode( processor.output(), 'utf-8' ) + page_content = text_type( processor.output(), 'utf-8' ) return trans.fill_template_mako( "page/display.mako", item=page, item_data=page_content, user_item_rating=user_item_rating, diff --git a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py index 6ececf57a49..dd917846dbd 100644 --- a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import logging import os +from six import text_type from galaxy import model, util from galaxy.web.base.controller import BaseUIController, UsesFormDefinitionsMixin, web @@ -465,7 +466,7 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): # Eliminate the output created using ssh from the tree if password_str in output: output = output.replace( password_str, '' ) - return unicode( output.replace( '\r\n', '
' ) ) + return text_type( output.replace( '\r\n', '
' ) ) @web.json def open_folder( self, trans, request_id, external_service_id, key ): diff --git a/lib/galaxy/webapps/galaxy/controllers/requests_common.py b/lib/galaxy/webapps/galaxy/controllers/requests_common.py index ae2168a0f95..d0858721286 100644 --- a/lib/galaxy/webapps/galaxy/controllers/requests_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/requests_common.py @@ -2,6 +2,7 @@ import csv import logging import re +from six import text_type from sqlalchemy import and_, false, func, select from markupsafe import escape @@ -121,7 +122,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): sample = trans.sa_session.query( self.app.model.Sample ).get( id ) if sample.state.name != state: rval[ id ] = { "state": sample.state.name, - "html_state": unicode( trans.fill_template( "requests/common/sample_state.mako", + "html_state": text_type( trans.fill_template( "requests/common/sample_state.mako", sample=sample), 'utf-8' ) } return rval @@ -142,7 +143,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): sample = trans.sa_session.query( self.app.model.Sample ).get( id ) if len( sample.datasets ) != number_of_datasets: rval[ id ] = { "datasets": len( sample.datasets ), - "html_datasets": unicode( trans.fill_template( "requests/common/sample_datasets.mako", + "html_datasets": text_type( trans.fill_template( "requests/common/sample_datasets.mako", sample=sample), 'utf-8' ) } return rval @@ -163,7 +164,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): sample_dataset = trans.sa_session.query( self.app.model.SampleDataset ).get( trans.security.decode_id( id ) ) if sample_dataset.status != transfer_status: rval[ id ] = { "status": sample_dataset.status, - "html_status": unicode( trans.fill_template( "requests/common/sample_dataset_transfer_status.mako", + "html_status": text_type( trans.fill_template( "requests/common/sample_dataset_transfer_status.mako", sample_dataset=sample_dataset), 'utf-8' ) } return rval diff --git a/lib/galaxy/webapps/galaxy/controllers/tag.py b/lib/galaxy/webapps/galaxy/controllers/tag.py index 3414c20709b..3e6dd444c90 100644 --- a/lib/galaxy/webapps/galaxy/controllers/tag.py +++ b/lib/galaxy/webapps/galaxy/controllers/tag.py @@ -3,12 +3,13 @@ Tags Controller: handles tagging/untagging of entities and provides autocomplete support. """ -from galaxy import web -from galaxy.web.base.controller import BaseUIController, UsesTagsMixin - +from six import text_type from sqlalchemy.sql import select from sqlalchemy.sql.expression import and_, func +from galaxy import web +from galaxy.web.base.controller import BaseUIController, UsesTagsMixin + import logging log = logging.getLogger( __name__ ) @@ -47,7 +48,7 @@ class TagsController ( BaseUIController, UsesTagsMixin ): trans.sa_session.flush() # Log. params = dict( item_id=item.id, item_class=item_class, tag=new_tag ) - trans.log_action( user, unicode( "tag" ), context, params ) + trans.log_action( user, text_type( "tag" ), context, params ) @web.expose @web.require_login( "remove tag from an item" ) @@ -62,7 +63,7 @@ class TagsController ( BaseUIController, UsesTagsMixin ): trans.sa_session.flush() # Log. params = dict( item_id=item.id, item_class=item_class, tag=tag_name ) - trans.log_action( user, unicode( "untag" ), context, params ) + trans.log_action( user, text_type( "untag" ), context, params ) # Retag an item. All previous tags are deleted and new tags are applied. @web.expose diff --git a/lib/galaxy/webapps/galaxy/controllers/visualization.py b/lib/galaxy/webapps/galaxy/controllers/visualization.py index 70d757d7e36..2262a8e4fa6 100644 --- a/lib/galaxy/webapps/galaxy/controllers/visualization.py +++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py @@ -2,9 +2,10 @@ from __future__ import absolute_import import logging -from sqlalchemy import and_, desc, false, or_, true -from paste.httpexceptions import HTTPNotFound, HTTPBadRequest from markupsafe import escape +from paste.httpexceptions import HTTPNotFound, HTTPBadRequest +from six import string_types +from sqlalchemy import and_, desc, false, or_, true from galaxy import managers from galaxy import model, web @@ -808,7 +809,7 @@ class VisualizationController( BaseUIController, SharableMixin, UsesVisualizatio # post to saved in order to save a visualization if type is None or config is None: return HTTPBadRequest( 'A visualization type and config are required to save a visualization' ) - if isinstance( config, basestring ): + if isinstance( config, string_types ): config = loads( config ) title = title or DEFAULT_VISUALIZATION_NAME @@ -931,7 +932,7 @@ class VisualizationController( BaseUIController, SharableMixin, UsesVisualizatio dataset = self.get_hda_or_ldda( trans, dataset_dict[ 'hda_ldda'], dataset_dict[ 'id' ] ) genome_data = self._get_genome_data( trans, dataset, dbkey ) - if not isinstance( genome_data, str ): + if not isinstance( genome_data, string_types ): track[ 'preloaded_data' ] = genome_data # define app configuration for generic mako template diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index 81a57007dba..74cbb8fe450 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -6,6 +6,7 @@ import os import sgmllib import urllib2 +from six import text_type from sqlalchemy import and_ from sqlalchemy.sql import expression from markupsafe import escape @@ -752,7 +753,7 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix workflow_svg=self._workflow_to_svg_canvas( trans, stored ).standalone_xml() ) # strip() b/c myExperiment XML parser doesn't allow white space before XML; utf-8 handles unicode characters. - request = unicode( request_raw.strip(), 'utf-8' ) + request = text_type( request_raw.strip(), 'utf-8' ) # Do request and get result. auth_header = base64.b64encode( '%s:%s' % ( myexp_username, myexp_password )) diff --git a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py index a7117efccbc..ed62395b2d1 100644 --- a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py @@ -9,6 +9,7 @@ import urlparse from paste.auth.basic import AuthBasicAuthenticator from paste.httpheaders import AUTH_TYPE from paste.httpheaders import REMOTE_USER +from six import string_types from galaxy.util import asbool from galaxy.util.hash_util import new_secure_hash @@ -99,7 +100,7 @@ class Hg( object ): # If all of these mechanisms fail, Mercurial will fail, printing an error message. In this case, it # will not let you commit until you set up a username. result = self.authentication( environ ) - if not isinstance( result, str ) and cmd == 'unbundle' and 'wsgi.input' in environ: + if not isinstance( result, string_types ) and cmd == 'unbundle' and 'wsgi.input' in environ: bundle_data_stream = environ[ 'wsgi.input' ] # Convert the incoming mercurial bundle into a json object and persit it to a temporary file for inspection. fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-hg-bundle" ) @@ -139,7 +140,7 @@ class Hg( object ): if len( entry ) == 2: # We possibly found an altered file entry. filename, change_list = entry - if filename and isinstance( filename, str ): + if filename and isinstance( filename, string_types ): if filename == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: # Make sure the any complex repository dependency definitions contain valid tags. is_valid, error_msg = self.repository_tags_are_valid( filename, change_list ) @@ -158,7 +159,7 @@ class Hg( object ): if len( entry ) == 2: # We possibly found an altered file entry. filename, change_list = entry - if filename and isinstance( filename, str ): + if filename and isinstance( filename, string_types ): if filename == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: # Make sure the any complex repository dependency definitions contain valid tags. is_valid, error_msg = self.repository_tags_are_valid( filename, change_list ) @@ -179,7 +180,7 @@ class Hg( object ): if len( entry ) == 2: # We possibly found an altered file entry. filename, change_list = entry - if filename and isinstance( filename, str ): + if filename and isinstance( filename, string_types ): if filename in [ rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: # We check both files since tool dependency definitions files can contain complex @@ -188,7 +189,7 @@ class Hg( object ): if not is_valid: log.debug( error_msg ) return self.__display_exception_remotely( start_response, error_msg ) - if isinstance( result, str ): + if isinstance( result, string_types ): # Authentication was successful AUTH_TYPE.update( environ, 'basic' ) REMOTE_USER.update( environ, result ) diff --git a/lib/galaxy_utils/sequence/fasta.py b/lib/galaxy_utils/sequence/fasta.py index 113061502a2..fd078a3d198 100644 --- a/lib/galaxy_utils/sequence/fasta.py +++ b/lib/galaxy_utils/sequence/fasta.py @@ -1,4 +1,5 @@ # Dan Blankenberg +from six import string_types class fastaSequence( object ): @@ -63,7 +64,7 @@ class fastaNamedReader( object ): return self.file.close() def get( self, sequence_id ): - if not isinstance( sequence_id, basestring ): + if not isinstance( sequence_id, string_types ): sequence_id = sequence_id.identifier rval = None if sequence_id in self.offset_dict: diff --git a/lib/galaxy_utils/sequence/fastq.py b/lib/galaxy_utils/sequence/fastq.py index af4509b9bae..88097206ec9 100644 --- a/lib/galaxy_utils/sequence/fastq.py +++ b/lib/galaxy_utils/sequence/fastq.py @@ -2,6 +2,7 @@ import math import string import transform +from six import string_types from sequence import SequencingRead from fasta import fastaSequence @@ -633,7 +634,7 @@ class fastqNamedReader( object ): def get( self, sequence_identifier ): # Input is either a sequence ID or a sequence object - if not isinstance( sequence_identifier, basestring ): + if not isinstance( sequence_identifier, string_types ): # Input was a sequence object (not a sequence ID). Get the sequence ID sequence_identifier = sequence_identifier.identifier # Get only the ID part of the sequence header @@ -764,7 +765,7 @@ class fastqJoiner( object ): def is_first_mate( self, sequence_id ): is_first = None - if not isinstance( sequence_id, basestring ): + if not isinstance( sequence_id, string_types ): sequence_id = sequence_id.identifier sequence_id, sequence_sep, sequence_desc = sequence_id.partition(' ') if sequence_id[-2] == '/': diff --git a/lib/pulsar/client/manager.py b/lib/pulsar/client/manager.py index ff4e4e9164a..d03b60a8411 100644 --- a/lib/pulsar/client/manager.py +++ b/lib/pulsar/client/manager.py @@ -5,6 +5,7 @@ try: except ImportError: from queue import Queue from os import getenv +from six import string_types from .client import JobClient from .client import InputCachingJobClient @@ -261,11 +262,7 @@ class ClientCacher(object): def _parse_destination_params(destination_params): - try: - unicode_type = unicode - except NameError: - unicode_type = str - if isinstance(destination_params, str) or isinstance(destination_params, unicode_type): + if isinstance(destination_params, string_types): destination_params = url_to_destination_params(destination_params) return destination_params diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index 5574fc84944..8351a935777 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -6,6 +6,7 @@ import tempfile import traceback from fabric.api import lcd +from six import string_types from sqlalchemy import or_ from galaxy import exceptions, util @@ -842,7 +843,7 @@ class InstallRepositoryManager( object ): str( tool_panel_section_key ) ) else: tool_section = None - if isinstance( repo_info_dict, basestring ): + if isinstance( repo_info_dict, string_types ): repo_info_dict = encoding_util.tool_shed_decode( repo_info_dict ) # Clone each repository to the configured location. self.update_tool_shed_repository_status( tool_shed_repository, diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/env_file_builder.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/env_file_builder.py index ae91b56feec..ea0552f0107 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/env_file_builder.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/env_file_builder.py @@ -1,6 +1,7 @@ import logging import os import stat +from six import string_types log = logging.getLogger( __name__ ) @@ -76,7 +77,7 @@ class EnvFileBuilder( object ): log.exception( str( e ) ) return 1 # Convert the received text to a list, in order to support adding one or more lines to the file. - if isinstance( text, basestring ): + if isinstance( text, string_types ): text = [ text ] for line in text: line = line.rstrip() diff --git a/lib/tool_shed/util/basic_util.py b/lib/tool_shed/util/basic_util.py index 3e13b44a320..3d530dbde3b 100644 --- a/lib/tool_shed/util/basic_util.py +++ b/lib/tool_shed/util/basic_util.py @@ -5,6 +5,7 @@ import sys from string import Template import markupsafe +from six import text_type from galaxy.util import nice_size, unicodify @@ -144,7 +145,7 @@ def to_html_string( text ): text = unicodify( text ) except UnicodeDecodeError, e: return "Error decoding string: %s" % str( e ) - text = unicode( markupsafe.escape( text ) ) + text = text_type( markupsafe.escape( text ) ) text = text.replace( '\n', '
' ) text = text.replace( ' ', '    ' ) text = text.replace( ' ', ' ' ) diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py index 091235c4d71..14c37001c3e 100644 --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -3,6 +3,7 @@ import logging import os import urllib import urllib2 +from six import string_types from galaxy import util from galaxy.util.odict import odict @@ -363,7 +364,7 @@ def url_join( base_url, pathspec=None, params=None ): """Return a valid URL produced by appending a base URL and a set of request parameters.""" url = base_url.rstrip( '/' ) if pathspec is not None: - if not isinstance( pathspec, basestring ): + if not isinstance( pathspec, string_types ): pathspec = '/'.join( pathspec ) url = '%s/%s' % ( url, pathspec ) if params is not None: diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py index d88d9df14ca..0eb1ee954ff 100644 --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -1,6 +1,7 @@ import logging import os import shutil +from six import text_type import galaxy.tools from galaxy import util @@ -131,7 +132,7 @@ def get_headers( fname, sep, count=60, is_multi_byte=False ): for idx, line in enumerate( file( fname ) ): line = line.rstrip( '\n\r' ) if is_multi_byte: - line = unicode( line, 'utf-8' ) + line = text_type( line, 'utf-8' ) sep = sep.encode( 'utf-8' ) headers.append( line.split( sep ) ) if idx == count: From fbc40f69f68f4e5d547019b561429fd958c237f8 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Tue, 16 Feb 2016 11:44:49 +0100 Subject: [PATCH 2/9] use six's text_type --- lib/pulsar/client/interface.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/pulsar/client/interface.py b/lib/pulsar/client/interface.py index 9d625f72c60..c346c573890 100644 --- a/lib/pulsar/client/interface.py +++ b/lib/pulsar/client/interface.py @@ -3,10 +3,8 @@ from abc import abstractmethod from string import Template from six import BytesIO -try: - from six import text_type -except ImportError: - from galaxy.util import unicodify as text_type +from six import text_type + try: from urllib import urlencode except ImportError: From da9cd12ae69d584be222f7979f2b5c4567106a99 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Tue, 16 Feb 2016 13:38:52 +0100 Subject: [PATCH 3/9] Use unicodify --- lib/galaxy/datatypes/data.py | 7 ++++--- lib/galaxy/datatypes/sniff.py | 3 ++- lib/galaxy/managers/taggable.py | 7 +++---- lib/galaxy/managers/tags.py | 7 +++---- lib/galaxy/model/__init__.py | 13 +++++++------ lib/galaxy/tools/__init__.py | 16 +++++++++++++++- lib/galaxy/util/__init__.py | 11 +++++++---- lib/galaxy/util/sanitize_html.py | 14 ++++++++------ lib/galaxy/web/framework/helpers/__init__.py | 7 ++----- lib/galaxy/web/framework/helpers/grids.py | 3 ++- .../galaxy/controllers/library_common.py | 4 ++-- lib/galaxy/webapps/galaxy/controllers/page.py | 4 ++-- .../galaxy/controllers/requests_common.py | 17 +++++++---------- .../webapps/galaxy/controllers/workflow.py | 4 ++-- lib/tool_shed/util/tool_util.py | 4 ++-- 15 files changed, 68 insertions(+), 53 deletions(-) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index a5694af120e..f8cced7a18f 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -13,6 +13,7 @@ from . import metadata from galaxy import util from galaxy.datatypes.metadata import MetadataElement # import directly to maintain ease of use in Datatype class definitions from galaxy.util import inflector +from galaxy.util import unicodify from galaxy.util.bunch import Bunch from galaxy.util.odict import odict from galaxy.util.sanitize_html import sanitize_html @@ -205,7 +206,7 @@ class Data( object ): if isinstance(line, text_type): out.append( '%s' % escape( line ) ) else: - out.append( '%s' % escape( text_type( line, 'utf-8' ) ) ) + out.append( '%s' % escape( unicodify( line, 'utf-8' ) ) ) out.append( '' ) out = "".join( out ) except Exception as exc: @@ -389,7 +390,7 @@ class Data( object ): if isinstance(dataset.name, text_type): return escape( dataset.name ) else: - return escape( text_type( dataset.name, 'utf-8 ') ) + return escape( unicodify( dataset.name, 'utf-8' ) ) except: return "name unavailable" @@ -406,7 +407,7 @@ class Data( object ): info = info.replace( '\n', '
' ) if not isinstance(info, text_type): - info = text_type( info, 'utf-8') + info = unicodify( info, 'utf-8' ) return info except: diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 8f514017fcb..d85807d1309 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -17,6 +17,7 @@ from six import text_type from galaxy import util from galaxy.util import multi_byte +from galaxy.util import unicodify from galaxy.util.checkers import check_binary, check_html, is_gzip from galaxy.datatypes.binary import Binary @@ -201,7 +202,7 @@ def get_headers( fname, sep, count=60, is_multi_byte=False ): line = line.rstrip('\n\r') if is_multi_byte: # TODO: fix this - sep is never found in line - line = text_type( line, 'utf-8' ) + line = unicodify( line, 'utf-8' ) sep = sep.encode( 'utf-8' ) headers.append( line.split(sep) ) if idx == count: diff --git a/lib/galaxy/managers/taggable.py b/lib/galaxy/managers/taggable.py index 26629915deb..4be6f2b6c2c 100644 --- a/lib/galaxy/managers/taggable.py +++ b/lib/galaxy/managers/taggable.py @@ -5,7 +5,8 @@ Mixins for Taggable model managers and serializers. # from galaxy import exceptions as galaxy_exceptions import logging -from six import text_type + +from galaxy.util import unicodify log = logging.getLogger( __name__ ) @@ -36,9 +37,7 @@ def _tags_from_strings( item, tag_handler, new_tags_list, user=None ): # TODO: duped from tags manager - de-dupe when moved to taggable mixin tag_handler.delete_item_tags( user, item ) new_tags_str = ','.join( new_tags_list ) - if not isinstance( new_tags_str, text_type): - new_tags_str = text_type( new_tags_str, 'utf-8' ) - tag_handler.apply_item_tags( user, item, new_tags_str ) + tag_handler.apply_item_tags( user, item, unicodify( new_tags_str, 'utf-8' ) ) # TODO:!! does the creation of new_tags_list mean there are now more and more unused tag rows in the db? diff --git a/lib/galaxy/managers/tags.py b/lib/galaxy/managers/tags.py index 17c31695160..dcc7fa286b9 100644 --- a/lib/galaxy/managers/tags.py +++ b/lib/galaxy/managers/tags.py @@ -1,7 +1,8 @@ import logging import re -from six import string_types, text_type +from galaxy.util import unicodify +from six import string_types from sqlalchemy.sql import select from sqlalchemy.sql.expression import func @@ -42,9 +43,7 @@ class TagManager( object ): self.delete_item_tags( user, item ) new_tags_str = ','.join( new_tags_list ) - if not isinstance( new_tags_str, text_type): - new_tags_str = text_type(new_tags_str, 'utf-8') - self.apply_item_tags( user, item, new_tags_str) + self.apply_item_tags( user, item, unicodify( new_tags_str, 'utf-8' ) ) self.app.model.context.flush() return item.tags diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index cfe6c3ae4b1..ac97b6a0e9b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -38,10 +38,11 @@ from galaxy.model.item_attrs import UsesAnnotations from galaxy.util.dictifiable import Dictifiable from galaxy.security import get_permitted_actions from galaxy.util import Params, restore_text, send_mail -from galaxy.util.multi_byte import is_multi_byte from galaxy.util import ready_name_for_url, unique_id -from galaxy.util.bunch import Bunch +from galaxy.util import unicodify +from galaxy.util.multi_byte import is_multi_byte from galaxy.util.hash_util import new_secure_hash +from galaxy.util.bunch import Bunch from galaxy.util.directory_hash import directory_hash_id from galaxy.util.sanitize_html import sanitize_html from galaxy.web.framework.helpers import to_unicode @@ -102,7 +103,7 @@ class HasName: """ name = self.name if not isinstance( name, text_type ): - name = text_type( name, 'utf-8' ) + name = unicodify( name, 'utf-8' ) return name @@ -114,17 +115,17 @@ class JobLike: def add_metric( self, plugin, metric_name, metric_value ): if isinstance( plugin, string_types ) and not isinstance( plugin, text_type ): - plugin = text_type( plugin, 'utf-8' ) + plugin = unicodify( plugin, 'utf-8' ) if isinstance( metric_name, string_types ) and not isinstance( metric_name, text_type ): - metric_name = text_type( metric_name, 'utf-8' ) + metric_name = unicodify( metric_name, 'utf-8' ) if isinstance( metric_value, numbers.Number ): metric = self._numeric_metric( plugin, metric_name, metric_value ) self.numeric_metrics.append( metric ) else: if isinstance( metric_value, string_types ) and not isinstance( metric_value, text_type ): - metric_value = text_type( metric_value, 'utf-8' ) + metric_value = unicodify( metric_value, 'utf-8' ) if len( metric_value ) > 1022: # Truncate these values - not needed with sqlite # but other backends must need it. diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 0ffa16f77dc..35a482aebc1 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -18,7 +18,8 @@ from cgi import FieldStorage from xml.etree import ElementTree from mako.template import Template from paste import httpexceptions -from six import string_types, text_type +from six import string_types +from six import text_type from galaxy import model from galaxy.managers import histories @@ -44,6 +45,7 @@ from galaxy.tools.toolbox import BaseGalaxyToolBox from galaxy.util import rst_to_html, string_as_bool from galaxy.util import ExecutionTimer from galaxy.util import listify +from galaxy.util import unicodify from galaxy.tools.parameters.meta import expand_meta_parameters from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext @@ -1851,8 +1853,20 @@ class Tool( object, Dictifiable ): # create tool help tool_help = '' if self.help: +<<<<<<< 015094511b27a2267ab55621577ed05f3be092d3 tool_help = self.help.render( static_path=url_for( '/static' ), host_url=url_for( '/', qualified=True ) ) tool_help = unicodify( tool_help, 'utf-8' ) +======= + tool_help = self.help + tool_help = tool_help.render( static_path=url_for( '/static' ), host_url=url_for('/', qualified=True) ) + if not isinstance( tool_help, text_type ): + tool_help = unicodify( tool_help, 'utf-8') + + # check if citations exist + tool_citations = False + if self.citations: + tool_citations = True +>>>>>>> Use unicodify # create tool versions tool_versions = [] diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 6faff5e69e2..a969e71b2fa 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -847,16 +847,19 @@ def roundify(amount, sfs=2): return amount[0:sfs] + '0' * (len(amount) - sfs) -def unicodify( value, encoding=DEFAULT_ENCODING, error='replace', default=None ): +def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', default=None): """ - Returns a unicode string or None + Returns a unicode string or None. """ if value is None: return None try: - if not isinstance(value, string_types): + if not isinstance(value, string_types) and not isinstance(value, binary_type): + # In Python 2, value is not an instance of basestring + # In Python 3, value is not an instance of bytes or str value = str(value) - # At this point value is of type str, which in Python 2 needs to be converted to unicode + # Now in Python 2, value is an instance of basestring, but may be not unicode + # Now in Python 3, value is an instance of bytes or str if not isinstance(value, text_type): value = text_type(value, encoding, error) except Exception: diff --git a/lib/galaxy/util/sanitize_html.py b/lib/galaxy/util/sanitize_html.py index 1cc30695d50..ab3e3914c32 100644 --- a/lib/galaxy/util/sanitize_html.py +++ b/lib/galaxy/util/sanitize_html.py @@ -5,8 +5,9 @@ HTML Sanitizer (ripped from feedparser) import re import sgmllib -from six import unichr +from galaxy.util import unicodify from six import text_type +from six import unichr # reversable htmlentitydefs mappings for Python 2.2 @@ -22,6 +23,7 @@ except: name2codepoint[name] = ord(codepoint) codepoint2name[ord(codepoint)] = name + _cp1252 = { unichr(128): unichr(8364), # euro sign unichr(130): unichr(8218), # single low-9 quotation mark @@ -114,15 +116,15 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): # thanks to Kevin Marks for this breathtaking hack to deal with (valid) high-bit attribute values in UTF-8 feeds if isinstance(value, text_type): try: - value = text_type(value, self.encoding) - except: - value = text_type(value, 'iso-8859-1') - uattrs.append((text_type(key, self.encoding), value)) + value = unicodify(value, self.encoding) + except Exception: + value = unicodify(value, 'iso-8859-1') + uattrs.append(unicodify(key, self.encoding), value) strattrs = u''.join([u' %s="%s"' % (key, val) for key, val in uattrs]) if self.encoding: try: strattrs = strattrs.encode(self.encoding) - except: + except Exception: pass if tag in self.elements_no_end_tag: self.pieces.append('<%(tag)s%(strattrs)s />' % locals()) diff --git a/lib/galaxy/web/framework/helpers/__init__.py b/lib/galaxy/web/framework/helpers/__init__.py index 3ba4aa9ca36..6ca9b41cdcb 100644 --- a/lib/galaxy/web/framework/helpers/__init__.py +++ b/lib/galaxy/web/framework/helpers/__init__.py @@ -4,9 +4,9 @@ Galaxy web framework helpers import time from datetime import datetime, timedelta -from six import string_types, text_type from galaxy.util import hash_util +from galaxy.util import unicodify from galaxy.util.json import safe_dumps as dumps # noqa (used by mako templates) from webhelpers import date from webhelpers.html.tags import stylesheet_link, javascript_link @@ -105,10 +105,7 @@ def to_unicode( a_string ): Convert a string to unicode in utf-8 format; if string is already unicode, does nothing because string's encoding cannot be determined by introspection. """ - if isinstance( a_string, text_type): - return a_string - elif isinstance( a_string, string_types): - return text_type( a_string, 'utf-8' ) + return unicodify( a_string, 'utf-8' ) def is_true( val ): diff --git a/lib/galaxy/web/framework/helpers/grids.py b/lib/galaxy/web/framework/helpers/grids.py index ff977178f42..54c8c85a2e4 100644 --- a/lib/galaxy/web/framework/helpers/grids.py +++ b/lib/galaxy/web/framework/helpers/grids.py @@ -8,6 +8,7 @@ from sqlalchemy.sql.expression import and_, func, or_, null, false, true from galaxy.model.item_attrs import RuntimeException, UsesAnnotations, UsesItemRatings from galaxy.util import sanitize_text +from galaxy.util import unicodify from galaxy.util.json import loads, dumps from galaxy.util.odict import odict from galaxy.web.framework import decorators @@ -163,7 +164,7 @@ class Grid( object ): # Filter is a list; process each item. for filter in column_filter: if not isinstance( filter, string_types ): - filter = text_type( filter ).encode("utf-8") + filter = unicodify( filter, 'utf-8' ) extra_url_args[ "f-" + column.key ] = dumps( column_filter ) else: # Process singleton filter. diff --git a/lib/galaxy/webapps/galaxy/controllers/library_common.py b/lib/galaxy/webapps/galaxy/controllers/library_common.py index f9b4f94f33f..f809f6baac3 100644 --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -12,7 +12,6 @@ import urllib2 import zipfile from markupsafe import escape -from six import text_type from sqlalchemy import and_, false from sqlalchemy.orm import eagerload_all @@ -20,6 +19,7 @@ from galaxy import util, web from galaxy.security import Action from galaxy.tools.actions import upload_common from galaxy.util import inflector +from galaxy.util import unicodify from galaxy.util.json import dumps, loads from galaxy.util.streamball import StreamBall from galaxy.web.base.controller import BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMetadataMixin, UsesLibraryMixinItems @@ -87,7 +87,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet job_ldda = job_ldda.copied_from_library_dataset_dataset_association rval[id] = { "state": data.state, - "html": text_type( trans.fill_template( "library/common/library_item_info.mako", ldda=data ), 'utf-8' ) + "html": unicodify( trans.fill_template( "library/common/library_item_info.mako", ldda=data ), 'utf-8' ) # "force_history_refresh": force_history_refresh } return rval diff --git a/lib/galaxy/webapps/galaxy/controllers/page.py b/lib/galaxy/webapps/galaxy/controllers/page.py index ab05fe36204..b3eef7bf8d1 100644 --- a/lib/galaxy/webapps/galaxy/controllers/page.py +++ b/lib/galaxy/webapps/galaxy/controllers/page.py @@ -1,9 +1,9 @@ from markupsafe import escape -from six import text_type from sqlalchemy import and_, desc, false, true from galaxy import managers, model, util, web from galaxy.model.item_attrs import UsesItemRatings +from galaxy.util import unicodify from galaxy.util.json import loads from galaxy.util.sanitize_html import sanitize_html, _BaseHTMLProcessor from galaxy.web import error, url_for @@ -626,7 +626,7 @@ class PageController( BaseUIController, SharableMixin, ave_item_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, page ) # Output is string, so convert to unicode for display. - page_content = text_type( processor.output(), 'utf-8' ) + page_content = unicodify( processor.output(), 'utf-8' ) return trans.fill_template_mako( "page/display.mako", item=page, item_data=page_content, user_item_rating=user_item_rating, diff --git a/lib/galaxy/webapps/galaxy/controllers/requests_common.py b/lib/galaxy/webapps/galaxy/controllers/requests_common.py index d0858721286..04e10186a99 100644 --- a/lib/galaxy/webapps/galaxy/controllers/requests_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/requests_common.py @@ -2,11 +2,11 @@ import csv import logging import re -from six import text_type from sqlalchemy import and_, false, func, select from markupsafe import escape from galaxy import model, util, web +from galaxy.util import unicodify from galaxy.security.validate_user_input import validate_email from galaxy.web.base.controller import BaseUIController, UsesFormDefinitionsMixin from galaxy.web.form_builder import build_select_field, CheckboxField, SelectField, TextField @@ -122,9 +122,8 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): sample = trans.sa_session.query( self.app.model.Sample ).get( id ) if sample.state.name != state: rval[ id ] = { "state": sample.state.name, - "html_state": text_type( trans.fill_template( "requests/common/sample_state.mako", - sample=sample), - 'utf-8' ) } + "html_state": unicodify( trans.fill_template( "requests/common/sample_state.mako", + sample=sample), 'utf-8' ) } return rval @web.json @@ -143,9 +142,8 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): sample = trans.sa_session.query( self.app.model.Sample ).get( id ) if len( sample.datasets ) != number_of_datasets: rval[ id ] = { "datasets": len( sample.datasets ), - "html_datasets": text_type( trans.fill_template( "requests/common/sample_datasets.mako", - sample=sample), - 'utf-8' ) } + "html_datasets": unicodify( trans.fill_template( "requests/common/sample_datasets.mako", + sample=sample), 'utf-8' ) } return rval @web.json @@ -164,9 +162,8 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): sample_dataset = trans.sa_session.query( self.app.model.SampleDataset ).get( trans.security.decode_id( id ) ) if sample_dataset.status != transfer_status: rval[ id ] = { "status": sample_dataset.status, - "html_status": text_type( trans.fill_template( "requests/common/sample_dataset_transfer_status.mako", - sample_dataset=sample_dataset), - 'utf-8' ) } + "html_status": unicodify( trans.fill_template( "requests/common/sample_dataset_transfer_status.mako", + sample_dataset=sample_dataset), 'utf-8' ) } return rval @web.expose diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index 74cbb8fe450..63cf2dc511e 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -6,7 +6,6 @@ import os import sgmllib import urllib2 -from six import text_type from sqlalchemy import and_ from sqlalchemy.sql import expression from markupsafe import escape @@ -20,6 +19,7 @@ from galaxy import web from galaxy.managers import workflows from galaxy.model.item_attrs import UsesItemRatings from galaxy.model.mapping import desc +from galaxy.util import unicodify from galaxy.util.sanitize_html import sanitize_html from galaxy.web import error, url_for from galaxy.web.base.controller import BaseUIController, SharableMixin, UsesStoredWorkflowMixin @@ -753,7 +753,7 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix workflow_svg=self._workflow_to_svg_canvas( trans, stored ).standalone_xml() ) # strip() b/c myExperiment XML parser doesn't allow white space before XML; utf-8 handles unicode characters. - request = text_type( request_raw.strip(), 'utf-8' ) + request = unicodify( request_raw.strip(), 'utf-8' ) # Do request and get result. auth_header = base64.b64encode( '%s:%s' % ( myexp_username, myexp_password )) diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py index 0eb1ee954ff..72bc36c331a 100644 --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -1,11 +1,11 @@ import logging import os import shutil -from six import text_type import galaxy.tools from galaxy import util from galaxy.util import checkers +from galaxy.util import unicodify from galaxy.util.expressions import ExpressionContext from galaxy.web.form_builder import SelectField @@ -132,7 +132,7 @@ def get_headers( fname, sep, count=60, is_multi_byte=False ): for idx, line in enumerate( file( fname ) ): line = line.rstrip( '\n\r' ) if is_multi_byte: - line = text_type( line, 'utf-8' ) + line = unicodify( line, 'utf-8' ) sep = sep.encode( 'utf-8' ) headers.append( line.split( sep ) ) if idx == count: From 0028ac263634ee846b0fb9e0e134f6b771f50e1e Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Tue, 16 Feb 2016 18:11:25 +0100 Subject: [PATCH 4/9] need binary type --- lib/galaxy/util/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index a969e71b2fa..6278e64e95c 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -26,6 +26,7 @@ import threading from os.path import relpath from hashlib import md5 +from six import binary_type from six import iteritems from six import PY3 from six import string_types, text_type From 8ee38f64f33305753aa96fc230d97a8fabda4885 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Wed, 17 Feb 2016 17:24:38 +0100 Subject: [PATCH 5/9] Do not test if text_type instance before calling unicodify() --- lib/galaxy/datatypes/data.py | 17 +++++------------ lib/galaxy/model/__init__.py | 11 +++++------ lib/galaxy/tools/__init__.py | 8 ++------ lib/galaxy/util/sanitize_html.py | 24 +++++------------------- lib/galaxy/web/form_builder.py | 10 ---------- 5 files changed, 17 insertions(+), 53 deletions(-) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index f8cced7a18f..f8a8b60f736 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -7,7 +7,7 @@ import tempfile import zipfile from cgi import escape from inspect import isclass -from six import string_types, text_type +from six import string_types from . import metadata from galaxy import util @@ -203,10 +203,7 @@ class Data( object ): line = line.strip() if not line: continue - if isinstance(line, text_type): - out.append( '%s' % escape( line ) ) - else: - out.append( '%s' % escape( unicodify( line, 'utf-8' ) ) ) + out.append( '%s' % escape( unicodify( line, 'utf-8' ) ) ) out.append( '' ) out = "".join( out ) except Exception as exc: @@ -387,11 +384,8 @@ class Data( object ): def display_name(self, dataset): """Returns formatted html of dataset name""" try: - if isinstance(dataset.name, text_type): - return escape( dataset.name ) - else: - return escape( unicodify( dataset.name, 'utf-8' ) ) - except: + return escape( unicodify( dataset.name, 'utf-8' ) ) + except Exception: return "name unavailable" def display_info(self, dataset): @@ -406,8 +400,7 @@ class Data( object ): if info.find( '\n' ) >= 0: info = info.replace( '\n', '
' ) - if not isinstance(info, text_type): - info = unicodify( info, 'utf-8' ) + info = unicodify( info, 'utf-8' ) return info except: diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index ac97b6a0e9b..9326372ba83 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -17,7 +17,7 @@ from datetime import datetime, timedelta from itertools import ifilter, imap from string import Template from uuid import UUID, uuid4 -from six import string_types, text_type +from six import string_types from sqlalchemy import and_, func, not_, or_, true, join, select from sqlalchemy.orm import joinedload, object_session, aliased @@ -102,8 +102,7 @@ class HasName: object. If string, convert to unicode object assuming 'utf-8' format. """ name = self.name - if not isinstance( name, text_type ): - name = unicodify( name, 'utf-8' ) + name = unicodify( name, 'utf-8' ) return name @@ -114,17 +113,17 @@ class JobLike: self.numeric_metrics = [] def add_metric( self, plugin, metric_name, metric_value ): - if isinstance( plugin, string_types ) and not isinstance( plugin, text_type ): + if isinstance( plugin, string_types ): plugin = unicodify( plugin, 'utf-8' ) - if isinstance( metric_name, string_types ) and not isinstance( metric_name, text_type ): + if isinstance( metric_name, string_types ): metric_name = unicodify( metric_name, 'utf-8' ) if isinstance( metric_value, numbers.Number ): metric = self._numeric_metric( plugin, metric_name, metric_value ) self.numeric_metrics.append( metric ) else: - if isinstance( metric_value, string_types ) and not isinstance( metric_value, text_type ): + if isinstance( metric_value, string_types ): metric_value = unicodify( metric_value, 'utf-8' ) if len( metric_value ) > 1022: # Truncate these values - not needed with sqlite diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 35a482aebc1..43e2e904a28 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -19,7 +19,6 @@ from xml.etree import ElementTree from mako.template import Template from paste import httpexceptions from six import string_types -from six import text_type from galaxy import model from galaxy.managers import histories @@ -1859,8 +1858,7 @@ class Tool( object, Dictifiable ): ======= tool_help = self.help tool_help = tool_help.render( static_path=url_for( '/static' ), host_url=url_for('/', qualified=True) ) - if not isinstance( tool_help, text_type ): - tool_help = unicodify( tool_help, 'utf-8') + tool_help = unicodify( tool_help, 'utf-8') # check if citations exist tool_citations = False @@ -2382,10 +2380,8 @@ def json_fix( val ): return [ json_fix( v ) for v in val ] elif isinstance( val, dict ): return dict( [ ( json_fix( k ), json_fix( v ) ) for ( k, v ) in val.iteritems() ] ) - elif isinstance( val, text_type ): - return val.encode( "utf8" ) else: - return val + return unicodify(val, "utf8" ) class InterruptedUpload( Exception ): diff --git a/lib/galaxy/util/sanitize_html.py b/lib/galaxy/util/sanitize_html.py index ab3e3914c32..2aabbe2dccd 100644 --- a/lib/galaxy/util/sanitize_html.py +++ b/lib/galaxy/util/sanitize_html.py @@ -6,7 +6,6 @@ import re import sgmllib from galaxy.util import unicodify -from six import text_type from six import unichr @@ -89,8 +88,6 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): data = re.sub(r'<([^<>\s]+?)\s*/>', self._shorttag_replace, data) data = data.replace(''', "'") data = data.replace('"', '"') - if self.encoding and isinstance(data, text_type): - data = data.encode(self.encoding) sgmllib.SGMLParser.feed(self, data) sgmllib.SGMLParser.close(self) @@ -113,23 +110,12 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): for key, value in attrs: value = value.replace('>', '>').replace('<', '<').replace('"', '"') value = self.bare_ampersand.sub("&", value) - # thanks to Kevin Marks for this breathtaking hack to deal with (valid) high-bit attribute values in UTF-8 feeds - if isinstance(value, text_type): - try: - value = unicodify(value, self.encoding) - except Exception: - value = unicodify(value, 'iso-8859-1') - uattrs.append(unicodify(key, self.encoding), value) - strattrs = u''.join([u' %s="%s"' % (key, val) for key, val in uattrs]) - if self.encoding: - try: - strattrs = strattrs.encode(self.encoding) - except Exception: - pass + uattrs.append((key, value)) + strattrs = ''.join([' %s="%s"' % (k, v) for k, v in uattrs]) if tag in self.elements_no_end_tag: - self.pieces.append('<%(tag)s%(strattrs)s />' % locals()) + self.pieces.append('<%s%s />' % (tag, strattrs)) else: - self.pieces.append('<%(tag)s%(strattrs)s>' % locals()) + self.pieces.append('<%s%s>' % (tag, strattrs)) def unknown_endtag(self, tag): # called for each end tag, e.g. for , tag will be 'pre' @@ -448,7 +434,7 @@ class _HTMLSanitizer(_BaseHTMLProcessor): def sanitize_html(htmlSource, encoding="utf-8", type="text/html"): p = _HTMLSanitizer(encoding, type) - p.feed(htmlSource) + p.feed(unicodify(htmlSource, encoding)) data = p.output() data = data.strip().replace('\r\n', '\n') return data diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 90ddb0c0ce0..7cab4188cdd 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -43,8 +43,6 @@ class TextField(BaseField): def get_html( self, prefix="", disabled=False ): value = self.value - if not isinstance( value, string_types ): - value = str( value ) value = unicodify( value ) return unicodify( '' % ( prefix, self.name, self.size, escape( value, quote=True ), self.get_disabled_str( disabled ) ) ) @@ -329,10 +327,6 @@ class SelectField(BaseField): rval.append( '
' % ( prefix, self.name ) ) # placeholder for the insertion of the Select All/Unselect All buttons for text, value, selected in self.options: style = "" - if not isinstance( value, string_types ): - value = str( value ) - if not isinstance( text, string_types ): - text = str( text ) text = unicodify( text ) escaped_value = escape( unicodify( value ), quote=True ) uniq_id = "%s%s|%s" % (prefix, self.name, escaped_value) @@ -391,10 +385,6 @@ class SelectField(BaseField): last_selected_value = str( last_selected_value ) else: selected_text = "" - if not isinstance( value, string_types ): - value = str( value ) - if not isinstance( text, string_types ): - text = str( text ) rval.append( '' % ( escape( unicodify( value ), quote=True ), selected_text, escape( unicodify( text ), quote=True ) ) ) if last_selected_value: last_selected_value = ' last_selected_value="%s"' % escape( unicodify( last_selected_value ), quote=True ) From 6653bcb63bb36c450f4a3d0a6722b4a8c04a6ae8 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Wed, 17 Feb 2016 18:10:24 +0100 Subject: [PATCH 6/9] remove unncessary if statements --- lib/galaxy/model/__init__.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 9326372ba83..d3cc43cbb7c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -113,18 +113,13 @@ class JobLike: self.numeric_metrics = [] def add_metric( self, plugin, metric_name, metric_value ): - if isinstance( plugin, string_types ): - plugin = unicodify( plugin, 'utf-8' ) - - if isinstance( metric_name, string_types ): - metric_name = unicodify( metric_name, 'utf-8' ) - + plugin = unicodify( plugin, 'utf-8' ) + metric_name = unicodify( metric_name, 'utf-8' ) if isinstance( metric_value, numbers.Number ): metric = self._numeric_metric( plugin, metric_name, metric_value ) self.numeric_metrics.append( metric ) else: - if isinstance( metric_value, string_types ): - metric_value = unicodify( metric_value, 'utf-8' ) + metric_value = unicodify( metric_value, 'utf-8' ) if len( metric_value ) > 1022: # Truncate these values - not needed with sqlite # but other backends must need it. From 66bce36a4075a0b111fff9832f100bba3566447f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 18 Feb 2016 10:39:04 +0100 Subject: [PATCH 7/9] Indentation fixes, import json_fix from galaxy.util.json --- lib/galaxy/tools/__init__.py | 13 ++----------- lib/galaxy/web/framework/helpers/grids.py | 3 +-- .../webapps/galaxy/controllers/requests_common.py | 9 ++++++--- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 43e2e904a28..e89a7d2ad64 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -49,6 +49,7 @@ from galaxy.tools.parameters.meta import expand_meta_parameters from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext from galaxy.util.hash_util import hmac_new +from galaxy.util.json import json_fix from galaxy.util.odict import odict from galaxy.util import unicodify from galaxy.util.template import fill_template @@ -1852,19 +1853,8 @@ class Tool( object, Dictifiable ): # create tool help tool_help = '' if self.help: -<<<<<<< 015094511b27a2267ab55621577ed05f3be092d3 tool_help = self.help.render( static_path=url_for( '/static' ), host_url=url_for( '/', qualified=True ) ) tool_help = unicodify( tool_help, 'utf-8' ) -======= - tool_help = self.help - tool_help = tool_help.render( static_path=url_for( '/static' ), host_url=url_for('/', qualified=True) ) - tool_help = unicodify( tool_help, 'utf-8') - - # check if citations exist - tool_citations = False - if self.citations: - tool_citations = True ->>>>>>> Use unicodify # create tool versions tool_versions = [] @@ -2375,6 +2365,7 @@ class BadValue( object ): self.value = value +<<<<<<< 371751996916d9995b60a600a1a95456c14bdb2a def json_fix( val ): if isinstance( val, list ): return [ json_fix( v ) for v in val ] diff --git a/lib/galaxy/web/framework/helpers/grids.py b/lib/galaxy/web/framework/helpers/grids.py index 54c8c85a2e4..928c41dc7b2 100644 --- a/lib/galaxy/web/framework/helpers/grids.py +++ b/lib/galaxy/web/framework/helpers/grids.py @@ -8,7 +8,6 @@ from sqlalchemy.sql.expression import and_, func, or_, null, false, true from galaxy.model.item_attrs import RuntimeException, UsesAnnotations, UsesItemRatings from galaxy.util import sanitize_text -from galaxy.util import unicodify from galaxy.util.json import loads, dumps from galaxy.util.odict import odict from galaxy.web.framework import decorators @@ -164,7 +163,7 @@ class Grid( object ): # Filter is a list; process each item. for filter in column_filter: if not isinstance( filter, string_types ): - filter = unicodify( filter, 'utf-8' ) + text_type( filter ).encode("utf-8") extra_url_args[ "f-" + column.key ] = dumps( column_filter ) else: # Process singleton filter. diff --git a/lib/galaxy/webapps/galaxy/controllers/requests_common.py b/lib/galaxy/webapps/galaxy/controllers/requests_common.py index 04e10186a99..0ca93fcfd44 100644 --- a/lib/galaxy/webapps/galaxy/controllers/requests_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/requests_common.py @@ -123,7 +123,8 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): if sample.state.name != state: rval[ id ] = { "state": sample.state.name, "html_state": unicodify( trans.fill_template( "requests/common/sample_state.mako", - sample=sample), 'utf-8' ) } + sample=sample), + 'utf-8' ) } return rval @web.json @@ -143,7 +144,8 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): if len( sample.datasets ) != number_of_datasets: rval[ id ] = { "datasets": len( sample.datasets ), "html_datasets": unicodify( trans.fill_template( "requests/common/sample_datasets.mako", - sample=sample), 'utf-8' ) } + sample=sample), + 'utf-8' ) } return rval @web.json @@ -163,7 +165,8 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): if sample_dataset.status != transfer_status: rval[ id ] = { "status": sample_dataset.status, "html_status": unicodify( trans.fill_template( "requests/common/sample_dataset_transfer_status.mako", - sample_dataset=sample_dataset), 'utf-8' ) } + sample_dataset=sample_dataset), + 'utf-8' ) } return rval @web.expose From f706ed7e86244e183482ce18f68e71dcd6fcd5c3 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 19 Feb 2016 10:40:37 +0100 Subject: [PATCH 8/9] properly encode column_filter --- lib/galaxy/web/framework/helpers/grids.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/galaxy/web/framework/helpers/grids.py b/lib/galaxy/web/framework/helpers/grids.py index 928c41dc7b2..489ab60421b 100644 --- a/lib/galaxy/web/framework/helpers/grids.py +++ b/lib/galaxy/web/framework/helpers/grids.py @@ -161,9 +161,7 @@ class Grid( object ): # that we can encode to UTF-8 and thus handle user input to filters. if isinstance( column_filter, list ): # Filter is a list; process each item. - for filter in column_filter: - if not isinstance( filter, string_types ): - text_type( filter ).encode("utf-8") + column_filter = [ text_type(_).encode('utf-8') if not isinstance(_, string_types) else _ for _ in column_filter ] extra_url_args[ "f-" + column.key ] = dumps( column_filter ) else: # Process singleton filter. From 92fe009f965700f1a3b4e61d33c67c5348084d2a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 19 Feb 2016 11:27:58 +0100 Subject: [PATCH 9/9] fix bad rebase --- lib/galaxy/tools/__init__.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index e89a7d2ad64..b84a62364c2 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -51,7 +51,6 @@ from galaxy.util.expressions import ExpressionContext from galaxy.util.hash_util import hmac_new from galaxy.util.json import json_fix from galaxy.util.odict import odict -from galaxy.util import unicodify from galaxy.util.template import fill_template from galaxy.web import url_for from galaxy.web.form_builder import SelectField @@ -2365,15 +2364,5 @@ class BadValue( object ): self.value = value -<<<<<<< 371751996916d9995b60a600a1a95456c14bdb2a -def json_fix( val ): - if isinstance( val, list ): - return [ json_fix( v ) for v in val ] - elif isinstance( val, dict ): - return dict( [ ( json_fix( k ), json_fix( v ) ) for ( k, v ) in val.iteritems() ] ) - else: - return unicodify(val, "utf8" ) - - class InterruptedUpload( Exception ): pass