From f5060628d2de4802be610fd6caf5c878afdb4312 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 19 Apr 2016 14:19:45 -0400 Subject: [PATCH 1/4] Fix pages encoding. Using str on previously unicodify'd extended set values does not work. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` In [2]: str("Blah blah blah, this breaks the page: ಠ_ಠ") Out[2]: 'Blah blah blah, this breaks the page: \xe0\xb2\xa0_\xe0\xb2\xa0' In [3]: str(unicodify("Blah blah blah, this breaks the page: ಠ_ಠ")) --------------------------------------------------------------------------- UnicodeEncodeError Traceback (most recent call last) in () ----> 1 str(unicodify("Blah blah blah, this breaks the page: ಠ_ಠ")) UnicodeEncodeError: 'ascii' codec can't encode character u'\u0ca0' in position 38: ordinal not in range(128) ``` --- lib/galaxy/util/sanitize_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/util/sanitize_html.py b/lib/galaxy/util/sanitize_html.py index 2aabbe2dccd..aff64ec34bb 100644 --- a/lib/galaxy/util/sanitize_html.py +++ b/lib/galaxy/util/sanitize_html.py @@ -194,7 +194,7 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): def output(self): '''Return processed HTML as a single string''' - return ''.join([str(p) for p in self.pieces]) + return ''.join([unicodify(p) for p in self.pieces]) class _HTMLSanitizer(_BaseHTMLProcessor): From 659a900693b72886fd27731a2568ce71657c4401 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 19 Apr 2016 15:39:51 -0400 Subject: [PATCH 2/4] Simplify expression, trust upstream pieces. --- lib/galaxy/util/sanitize_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/util/sanitize_html.py b/lib/galaxy/util/sanitize_html.py index aff64ec34bb..aec55f9f53a 100644 --- a/lib/galaxy/util/sanitize_html.py +++ b/lib/galaxy/util/sanitize_html.py @@ -194,7 +194,7 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): def output(self): '''Return processed HTML as a single string''' - return ''.join([unicodify(p) for p in self.pieces]) + return ''.join(self.pieces) class _HTMLSanitizer(_BaseHTMLProcessor): From 7c7d6edbfd94e5b3e187778b89ac1febb5690cb9 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 19 Apr 2016 20:54:24 -0400 Subject: [PATCH 3/4] Swap validate_and_sanitize_basestring to use unicodify --- lib/galaxy/util/validation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/util/validation.py b/lib/galaxy/util/validation.py index 7b0686cec19..9bf4a8a5d84 100644 --- a/lib/galaxy/util/validation.py +++ b/lib/galaxy/util/validation.py @@ -3,6 +3,7 @@ TODO: Refactor BaseController references to similar methods to use this module. """ from galaxy import exceptions +from galaxy.util import unicodify from galaxy.util.sanitize_html import sanitize_html from six import string_types, text_type @@ -12,7 +13,7 @@ def validate_and_sanitize_basestring( key, val ): if not isinstance( val, string_types ): raise exceptions.RequestParameterInvalidException( '%s must be a string or unicode: %s' % ( key, str( type( val ) ) ) ) - return text_type( sanitize_html( val, 'utf-8', 'text/html' ), 'utf-8' ) + return unicodify( sanitize_html( val, 'utf-8', 'text/html' ), 'utf-8' ) def validate_and_sanitize_basestring_list( key, val ): From 0cde386a2cfec1c18f68f1dfd939843328f8046f Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 20 Apr 2016 10:00:08 -0400 Subject: [PATCH 4/4] Remove more unnecessary unicode/text_type, since sanitize_html will always return unicode. --- lib/galaxy/util/validation.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/util/validation.py b/lib/galaxy/util/validation.py index 9bf4a8a5d84..a163abf0b44 100644 --- a/lib/galaxy/util/validation.py +++ b/lib/galaxy/util/validation.py @@ -3,23 +3,22 @@ TODO: Refactor BaseController references to similar methods to use this module. """ from galaxy import exceptions -from galaxy.util import unicodify from galaxy.util.sanitize_html import sanitize_html -from six import string_types, text_type +from six import string_types def validate_and_sanitize_basestring( key, val ): if not isinstance( val, string_types ): raise exceptions.RequestParameterInvalidException( '%s must be a string or unicode: %s' % ( key, str( type( val ) ) ) ) - return unicodify( sanitize_html( val, 'utf-8', 'text/html' ), 'utf-8' ) + return sanitize_html( val, 'utf-8', 'text/html' ) def validate_and_sanitize_basestring_list( key, val ): try: assert isinstance( val, list ) - return [ text_type( sanitize_html( t, 'utf-8', 'text/html' ), 'utf-8' ) for t in val ] + return [ sanitize_html( t, 'utf-8', 'text/html' ) for t in val ] except ( AssertionError, TypeError ): raise exceptions.RequestParameterInvalidException( '%s must be a list of strings: %s' % ( key, str( type( val ) ) ) )