From 354d0447df4ca884b63a1666c3953be2bdcdf76b Mon Sep 17 00:00:00 2001
From: Greg Von Kuster <greg@bx.psu.edu>
Date: Fri, 20 Mar 2009 02:49:30 -0400
Subject: [PATCH] Fixes for i18n support - all charsets set to utf-8, now able
 to paste multi-byte chars in upload form, templates use utf-8 as
 output_charset, other miscellaneous fixes.

---
 lib/galaxy/datatypes/sniff.py                 |  3 +-
 lib/galaxy/tools/actions/upload.py            | 30 +++++++--------
 lib/galaxy/util/__init__.py                   |  4 +-
 lib/galaxy/web/controllers/ucsc_proxy.py      |  2 +-
 lib/galaxy/web/framework/__init__.py          | 13 +++++--
 lib/galaxy/web/framework/base.py              | 38 +++++++++----------
 .../reports/templates/base_panels.mako        |  2 +-
 .../webapps/reports/templates/masthead.mako   |  2 +-
 scripts/paster.py                             |  1 -
 templates/base.mako                           |  2 +-
 templates/base_panels.mako                    |  2 +-
 templates/dataset/errors.tmpl                 |  2 +-
 templates/dataset/validation.tmpl             |  2 +-
 templates/root/history.mako                   |  2 +-
 templates/root/masthead.mako                  |  2 +-
 templates/root/tool_menu.mako                 |  2 +-
 templates/tool_executed.mako                  |  2 +-
 templates/tool_form.mako                      |  2 +-
 .../workflow/build_from_current_history.mako  |  2 +-
 tools/data_source/wormbase.xml                |  2 +-
 tools/data_source/wormbase_test.xml           |  2 +-
 21 files changed, 60 insertions(+), 59 deletions(-)

diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py
index da913abf86f..efc8a9d3a9b 100644
--- a/lib/galaxy/datatypes/sniff.py
+++ b/lib/galaxy/datatypes/sniff.py
@@ -21,7 +21,8 @@ def stream_to_file( stream, suffix='', prefix='', dir=None, text=False ):
         chunk = stream.read(1048576)
         if not chunk:
             break
-        os.write(fd, chunk)
+        # TODO: does this work on binary files?
+        os.write( fd, chunk.encode( "utf-8" ) )
     os.close(fd)
     return temp_name
 
diff --git a/lib/galaxy/tools/actions/upload.py b/lib/galaxy/tools/actions/upload.py
index c093fa25794..9817c4da8d2 100644
--- a/lib/galaxy/tools/actions/upload.py
+++ b/lib/galaxy/tools/actions/upload.py
@@ -105,7 +105,7 @@ class UploadToolAction( object ):
                     try:
                         data_list.append( self.add_file( trans, temp_name, 'Pasted Entry', file_type, dbkey, info="pasted entry", space_to_tab=space_to_tab ) )
                     except Exception, e:
-                        log.excception( 'exception in add_file using StringIO.StringIO( url_paste ) temp_name %s: %s' % ( str( temp_name ), str( e ) ) )
+                        log.exception( 'exception in add_file using StringIO.StringIO( url_paste ) temp_name %s: %s' % ( str( temp_name ), str( e ) ) )
                         self.remove_tempfile( temp_name )
                         return self.upload_empty( trans, job, "Error:", str( e ) )
                 else:
@@ -146,12 +146,11 @@ class UploadToolAction( object ):
 
     def add_file( self, trans, temp_name, file_name, file_type, dbkey, info=None, space_to_tab=False ):
         data_type = None
-        
         # See if we have an empty file
         if not os.path.getsize( temp_name ) > 0:
             raise BadFileException( "you attempted to upload an empty file." )
-        
-        # See if we have a gzipped file, which, if it passes our restrictions, we'll decompress on the fly.
+        # See if we have a gzipped file, which, if it passes our restrictions,
+        # we'll decompress on the fly.
         is_gzipped, is_valid = self.check_gzip( temp_name )
         if is_gzipped and not is_valid:
             raise BadFileException( "you attempted to upload an inappropriate file." )
@@ -176,7 +175,7 @@ class UploadToolAction( object ):
             shutil.move( uncompressed, temp_name )
             file_name = file_name.rstrip( '.gz' )
             data_type = 'gzip'
-
+        ext = ''
         if not data_type:
             # See if we have a zip archive
             is_zipped, is_valid, test_ext = self.check_zip( temp_name )
@@ -193,30 +192,28 @@ class UploadToolAction( object ):
                     raise BadFileException( "you must manually set the 'File Format' to either 'Binseq.zip' or 'Txtseq.zip' when uploading zip files." )
                 data_type = 'zip'
                 ext = file_type
-
         if not data_type:
             if self.check_binary( temp_name ):
-                ext = file_name.split( "." )[1].strip().lower()
-                if not( ext == 'ab1' or ext == 'scf' ):
-                    raise BadFileException( "you attempted to upload an inappropriate file." )
-                if ext == 'ab1' and file_type != 'ab1':
-                    raise BadFileException( "you must manually set the 'File Format' to 'Ab1' when uploading ab1 files." )
-                elif ext == 'scf' and file_type != 'scf':
-                    raise BadFileException( "you must manually set the 'File Format' to 'Scf' when uploading scf files." )
+                parts = file_name.split( "." )
+                if len( parts ) > 1:
+                    ext = parts[1].strip().lower()
+                    if not( ext == 'ab1' or ext == 'scf' ):
+                        raise BadFileException( "you attempted to upload an inappropriate file." )
+                    if ext == 'ab1' and file_type != 'ab1':
+                        raise BadFileException( "you must manually set the 'File Format' to 'Ab1' when uploading ab1 files." )
+                    elif ext == 'scf' and file_type != 'scf':
+                        raise BadFileException( "you must manually set the 'File Format' to 'Scf' when uploading scf files." )
                 data_type = 'binary'
-        
         if not data_type:
             # We must have a text file
             if self.check_html( temp_name ):
                 raise BadFileException( "you attempted to upload an inappropriate file." )
-
         if data_type != 'binary' and data_type != 'zip':
             if space_to_tab:
                 self.line_count = sniff.convert_newlines_sep2tabs( temp_name )
             else:
                 self.line_count = sniff.convert_newlines( temp_name )
             if file_type == 'auto':
-                log.debug("In upload, in if file_type == 'auto':")
                 ext = sniff.guess_ext( temp_name, sniff_order=trans.app.datatypes_registry.sniff_order )    
             else:
                 ext = file_type
@@ -312,7 +309,6 @@ class UploadToolAction( object ):
         if chunk is None:
             temp.close()
         return False
-
     def check_binary( self, temp_name, chunk=None ):
         if chunk is None:
             temp = open( temp_name, "U" )
diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py
index cd0d1483270..13e121e7d62 100644
--- a/lib/galaxy/util/__init__.py
+++ b/lib/galaxy/util/__init__.py
@@ -107,9 +107,9 @@ def sanitize_text(text):
 
 def sanitize_param(value):
     """Clean incoming parameters (strings or lists)"""
-    if type(value) == type('x'):
+    if isinstance( value, basestring ):
         return sanitize_text(value)
-    elif type(value) == type([]):
+    elif isinstance( value, list ):
         return map(sanitize_text, value)
     else:
         raise Exception, 'Unknown parameter type'
diff --git a/lib/galaxy/web/controllers/ucsc_proxy.py b/lib/galaxy/web/controllers/ucsc_proxy.py
index 9ad6225fcfb..c95f67fdfc6 100644
--- a/lib/galaxy/web/controllers/ucsc_proxy.py
+++ b/lib/galaxy/web/controllers/ucsc_proxy.py
@@ -121,7 +121,7 @@ beginning = '''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "h
 
 <head>
 <title>Galaxy</title>
-<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <link href="/static/style/base.css" rel="stylesheet" type="text/css" />
 <script language="javascript" type="text/javascript">
 function changeTarget(target)
diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py
index f3f37976f2b..43c9360c871 100644
--- a/lib/galaxy/web/framework/__init__.py
+++ b/lib/galaxy/web/framework/__init__.py
@@ -87,7 +87,8 @@ class WebApplication( base.WebApplication ):
         self.mako_template_lookup = mako.lookup.TemplateLookup(
             directories = [ galaxy_app.config.template_path ] ,
             module_directory = galaxy_app.config.template_cache,
-            collection_size = 500 )
+            collection_size = 500,
+            output_encoding = 'utf-8' )
         # Security helper
         self.security = galaxy_app.security
     def handle_controller_exception( self, e, trans, **kwargs ):
@@ -131,7 +132,7 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
             # Default to English
             locales = 'en'
         t = Translations.load( dirname='locale', locales=locales, domain='ginga' )
-        self.template_context.update ( dict( _=t.gettext, n_=t.gettext, N_=t.ngettext ) )
+        self.template_context.update ( dict( _=t.ugettext, n_=t.ugettext, N_=t.ungettext ) )
     @property
     def sa_session( self ):
         """
@@ -465,10 +466,12 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
             return self.fill_template_mako( filename, **kwargs )
         else:
             template = Template( file=os.path.join(self.app.config.template_path, filename), 
-                                searchList=[kwargs, self.template_context, dict(caller=self, t=self, h=webhelpers, util=util, request=self.request, response=self.response, app=self.app)] )
+                                 searchList=[kwargs, self.template_context, dict(caller=self, t=self, h=webhelpers, util=util, request=self.request, response=self.response, app=self.app)],
+                                 output_encoding='utf-8' )
             return str( template )
     def fill_template_mako( self, filename, **kwargs ):
         template = self.webapp.mako_template_lookup.get_template( filename )
+        template.output_encoding = 'utf-8' 
         data = dict( caller=self, t=self, trans=self, h=webhelpers, util=util, request=self.request, response=self.response, app=self.app )
         data.update( self.template_context )
         data.update( kwargs )
@@ -477,7 +480,9 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
         """
         Fill in a template, putting any keyword arguments on the context.
         """
-        template = Template( source=template_string, searchList=[context or kwargs, dict(caller=self)] )
+        template = Template( source=template_string,
+                             searchList=[context or kwargs, dict(caller=self)],
+                             output_encoding='utf-8' )
         return str(template)
         
 class FormBuilder( object ):
diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py
index 6dff0719d2a..5e6029868b9 100644
--- a/lib/galaxy/web/framework/base.py
+++ b/lib/galaxy/web/framework/base.py
@@ -14,8 +14,10 @@ import pkg_resources;
 pkg_resources.require( "Paste" )
 pkg_resources.require( "Routes" )
 pkg_resources.require( "flup" )
+pkg_resources.require( "WebOb" )
 
 import routes
+import webob
 
 # We will use some very basic HTTP/wsgi utilities from the paste library
 from paste.request import parse_headers, get_cookies, parse_formvars
@@ -212,7 +214,7 @@ class DefaultWebTransaction( object ):
         else:
             return None
     
-class Request( object ):
+class Request( webob.Request ):
     """
     Encapsulates an HTTP request. 
     """
@@ -220,26 +222,24 @@ class Request( object ):
         """
         Create a new request wrapping the WSGI environment `environ`
         """
-        self.environ = environ
+        ## self.environ = environ
+        webob.Request.__init__( self, environ, charset='utf-8', decode_param_names=False )
     # Properties that are computed and cached on first use
     @lazy_property
     def remote_host( self ):
         try:
-            return socket.gethostbyname( self.environ['REMOTE_ADDR'] )
+            return socket.gethostbyname( self.remote_addr )
         except socket.error:
-            return self.environ['REMOTE_ADDR']
-    @lazy_property
-    def headers( self ):
-        return dict( parse_headers( self.environ ) )
+            return self.remote_addr
     @lazy_property
     def cookies( self ):
         return get_cookies( self.environ )
     @lazy_property
     def base( self ):
-        return ( self.scheme + "://" + self.environ['HTTP_HOST'] )
-    @lazy_property
-    def params( self ):
-        return parse_formvars( self.environ )
+        return ( self.scheme + "://" + self.host )
+    ## @lazy_property
+    ## def params( self ):
+    ##     return parse_formvars( self.environ )
     @lazy_property
     def path( self ):
         return self.environ['SCRIPT_NAME'] + self.environ['PATH_INFO']
@@ -247,15 +247,15 @@ class Request( object ):
     def browser_url( self ):
         return self.base + self.path        
     # Descriptors that map properties to the associated environment
-    scheme = WSGIEnvironmentProperty( 'wsgi.url_scheme' )
-    remote_addr = WSGIEnvironmentProperty( 'REMOTE_ADDR' )
+    ## scheme = WSGIEnvironmentProperty( 'wsgi.url_scheme' )
+    ## remote_addr = WSGIEnvironmentProperty( 'REMOTE_ADDR' )
     remote_port = WSGIEnvironmentProperty( 'REMOTE_PORT' )
-    method = WSGIEnvironmentProperty( 'REQUEST_METHOD' )
-    script_name = WSGIEnvironmentProperty( 'SCRIPT_NAME' )
+    ## method = WSGIEnvironmentProperty( 'REQUEST_METHOD' )
+    ## script_name = WSGIEnvironmentProperty( 'SCRIPT_NAME' )
     protocol = WSGIEnvironmentProperty( 'SERVER_PROTOCOL' )
-    query_string = WSGIEnvironmentProperty( 'QUERY_STRING' )
-    path_info = WSGIEnvironmentProperty( 'PATH_INFO' )
-    
+    ## query_string = WSGIEnvironmentProperty( 'QUERY_STRING' )
+    ## path_info = WSGIEnvironmentProperty( 'PATH_INFO' )
+
 class Response( object ):
     """
     Describes an HTTP response. Currently very simple since the actual body
@@ -298,7 +298,7 @@ class Response( object ):
             return "%d %s" % ( exception.code, exception.title )
         else:
             return self.status
-        
+
 # ---- Utilities ------------------------------------------------------------
 
 CHUNK_SIZE = 2**16
diff --git a/lib/galaxy/webapps/reports/templates/base_panels.mako b/lib/galaxy/webapps/reports/templates/base_panels.mako
index ebee4dc28f3..69cc3b899bb 100644
--- a/lib/galaxy/webapps/reports/templates/base_panels.mako
+++ b/lib/galaxy/webapps/reports/templates/base_panels.mako
@@ -31,7 +31,7 @@
 <html lang="en">
   <head>
     <title>${self.title()}</title>
-    <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
     ${self.javascripts()}
     ${self.stylesheets()}
   </head>
diff --git a/lib/galaxy/webapps/reports/templates/masthead.mako b/lib/galaxy/webapps/reports/templates/masthead.mako
index 9c545654d46..f4a335f3cbe 100644
--- a/lib/galaxy/webapps/reports/templates/masthead.mako
+++ b/lib/galaxy/webapps/reports/templates/masthead.mako
@@ -2,7 +2,7 @@
 <html>
   <head>
     <title>Galaxy</title>
-    <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
     <link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" />
     <link href="${h.url_for('/static/style/masthead.css')}" rel="stylesheet" type="text/css" />
   </head>
diff --git a/scripts/paster.py b/scripts/paster.py
index e6736a712f5..7624257a665 100755
--- a/scripts/paster.py
+++ b/scripts/paster.py
@@ -7,7 +7,6 @@ top level directly.
 
 import os, sys
 
-sys.setdefaultencoding( 'utf_8' )
 assert sys.version_info[:2] >= ( 2, 4 )
 
 new_path = [ os.path.join( os.getcwd(), "lib" ) ]
diff --git a/templates/base.mako b/templates/base.mako
index ad343b38195..ad8526f55bf 100644
--- a/templates/base.mako
+++ b/templates/base.mako
@@ -4,7 +4,7 @@
 
 <head>
 <title>a ${self.title()}</title>
-<meta http-equiv="Content-Type" content="text/html; charset=${_('iso-8859-1')}" />
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 ${self.stylesheets()}
 ${self.javascripts()}
 </head>
diff --git a/templates/base_panels.mako b/templates/base_panels.mako
index 99dd532519f..6f88bae855b 100644
--- a/templates/base_panels.mako
+++ b/templates/base_panels.mako
@@ -42,7 +42,7 @@
 <html>
     <head>
     <title>${self.title()}</title>
-    <meta http-equiv="Content-Type" content="text/html; charset=${_('iso-8859-1')}" />
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
     ${self.javascripts()}
     ${self.stylesheets()}
     </head>
diff --git a/templates/dataset/errors.tmpl b/templates/dataset/errors.tmpl
index 4746869cc08..86f69327a34 100644
--- a/templates/dataset/errors.tmpl
+++ b/templates/dataset/errors.tmpl
@@ -2,7 +2,7 @@
 <html>
     <head>
         <title>Dataset generation errors</title>
-        <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
         <link href="/static/style/base.css" rel="stylesheet" type="text/css" />
         <style>
             pre
diff --git a/templates/dataset/validation.tmpl b/templates/dataset/validation.tmpl
index 62488639374..0407b247a31 100644
--- a/templates/dataset/validation.tmpl
+++ b/templates/dataset/validation.tmpl
@@ -3,7 +3,7 @@
 
 <head>
 <title>Galaxy</title>
-<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <link href="/static/style/base.css" rel="stylesheet" type="text/css" />
 <script type="text/javascript" src="/static/universe.js">var dummy1=0;</script>
 </head>
diff --git a/templates/root/history.mako b/templates/root/history.mako
index 1f05b39cbb4..0f4cf21430b 100644
--- a/templates/root/history.mako
+++ b/templates/root/history.mako
@@ -11,7 +11,7 @@
 <!-- running: do not change this comment, used by TwillTestCase.wait -->
 %endif
 
-<meta http-equiv="Content-Type" content="text/html; charset=${_('iso-8859-1')}" />
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta http-equiv="Pragma" content="no-cache">
 <link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" />
 <link href="${h.url_for('/static/style/history.css')}" rel="stylesheet" type="text/css" />
diff --git a/templates/root/masthead.mako b/templates/root/masthead.mako
index c9a26e24b79..c2d66dc080c 100644
--- a/templates/root/masthead.mako
+++ b/templates/root/masthead.mako
@@ -5,7 +5,7 @@
 
 <head>
 <title>Galaxy</title>
-<meta http-equiv="Content-Type" content="text/html; charset=${_('iso-8859-1')}" />
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" />
 <link href="${h.url_for('/static/style/masthead.css')}" rel="stylesheet" type="text/css" />
 </head>
diff --git a/templates/root/tool_menu.mako b/templates/root/tool_menu.mako
index d4385de800d..9e1fc8db854 100644
--- a/templates/root/tool_menu.mako
+++ b/templates/root/tool_menu.mako
@@ -49,7 +49,7 @@
 <html>
     <head>
         <title>${_('Galaxy Tools')}</title>
-        <meta http-equiv="Content-Type" content="text/html; charset=${_('iso-8859-1')}" />
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
         <link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" />
         <link href="${h.url_for('/static/style/tool_menu.css')}" rel="stylesheet" type="text/css" />
 
diff --git a/templates/tool_executed.mako b/templates/tool_executed.mako
index 9fd7ed1bc08..a241ce9d7c4 100644
--- a/templates/tool_executed.mako
+++ b/templates/tool_executed.mako
@@ -3,7 +3,7 @@
 
 <head>
 <title>Galaxy</title>
-<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" />
 <script type="text/javascript">
   var inside_galaxy_frameset = false;
diff --git a/templates/tool_form.mako b/templates/tool_form.mako
index 83cc57880f0..fbe41551268 100644
--- a/templates/tool_form.mako
+++ b/templates/tool_form.mako
@@ -9,7 +9,7 @@ from galaxy.util.expressions import ExpressionContext
 
 <head>
 <title>Galaxy</title>
-<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" />
 <script type='text/javascript' src="${h.url_for('/static/scripts/jquery.js')}"> </script>
 <script type="text/javascript">
diff --git a/templates/workflow/build_from_current_history.mako b/templates/workflow/build_from_current_history.mako
index 2cfadc4161d..e22ebc3b843 100644
--- a/templates/workflow/build_from_current_history.mako
+++ b/templates/workflow/build_from_current_history.mako
@@ -6,7 +6,7 @@
 
 <head>
 <title>Galaxy</title>
-<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <link href="${h.url_for('/static/style/history.css')}" rel="stylesheet" type="text/css" />
 <link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" />
 
diff --git a/tools/data_source/wormbase.xml b/tools/data_source/wormbase.xml
index abdb99d1d97..74cccb30bf6 100644
--- a/tools/data_source/wormbase.xml
+++ b/tools/data_source/wormbase.xml
@@ -2,7 +2,7 @@
 <tool name="Wormbase" id="wormbase" tool_type="data_source" URL_method="post">
     <description>server</description>
     <command interpreter="python">data_source.py $output</command>
-    <inputs action="http://www.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" method="get" target="_top"> 
+    <inputs action="http://www.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" target="_top"> 
         <display>go to Wormbase server $GALAXY_URL</display>
         <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=wormbase" />
     </inputs>
diff --git a/tools/data_source/wormbase_test.xml b/tools/data_source/wormbase_test.xml
index 73efe44aad7..32b32519f43 100644
--- a/tools/data_source/wormbase_test.xml
+++ b/tools/data_source/wormbase_test.xml
@@ -2,7 +2,7 @@
 <tool name="Wormbase" id="wormbase_test" tool_type="data_source" URL_method="post">
 	<description>test server</description>
 	<command interpreter="python">data_source.py $output</command>
-	<inputs action="http://dev.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" method="get" target="_top"> 
+	<inputs action="http://dev.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" target="_top"> 
 		<display>go to Wormbase test server $GALAXY_URL</display>
 		<param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=wormbase_test" />
 	</inputs>