From d14eb4d6538996c2ec90a9853d1b5c8efd85150a Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 29 Sep 2009 17:14:20 -0400 Subject: [PATCH] Reintroduce nginx upload module support. http://www.grid.net.ru/nginx/upload.en.html The following config variables are added to universe_wsgi.ini: nginx_upload_store = Path to nginx upload store ex: = database/upload_store nginx_upload_path = URL (from root of the Galaxy server) to direct upload POSTs to ex: = /_upload The following nginx config supports such a configuration: location /_upload { upload_store /path/to/galaxy/database/upload_store; upload_pass_form_field "tool_id"; upload_pass_form_field "tool_state"; upload_pass_form_field "async_datasets"; upload_pass_form_field "^files_[0-9]+\|.*"; upload_pass_form_field "file_type"; upload_pass_form_field "dbkey"; upload_pass_form_field "runtool_btn"; upload_pass_form_field "ajax_upload"; upload_pass_form_field "upload_option"; upload_pass_form_field "library_id"; upload_pass_form_field "folder_id"; upload_pass_form_field "message"; upload_pass_form_field "roles"; upload_set_form_field "__${upload_field_name}__is_composite" "true"; upload_set_form_field "__${upload_field_name}__keys" "name path"; upload_set_form_field "${upload_field_name}_name" "$upload_file_name"; upload_set_form_field "${upload_field_name}_path" "$upload_tmp_path"; upload_pass_args on; upload_pass /_upload_done; } location /_upload_done { set $dst /tool_runner/index; if ($args ~ nginx_redir=([^&]+)) { set $dst $1; } rewrite "" $dst; } --- lib/galaxy/config.py | 7 ++--- lib/galaxy/tools/__init__.py | 6 ++++- lib/galaxy/tools/actions/upload_common.py | 2 ++ lib/galaxy/tools/parameters/basic.py | 27 +++++++++---------- lib/galaxy/web/controllers/library.py | 5 ++++ lib/galaxy/web/controllers/library_admin.py | 5 ++++ templates/admin/library/upload.mako | 2 +- templates/library/library_dataset_common.mako | 4 +-- templates/library/upload.mako | 2 +- tools/data_source/upload.xml | 2 +- 10 files changed, 39 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 6c1670487f5..1be2811b9e1 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -89,9 +89,10 @@ class Configuration( object ): raise ConfigurationError( "user_library_import_dir specified in config (%s) does not exist" % self.user_library_import_dir ) # Configuration options for taking advantage of nginx features self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False ) - self.nginx_upload_location = kwargs.get( 'nginx_upload_store', False ) - if self.nginx_upload_location: - self.nginx_upload_location = os.path.abspath( self.nginx_upload_location ) + self.nginx_upload_store = kwargs.get( 'nginx_upload_store', False ) + self.nginx_upload_path = kwargs.get( 'nginx_upload_path', False ) + if self.nginx_upload_store: + self.nginx_upload_store = os.path.abspath( self.nginx_upload_store ) # Parse global_conf and save the parser global_conf = kwargs.get( 'global_conf', None ) global_conf_parser = ConfigParser.ConfigParser() diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index cf341b17737..4f03768fdf4 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -464,7 +464,11 @@ class Tool: if input_elem: # Handle properties of the input form self.check_values = util.string_as_bool( input_elem.get("check_values", "true") ) - self.action = input_elem.get( "action", "/tool_runner/index") + self.nginx_upload = util.string_as_bool( input_elem.get( "nginx_upload", "false" ) ) + if self.nginx_upload and self.app.config.nginx_upload_path: + self.action = input_elem.get( "action", self.app.config.nginx_upload_path + "?nginx_redir=/tool_runner/index" ) + else: + self.action = input_elem.get( "action", "/tool_runner/index") self.target = input_elem.get( "target", "galaxy_main" ) self.method = input_elem.get( "method", "post" ) # Parse the actual parameters diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index c2e45d94456..dbf3f950c42 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -23,6 +23,8 @@ def persist_uploads( params ): f.file.close() upload_dataset['file_data'] = dict( filename = f.filename, local_filename = local_filename ) + elif type( f ) == dict and 'filename' and 'local_filename' not in f: + raise Exception( 'Uploaded file was encoded in a way not understood by Galaxy.' ) if upload_dataset['url_paste'].strip() != '': upload_dataset['url_paste'] = datatypes.sniff.stream_to_file( StringIO.StringIO( upload_dataset['url_paste'] ), prefix="strio_url_paste_" )[0] else: diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 9790ded96db..3bfbc8dc495 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -304,22 +304,21 @@ class FileToolParameter( ToolParameter ): def get_html_field( self, trans=None, value=None, other_values={} ): return form_builder.FileField( self.name, ajax = self.ajax, value = value ) def from_html( self, value, trans=None, other_values={} ): - # TODO: Fix nginx upload module support # Middleware or proxies may encode files in special ways (TODO: this # should be pluggable) - #if type( value ) == dict: - # upload_location = self.tool.app.config.nginx_upload_location - # assert upload_location, \ - # "Request appears to have been processed by nginx_upload_module \ - # but Galaxy is not configured to recgonize it" - # # Check that the file is in the right location - # local_filename = os.path.abspath( value['path'] ) - # assert local_filename.startswith( upload_location ), \ - # "Filename provided by nginx is not in correct directory" - # value = Bunch( - # filename = value["name"], - # local_filename = local_filename - # ) + if type( value ) == dict: + upload_store = self.tool.app.config.nginx_upload_store + assert upload_store, \ + "Request appears to have been processed by nginx_upload_module \ + but Galaxy is not configured to recognize it" + # Check that the file is in the right location + local_filename = os.path.abspath( value['path'] ) + assert local_filename.startswith( upload_store ), \ + "Filename provided by nginx is not in correct directory" + value = dict( + filename = value["name"], + local_filename = local_filename + ) return value def get_required_enctype( self ): """ diff --git a/lib/galaxy/web/controllers/library.py b/lib/galaxy/web/controllers/library.py index 1bbdf9fd8aa..de0442d6725 100644 --- a/lib/galaxy/web/controllers/library.py +++ b/lib/galaxy/web/controllers/library.py @@ -792,8 +792,13 @@ class Library( BaseController ): # Send the current history to the form to enable importing datasets from history to library history = trans.get_history() history.refresh() + # If we're using nginx upload, override the form action + action = web.url_for( controller='library', action='library_dataset_dataset_association' ) + if upload_option == 'upload_file' and trans.app.config.nginx_upload_path: + action = web.url_for( trans.app.config.nginx_upload_path ) + '?nginx_redir=' + action return trans.fill_template( '/library/upload.mako', upload_option=upload_option, + action=action, library_id=library_id, folder_id=folder_id, replace_dataset=replace_dataset, diff --git a/lib/galaxy/web/controllers/library_admin.py b/lib/galaxy/web/controllers/library_admin.py index 6865f395d17..ca1fbd7193d 100644 --- a/lib/galaxy/web/controllers/library_admin.py +++ b/lib/galaxy/web/controllers/library_admin.py @@ -487,8 +487,13 @@ class LibraryAdmin( BaseController ): # Send the current history to the form to enable importing datasets from history to library history = trans.get_history() history.refresh() + # If we're using nginx upload, override the form action + action = web.url_for( controller='library_admin', action='library_dataset_dataset_association' ) + if upload_option == 'upload_file' and trans.app.config.nginx_upload_path: + action = web.url_for( trans.app.config.nginx_upload_path ) + '?nginx_redir=' + action return trans.fill_template( '/admin/library/upload.mako', upload_option=upload_option, + action=action, library_id=library_id, folder_id=folder_id, replace_dataset=replace_dataset, diff --git a/templates/admin/library/upload.mako b/templates/admin/library/upload.mako index 14a5c9fed70..3c5ec1c4e71 100644 --- a/templates/admin/library/upload.mako +++ b/templates/admin/library/upload.mako @@ -32,4 +32,4 @@ ${render_msg( msg, messagetype )} %endif -${render_upload_form( 'library_admin', upload_option, library_id, folder_id, replace_dataset, file_formats, dbkeys, roles, history, )} +${render_upload_form( 'library_admin', upload_option, action, library_id, folder_id, replace_dataset, file_formats, dbkeys, roles, history, )} diff --git a/templates/library/library_dataset_common.mako b/templates/library/library_dataset_common.mako index f78c39232c1..5e12404a771 100644 --- a/templates/library/library_dataset_common.mako +++ b/templates/library/library_dataset_common.mako @@ -1,4 +1,4 @@ -<%def name="render_upload_form( controller, upload_option, library_id, folder_id, replace_dataset, file_formats, dbkeys, roles, history )"> +<%def name="render_upload_form( controller, upload_option, action, library_id, folder_id, replace_dataset, file_formats, dbkeys, roles, history )"> <% import os, os.path %> %if upload_option in [ 'upload_file', 'upload_directory' ]:
@@ -8,7 +8,7 @@
Upload a directory of files
%endif
-
+ diff --git a/templates/library/upload.mako b/templates/library/upload.mako index 3993ca8d3fb..2e8a161a8a8 100644 --- a/templates/library/upload.mako +++ b/templates/library/upload.mako @@ -32,4 +32,4 @@ ${render_msg( msg, messagetype )} %endif -${render_upload_form( 'library', upload_option, library_id, folder_id, replace_dataset, file_formats, dbkeys, roles, history, )} +${render_upload_form( 'library', upload_option, action, library_id, folder_id, replace_dataset, file_formats, dbkeys, roles, history, )} diff --git a/tools/data_source/upload.xml b/tools/data_source/upload.xml index 81756f52f88..60f55be9c74 100644 --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -14,7 +14,7 @@ ${output.dataset.dataset.id}:${output} #end while - +