Merge pull request #110 from natefoo/release_15.03

[STABLE] Fix job file stageout interruptibility

config/galaxy.ini.sample

@@ -580,6 +580,13 @@ nglims_config_file = tool-data/nglims.yaml
 # requests.
 #nginx_upload_path = False
 
+# Galaxy can also use nginx_upload_module to receive files staged out upon job
+# completion by remote job runners (i.e. Pulsar) that initiate staging
+# operations on the remote end.  See the Galaxy nginx documentation for the
+# corresponding nginx configuration.
+#nginx_upload_job_files_store = False
+#nginx_upload_job_files_path = False
+
 # Have Galaxy manage dynamic proxy component for routing requests to other
 # services based on Galaxy's session cookie.  It will attempt to do this by
 # default though you do need to install node+npm and do an npm install from

lib/galaxy/config.py

@@ -293,6 +293,8 @@ class Configuration( object ):
         self.nginx_x_archive_files_base = kwargs.get( 'nginx_x_archive_files_base', False )
         self.nginx_upload_store = kwargs.get( 'nginx_upload_store', False )
         self.nginx_upload_path = kwargs.get( 'nginx_upload_path', False )
+        self.nginx_upload_job_files_store = kwargs.get( 'nginx_upload_job_files_store', False )
+        self.nginx_upload_job_files_path = kwargs.get( 'nginx_upload_job_files_path', False )
         if self.nginx_upload_store:
             self.nginx_upload_store = os.path.abspath( self.nginx_upload_store )
         self.object_store = kwargs.get( 'object_store', 'disk' )

lib/galaxy/jobs/runners/pulsar.py

@@ -342,7 +342,12 @@ class PulsarJobRunner( AsynchronousJobRunner ):
 
         encoded_job_id = self.app.security.encode_id(job_id)
         job_key = self.app.security.encode_id( job_id, kind="jobs_files" )
-        files_endpoint = "%s/api/jobs/%s/files?job_key=%s" % (
+        endpoint_base = "%s/api/jobs/%s?job_key=%s"
+        if self.app.config.nginx_upload_job_files_path:
+            endpoint_base = "%s" + \
+                            self.app.config.nginx_upload_job_files_path + \
+                            "?job_id=%s&job_key=%s"
+        files_endpoint = endpoint_base % (
             self.galaxy_url,
             encoded_job_id,
             job_key

lib/galaxy/webapps/galaxy/api/job_files.py

@@ -83,9 +83,21 @@ class JobFilesAPIController( BaseAPIController ):
         self.__check_job_can_write_to_path( trans, job, path )
 
         # Is this writing an unneeded file? Should this just copy in Python?
-        input_file = payload.get( "file", payload.get( "__file", None ) ).file
+        if '__file_path' in payload:
+            file_path = payload.get( '__file_path' )
+            upload_store = trans.app.config.nginx_upload_job_files_store
+            assert upload_store, ( "Request appears to have been processed by"
+                                   " nginx_upload_module but Galaxy is not"
+                                   " configured to recognize it" )
+            assert file_path.startswith( upload_store ), \
+                    ( "Filename provided by nginx (%s) is not in correct"
+                      " directory (%s)" % ( file_path, upload_store ) )
+            input_file = open( file_path )
+        else:
+            input_file = payload.get( "file",
+                                      payload.get( "__file", None ) ).file
         try:
-            shutil.copyfile( input_file.name, path )
+            shutil.move( input_file.name, path )
         finally:
             input_file.close()
         return {"message": "ok"}