diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample index fcd4d6e73f3..cba7dbe4c5b 100644 --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -580,6 +580,13 @@ nglims_config_file = tool-data/nglims.yaml # requests. #nginx_upload_path = False +# Galaxy can also use nginx_upload_module to receive files staged out upon job +# completion by remote job runners (i.e. Pulsar) that initiate staging +# operations on the remote end. See the Galaxy nginx documentation for the +# corresponding nginx configuration. +#nginx_upload_job_files_store = False +#nginx_upload_job_files_path = False + # Have Galaxy manage dynamic proxy component for routing requests to other # services based on Galaxy's session cookie. It will attempt to do this by # default though you do need to install node+npm and do an npm install from diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 4ba5e7d8547..34472c7bdaf 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -293,6 +293,8 @@ class Configuration( object ): self.nginx_x_archive_files_base = kwargs.get( 'nginx_x_archive_files_base', False ) self.nginx_upload_store = kwargs.get( 'nginx_upload_store', False ) self.nginx_upload_path = kwargs.get( 'nginx_upload_path', False ) + self.nginx_upload_job_files_store = kwargs.get( 'nginx_upload_job_files_store', False ) + self.nginx_upload_job_files_path = kwargs.get( 'nginx_upload_job_files_path', False ) if self.nginx_upload_store: self.nginx_upload_store = os.path.abspath( self.nginx_upload_store ) self.object_store = kwargs.get( 'object_store', 'disk' ) diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index f1da51ff963..763e41c9a21 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -342,7 +342,12 @@ class PulsarJobRunner( AsynchronousJobRunner ): encoded_job_id = self.app.security.encode_id(job_id) job_key = self.app.security.encode_id( job_id, kind="jobs_files" ) - files_endpoint = "%s/api/jobs/%s/files?job_key=%s" % ( + endpoint_base = "%s/api/jobs/%s?job_key=%s" + if self.app.config.nginx_upload_job_files_path: + endpoint_base = "%s" + \ + self.app.config.nginx_upload_job_files_path + \ + "?job_id=%s&job_key=%s" + files_endpoint = endpoint_base % ( self.galaxy_url, encoded_job_id, job_key diff --git a/lib/galaxy/webapps/galaxy/api/job_files.py b/lib/galaxy/webapps/galaxy/api/job_files.py index 27445beac38..4b21c1ceb5b 100644 --- a/lib/galaxy/webapps/galaxy/api/job_files.py +++ b/lib/galaxy/webapps/galaxy/api/job_files.py @@ -83,9 +83,21 @@ class JobFilesAPIController( BaseAPIController ): self.__check_job_can_write_to_path( trans, job, path ) # Is this writing an unneeded file? Should this just copy in Python? - input_file = payload.get( "file", payload.get( "__file", None ) ).file + if '__file_path' in payload: + file_path = payload.get( '__file_path' ) + upload_store = trans.app.config.nginx_upload_job_files_store + assert upload_store, ( "Request appears to have been processed by" + " nginx_upload_module but Galaxy is not" + " configured to recognize it" ) + assert file_path.startswith( upload_store ), \ + ( "Filename provided by nginx (%s) is not in correct" + " directory (%s)" % ( file_path, upload_store ) ) + input_file = open( file_path ) + else: + input_file = payload.get( "file", + payload.get( "__file", None ) ).file try: - shutil.copyfile( input_file.name, path ) + shutil.move( input_file.name, path ) finally: input_file.close() return {"message": "ok"}