diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index d3cc43cbb7c..0da7f7a8167 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -25,11 +25,6 @@ from sqlalchemy.ext import hybrid from sqlalchemy import types from sqlalchemy import type_coerce -try: - import pexpect -except ImportError: - pexpect = None - import galaxy.model.orm.now import galaxy.model.metadata import galaxy.security.passwords @@ -60,9 +55,6 @@ _datatypes_registry = None # this be unlimited - filter in Python if over this limit. MAX_IN_FILTER_LENGTH = 100 -PEXPECT_IMPORT_MESSAGE = ('The Python pexpect package is required to use this ' - 'feature, please install it') - class NoConverterException(Exception): def __init__(self, value): @@ -4621,30 +4613,6 @@ class Sample( object, Dictifiable ): untransferred_datasets.append( dataset ) return untransferred_datasets - def get_untransferred_dataset_size( self, filepath, scp_configs ): - def print_ticks( d ): - pass - if pexpect is None: - return PEXPECT_IMPORT_MESSAGE - error_msg = 'Error encountered in determining the file size of %s on the external_service.' % filepath - if not scp_configs['host'] or not scp_configs['user_name'] or not scp_configs['password']: - return error_msg - login_str = '%s@%s' % ( scp_configs['user_name'], scp_configs['host'] ) - cmd = 'ssh %s "du -sh \'%s\'"' % ( login_str, filepath ) - try: - output = pexpect.run( cmd, - events={ '.ssword:*': scp_configs['password'] + '\r\n', - pexpect.TIMEOUT: print_ticks}, - timeout=10 ) - except Exception: - return error_msg - # cleanup the output to get just the file size - return output.replace( filepath, '' )\ - .replace( 'Password:', '' )\ - .replace( "'s password:", '' )\ - .replace( login_str, '' )\ - .strip() - @property def run_details( self ): # self.runs is a list of SampleRunAssociations ordered descending on update_time. diff --git a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py index dd917846dbd..01c9d5e6b78 100644 --- a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py @@ -9,15 +9,7 @@ from galaxy.web.form_builder import build_select_field from galaxy.web.framework.helpers import time_ago, grids from .requests_common import RequestsGrid, invalid_id_redirect from markupsafe import escape -import amqp -try: - import pexpect -except ImportError: - pexpect = None - -PEXPECT_IMPORT_MESSAGE = ('The Python pexpect package is required to use this ' - 'feature, please install it') log = logging.getLogger( __name__ ) @@ -371,207 +363,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): action='manage_datasets', sample_id=sample_id ) ) - @web.expose - @web.require_admin - def select_datasets_to_transfer( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - request_id = kwd.get( 'request_id', None ) - external_service_id = kwd.get( 'external_service_id', None ) - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - # Load the data transfer settings - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - selected_datasets_to_transfer = util.restore_text( params.get( 'selected_datasets_to_transfer', '' ) ) - if selected_datasets_to_transfer: - selected_datasets_to_transfer = selected_datasets_to_transfer.split(',') - else: - selected_datasets_to_transfer = [] - sample_id = kwd.get( 'sample_id', 'none' ) - sample_id_select_field = self.__build_sample_id_select_field( trans, request, sample_id ) - if sample_id != 'none': - sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) ) - else: - sample = None - # The __get_files() method redirects here with a status of 'error' and a message if there - # was a problem retrieving the files. - if params.get( 'select_datasets_to_transfer_button', False ): - # Get the sample that was sequenced to produce these datasets. - if sample_id == 'none': - del kwd[ 'select_datasets_to_transfer_button' ] - message = 'Select the sample that was sequenced to produce the datasets you want to transfer.' - kwd[ 'message' ] = message - kwd[ 'status' ] = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='select_datasets_to_transfer', - **kwd ) ) - if not sample.library: - # Display an error if a sample has been selected that - # has not yet been associated with a destination library. - message = 'Select a target data library and folder for the sample before selecting the datasets.' - status = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_common', - action='edit_samples', - cntrller='requests_admin', - id=trans.security.encode_id( request.id ), - status=status, - message=message ) ) - # Save the sample datasets - sample_dataset_file_names = self.__create_sample_datasets( trans, sample, selected_datasets_to_transfer, external_service ) - if sample_dataset_file_names: - message = 'Datasets (%s) have been selected for sample (%s)' % \ - ( str( sample_dataset_file_names )[1:-1].replace( "'", "" ), sample.name ) - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - request_id=request_id, - sample_id=sample_id, - message=message, - status=status ) ) - return trans.fill_template( '/admin/requests/select_datasets_to_transfer.mako', - cntrller='requests_admin', - request=request, - external_service=external_service, - scp_configs=scp_configs, - sample=sample, - sample_id_select_field=sample_id_select_field, - status=status, - message=message ) - - @web.json - def get_file_details( self, trans, request_id, external_service_id, folder_path ): - def print_ticks( d ): - # pexpect timeout method - pass - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - if pexpect is None: - return PEXPECT_IMPORT_MESSAGE - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - cmd = 'ssh %s@%s "ls -oghp \'%s\'"' % ( scp_configs[ 'user_name' ], - scp_configs[ 'host' ], - folder_path ) - # Handle the authentication message if ssh keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', - '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - for password_str in [ 'Password:\r\n', 'password:\r\n' ]: - # Eliminate the output created using ssh from the tree - if password_str in output: - output = output.replace( password_str, '' ) - return text_type( output.replace( '\r\n', '
' ) ) - - @web.json - def open_folder( self, trans, request_id, external_service_id, key ): - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - folder_path = key - files_list = self.__get_files( trans, request, external_service, folder_path ) - folder_contents = [] - for filename in files_list: - is_folder = False - if filename and filename[-1] == os.sep: - is_folder = True - if filename: - full_path = os.path.join( folder_path, filename ) - node = { "title": filename, - "isFolder": is_folder, - "isLazy": is_folder, - "tooltip": full_path, - "key": full_path } - folder_contents.append( node ) - return folder_contents - - def __get_files( self, trans, request, external_service, folder_path ): - # Retrieves the filenames to be transferred from the remote host. - ok = True - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - if not scp_configs[ 'host' ] or not scp_configs[ 'user_name' ] or not scp_configs[ 'password' ]: - status = 'error' - message = "Error in external service login information." - ok = False - - def print_ticks( d ): - pass - cmd = 'ssh %s@%s "ls -p \'%s\'"' % ( scp_configs[ 'user_name' ], scp_configs[ 'host' ], folder_path ) - # Handle the authentication message if keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - if pexpect is not None: - output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', - '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - if 'No such file or directory' in output: - status = 'error' - message = "No folder named (%s) exists on the external service." % folder_path - ok = False - else: - status = 'error' - message = PEXPECT_IMPORT_MESSAGE - ok = False - if ok: - if 'assword:' in output: - # Eliminate the output created using ssh from the tree - output_as_list = output.splitlines()[ 1: ] - else: - output_as_list = output.splitlines() - return output_as_list - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='select_datasets_to_transfer', - request_id=trans.security.encode_id( request.id ), - external_service_id=trans.security.encode_id( external_service.id ), - status=status, - message=message ) ) - - def __create_sample_datasets( self, trans, sample, selected_datasets_to_transfer, external_service ): - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - sample_dataset_file_names = [] - if selected_datasets_to_transfer: - for filepath in selected_datasets_to_transfer: - # FIXME: handle folder selection - ignore folders for now - if filepath[-1] != os.sep: - name = self.__rename_dataset( sample, filepath.split( '/' )[-1], scp_configs ) - status = trans.app.model.SampleDataset.transfer_status.NOT_STARTED - size = sample.get_untransferred_dataset_size( filepath, scp_configs ) - sample_dataset = trans.model.SampleDataset( sample=sample, - file_path=filepath, - status=status, - name=name, - error_msg='', - size=size, - external_service=external_service ) - trans.sa_session.add( sample_dataset ) - trans.sa_session.flush() - sample_dataset_file_names.append( str( sample_dataset.name ) ) - return sample_dataset_file_names - - def __rename_dataset( self, sample, filepath, scp_configs ): - name = filepath.split( '/' )[-1] - options = sample.request.type.rename_dataset_options - option = scp_configs.get( 'rename_dataset', options.NO ) - if option == options.SAMPLE_NAME: - new_name = sample.name + '_' + name - if option == options.EXPERIMENT_AND_SAMPLE_NAME: - new_name = sample.request.name + '_' + sample.name + '_' + name - if option == options.EXPERIMENT_NAME: - new_name = sample.request.name + '_' + name - else: - new_name = name - return util.sanitize_for_filename( new_name ) - def __ensure_library_add_permission( self, trans, target_library, target_folder ): """ Ensures the current admin user has ADD_LIBRARY permission on the target data library and folder. @@ -594,92 +385,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): if flush_needed: trans.sa_session.flush() - def __create_data_transfer_messages( self, trans, sample, selected_sample_datasets ): - """ - Creates the xml messages to send to the rabbitmq server. It returns a dictionary of messages - keyed by the external service used to transfer the datasets - """ - # Create the xml message based on the following template - xml = \ - ''' - %(GALAXY_HOST)s - %(API_KEY)s - %(DATA_HOST)s - %(DATA_USER)s - %(DATA_PASSWORD)s - %(REQUEST_ID)s - %(SAMPLE_ID)s - %(LIBRARY_ID)s - %(FOLDER_ID)s - %(DATASETS)s - ''' - dataset_xml = \ - ''' - %(ID)s - %(NAME)s - %(FILE)s - ''' - # Here we group all the sample_datasets by the external service used to transfer them. - # The idea is to bundle up the sample_datasets which uses the same external service and - # send a single AMQP message to the galaxy_listener - dataset_elements = {} - for sample_dataset in selected_sample_datasets: - external_service = sample_dataset.external_service - if sample_dataset.status == trans.app.model.SampleDataset.transfer_status.NOT_STARTED: - if external_service not in dataset_elements: - dataset_elements[ external_service ] = '' - dataset_elements[ external_service ] += dataset_xml % dict( ID=str( sample_dataset.id ), - NAME=sample_dataset.name, - FILE=sample_dataset.file_path ) - # update the dataset transfer status - sample_dataset.status = trans.app.model.SampleDataset.transfer_status.IN_QUEUE - trans.sa_session.add( sample_dataset ) - trans.sa_session.flush() - # Finally prepend the external service info to the sets of sample datasets - messages = [] - for external_service, dataset_elem in dataset_elements.items(): - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - # Check data transfer settings - err_msg = self.__validate_data_transfer_settings( trans, sample.request.type, scp_configs ) - if err_msg: - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - sample_id=trans.security.encode_id( sample.id ), - status='error', - message=err_msg ) ) - message = xml % dict( GALAXY_HOST=trans.request.host, - API_KEY=trans.user.api_keys[0].key, - DATA_HOST=scp_configs[ 'host' ], - DATA_USER=scp_configs[ 'user_name' ], - DATA_PASSWORD=scp_configs[ 'password' ], - REQUEST_ID=str( sample.request.id ), - SAMPLE_ID=str( sample.id ), - LIBRARY_ID=str( sample.library.id ), - FOLDER_ID=str( sample.folder.id ), - DATASETS=dataset_elem ) - messages.append( message.replace( '\n', '' ).replace( '\r', '' ) ) - return messages - - def __validate_data_transfer_settings( self, trans, request_type, scp_configs ): - err_msg = '' - # check the external service login info - if not scp_configs.get( 'host', '' ) or \ - not scp_configs.get( 'user_name', '' ) or \ - not scp_configs.get( 'password', '' ): - err_msg += "Error in external service login information. " - if not trans.user.api_keys: - err_msg += "Set your API Key in your User Preferences to transfer datasets. " - # Check if library_import_dir is set - if not trans.app.config.library_import_dir: - err_msg = "'The library_import_dir' setting is not correctly set in the Galaxy config file. " - # Check the RabbitMQ server settings in the config file - for k, v in trans.app.config.amqp.items(): - if not v: - err_msg += 'Set RabbitMQ server settings in the "galaxy_amqp" section of the Galaxy config file, specifically "%s" is not set.' % k - break - return err_msg - @web.expose @web.require_admin def initiate_data_transfer( self, trans, sample_id, sample_datasets=[], sample_dataset_id='' ): @@ -721,35 +426,8 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): external_service=external_service, external_service_type=external_service_type ) else: - # TODO: Using RabbitMq for now, but eliminate this entire block when we replace RabbitMq with Galaxy's - # own messaging engine. We're holding off on using the new way to transfer files manually until we - # implement a Galaxy-proprietary messaging engine because the deferred job plugins currently perform - # constant db hits to check for deferred jobs that are not in a finished state. - # Create the message - messages = self.__create_data_transfer_messages( trans, sample, sample_datasets ) - # Send the messages - for rmq_msg in messages: - try: - conn = amqp.Connection( host=trans.app.config.amqp[ 'host' ] + ":" + trans.app.config.amqp[ 'port' ], - userid=trans.app.config.amqp[ 'userid' ], - password=trans.app.config.amqp[ 'password' ], - virtual_host=trans.app.config.amqp[ 'virtual_host' ]) - chan = conn.channel() - msg = amqp.Message( rmq_msg, - content_type='text/plain', - application_headers={ 'msg_type': 'data_transfer' } ) - msg.properties[ "delivery_mode" ] = 2 - chan.basic_publish( msg, - exchange=trans.app.config.amqp[ 'exchange' ], - routing_key=trans.app.config.amqp[ 'routing_key' ] ) - chan.close() - conn.close() - except Exception, e: - message = "Error sending the data transfer message to the Galaxy AMQP message queue:
%s" % str(e) - status = "error" - if not message: - message = "%i datasets have been queued for transfer from the external service." % len( sample_datasets ) - status = "done" + message = "Message queue transfer is no longer supported, please set enable_beta_job_managers = True in galaxy.ini" + status = "error" return trans.response.send_redirect( web.url_for( controller='requests_admin', action='manage_datasets', sample_id=trans.security.encode_id( sample.id ), diff --git a/templates/admin/requests/select_datasets_to_transfer.mako b/templates/admin/requests/select_datasets_to_transfer.mako deleted file mode 100644 index e81b9c38b44..00000000000 --- a/templates/admin/requests/select_datasets_to_transfer.mako +++ /dev/null @@ -1,148 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/requests/common/common.mako" import="render_sample_datasets" /> -<%namespace file="/requests/common/common.mako" import="common_javascripts" /> - -<%def name="javascripts()"> - ${parent.javascripts()} - ${common_javascripts()} - - -${h.js( "libs/jquery/jquery-ui", "libs/jquery/jquery.cookie", "libs/jquery/jquery.dynatree" )} -${h.css( "dynatree_skin/ui.dynatree" )} - - - -<% - is_admin = cntrller == 'requests_admin' and trans.user_is_admin() - can_transfer_datasets = is_admin and sample.untransferred_dataset_files and sample.library and sample.folder -%> - -

- - -%if not sample: -
- Select a sample before selecting datasets to transfer -
-%endif - -%if request.samples_without_library_destinations: -

- Select a target data library and folder for a sample before selecting its datasets to transfer from the external service -

-%endif - -%if message: - ${render_msg( message, status )} -%endif - -
-
Select datasets to transfer from data directory configured for the external service
-
-
- - ${sample_id_select_field.get_html()} -
- Select the sample that was sequenced to produce the datasets you want to transfer. -
-
-
- -
- Loading... -
- -
-
    -
  • Click the external service configuration button and change the Data directory setting to redefine the source data location.
    • -
  • Select a folder to select all of the individual files within that folder.
    • -
  • Click the Select datasets button when desired dataset check boxes are checked.
    • -
-
-
-
-
-
-
- -
-
-
- -%if sample and sample.datasets: - <% title = 'All selected datasets for "%s"' % sample.name %> -

- ${render_sample_datasets( 'requests_admin', sample, sample.datasets, title )} -%endif diff --git a/templates/webapps/galaxy/requests/common/common.mako b/templates/webapps/galaxy/requests/common/common.mako index 216b7165cbf..28808007832 100644 --- a/templates/webapps/galaxy/requests/common/common.mako +++ b/templates/webapps/galaxy/requests/common/common.mako @@ -333,7 +333,6 @@ can_add_samples = is_unsubmitted can_delete_samples = not adding_new_samples and request.samples and ( ( is_admin and not is_complete ) or is_unsubmitted ) can_edit_samples = request.samples and ( is_admin or not is_complete ) - can_select_datasets = is_admin and displayable_sample_widgets and ( is_submitted or is_complete ) can_transfer_datasets = is_admin and request.samples and not request.is_rejected display_checkboxes = not adding_new_samples and ( is_complete or is_rejected or is_submitted ) display_bar_code = request.samples and ( is_complete or is_rejected or is_submitted ) @@ -407,7 +406,7 @@ %elif sample: - %if sample.state and ( can_select_datasets or can_transfer_datasets ): + %if sample.state and can_transfer_datasets: ## A sample will have a state only after the request has been submitted. <% encoded_id = trans.security.encode_id( sample.id ) @@ -419,14 +418,6 @@ ${sample.name | h}

- %if can_select_datasets: - %for external_service in sample.request.type.get_external_services_for_manual_data_transfer( trans ): - <% - menu_item_label = "Select datasets to transfer using %s" % external_service.name - %> -
  • ${menu_item_label}
    • - %endfor - %endif %if sample.datasets and len( sample.datasets ) > len( transferred_dataset_files ) and sample.library and sample.folder:
  • Manage selected datasets
    • %elif sample.datasets and len( sample.datasets ) == len( transferred_dataset_files ): @@ -665,7 +656,6 @@ is_admin = cntrller == 'requests_admin' and trans.user_is_admin() is_complete = sample.request.is_complete is_submitted = sample.request.is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_transfer_datasets = is_admin and sample.untransferred_dataset_files %> ## The transfer status should update only when the request has been submitted or complete diff --git a/templates/webapps/galaxy/requests/common/view_request_history.mako b/templates/webapps/galaxy/requests/common/view_request_history.mako index 89faaaafd48..913914ddf80 100644 --- a/templates/webapps/galaxy/requests/common/view_request_history.mako +++ b/templates/webapps/galaxy/requests/common/view_request_history.mako @@ -11,7 +11,6 @@ can_add_samples = is_unsubmitted can_edit_request = ( is_admin and not is_complete ) or is_unsubmitted can_reject = is_admin and is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_submit_request = request.samples and is_unsubmitted %> diff --git a/templates/webapps/galaxy/requests/common/view_sample_datasets.mako b/templates/webapps/galaxy/requests/common/view_sample_datasets.mako index 986c88d78a9..9f4210f9ed5 100644 --- a/templates/webapps/galaxy/requests/common/view_sample_datasets.mako +++ b/templates/webapps/galaxy/requests/common/view_sample_datasets.mako @@ -12,7 +12,6 @@ is_admin = cntrller == 'requests_admin' and trans.user_is_admin() is_complete = sample.request.is_complete is_submitted = sample.request.is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_transfer_datasets = is_admin and sample.untransferred_dataset_files and sample.library and sample.folder %> @@ -24,9 +23,6 @@ %endif
  • Dataset Actions
    • - %if can_select_datasets: -
  • Select more datasets
    • - %endif
  • View target Data Library
  • Browse this request