diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index ec34c4c780c..6d8dad4422e 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -34,9 +34,11 @@ class UniverseApplication( object ): #Load security policy self.security_agent = self.model.security_agent # Start the job queue - job_dispatcher = jobs.DefaultJobDispatcher( self ) - self.job_queue = jobs.JobQueue( self, job_dispatcher ) - self.job_stop_queue = jobs.JobStopQueue( self, job_dispatcher ) + self.job_manager = jobs.JobManager( self ) + # FIXME: These are exposed directly for backward compatibility + self.job_queue = self.job_manager.job_queue + self.job_stop_queue = self.job_manager.job_stop_queue + # Heartbeat and memdump for thread / heap profiling self.heartbeat = None self.memdump = None # Start the heartbeat process if configured and available @@ -51,7 +53,6 @@ class UniverseApplication( object ): if memdump.Memdump: self.memdump = memdump.Memdump() def shutdown( self ): - self.job_stop_queue.shutdown() - self.job_queue.shutdown() + self.job_manager.shutdown() if self.heartbeat: self.heartbeat.shutdown() diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 8473eae5ac6..feb482b2759 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -67,6 +67,11 @@ class Configuration( object ): self.library_import_dir = kwargs.get( 'library_import_dir', None ) if self.library_import_dir is not None and not os.path.exists( self.library_import_dir ): raise ConfigurationError( "library_import_dir specified in config (%s) does not exist" % self.library_import_dir ) + # Configuration options for taking advantage of nginx features + self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False ) + self.nginx_upload_location = kwargs.get( 'nginx_upload_store', False ) + if self.nginx_upload_location: + self.nginx_upload_location = os.path.abspath( self.nginx_upload_location ) # Parse global_conf and save the parser global_conf = kwargs.get( 'global_conf', None ) global_conf_parser = ConfigParser.ConfigParser() @@ -81,6 +86,11 @@ class Configuration( object ): self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' ) def get( self, key, default ): return self.config_dict.get( key, default ) + def get_bool( self, key, default ): + if key in self.config_dict: + return string_as_bool( key ) + else: + return default def check( self ): # Check that required directories exist for path in self.root, self.file_path, self.tool_path, self.tool_data_path, self.template_path, self.job_working_directory: diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 90a40d6a2d7..e0135c75afc 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -17,6 +17,27 @@ log = logging.getLogger( __name__ ) # States for running a job. These are NOT the same as data states JOB_WAIT, JOB_ERROR, JOB_INPUT_ERROR, JOB_INPUT_DELETED, JOB_OK, JOB_READY, JOB_DELETED = 'wait', 'error', 'input_error', 'input_deleted', 'ok', 'ready', 'deleted' +class JobManager( object ): + """ + Highest level interface to job management. + + TODO: Currently the app accesses "job_queue" and "job_stop_queue" directly. + This should be decoupled. + """ + def __init__( self, app ): + self.app = app + if self.app.config.get_bool( "enable_job_running", True ): + # The dispatcher launches the underlying job runners + self.dispatcher = DefaultJobDispatcher( app ) + # Queues for starting and stopping jobs + self.job_queue = JobQueue( app, self.dispatcher ) + self.job_stop_queue = JobStopQueue( app, self.dispatcher ) + else: + self.job_queue = self.job_stop_queue = NoopQueue() + def shutdown( self ): + self.job_queue.shutdown() + self.job_stop_queue.shutdown() + class Sleeper( object ): """ Provides a 'sleep' method that sleeps for a number of seconds *unless* @@ -443,7 +464,7 @@ class JobWrapper( object ): # custom post process setup inp_data = dict( [ ( da.name, da.dataset ) for da in job.input_datasets ] ) out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] ) - param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] ) # why not re-use self.param_dict here? + param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] ) # why not re-use self.param_dict here? ##dunno...probably should, this causes tools.parameters.basic.UnvalidatedValue to be used in following methods instead of validated and transformed values during i.e. running workflows param_dict = self.tool.params_from_strings( param_dict, self.app ) # Check for and move associated_files self.tool.collect_associated_files(out_data) @@ -594,50 +615,12 @@ class JobStopQueue( object ): pass for job in jobs: - # jobs in a non queued/running/new state do not need to be stopped - if job.state not in [ model.Job.states.QUEUED, model.Job.states.RUNNING, model.Job.states.NEW ]: - return - # job has multiple datasets that aren't parent/child and not all of them are deleted. - if not self.check_if_output_datasets_deleted( job.id ): - return - self.mark_deleted( job.id ) # job is in JobQueue or FooJobRunner, will be dequeued due to state change above if job.job_runner_name is None: return # tell the dispatcher to stop the job self.dispatcher.stop( job ) - def check_if_output_datasets_deleted( self, job_id ): - job = model.Job.get( job_id ) - for dataset_assoc in job.output_datasets: - dataset = dataset_assoc.dataset - dataset.refresh() - #only the originator of the job can delete a dataset to cause - #cancellation of the job, no need to loop through history_associations - if not dataset.deleted: - return False - return True - - def mark_deleted( self, job_id ): - job = model.Job.get( job_id ) - job.refresh() - job.state = job.states.DELETED - job.info = "Job output deleted by user before job completed." - job.flush() - for dataset_assoc in job.output_datasets: - dataset = dataset_assoc.dataset - dataset.refresh() - dataset.deleted = True - dataset.state = dataset.states.DISCARDED - dataset.dataset.flush() - for dataset in dataset.dataset.history_associations: - #propagate info across shared datasets - dataset.deleted = True - dataset.blurb = 'deleted' - dataset.peek = 'Job deleted' - dataset.info = 'Job output deleted by user before job completed' - dataset.flush() - def put( self, job ): self.queue.put( job ) @@ -652,3 +635,12 @@ class JobStopQueue( object ): self.queue.put( self.STOP_SIGNAL ) self.sleeper.wake() log.info( "job stopper stopped" ) + +class NoopQueue( object ): + """ + Implements the JobQueue / JobStopQueue interface but does nothing + """ + def put( self, *args ): + return + def shutdown( self ): + return \ No newline at end of file diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 05dc038e448..0e0362753ac 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -94,7 +94,35 @@ class Job( object ): tool = app.toolbox.tools_by_id[self.tool_id] param_dict = tool.params_from_strings( param_dict, app ) return param_dict - + def check_if_output_datasets_deleted( self ): + """ + Return true if all of the output datasets associated with this job are + in the deleted state + """ + for dataset_assoc in self.output_datasets: + dataset = dataset_assoc.dataset + # only the originator of the job can delete a dataset to cause + # cancellation of the job, no need to loop through history_associations + if not dataset.deleted: + return False + return True + def mark_deleted( self ): + """ + Mark this job as deleted, and mark any output datasets as discarded. + """ + self.state = Job.states.DELETED + self.info = "Job output deleted by user before job completed." + for dataset_assoc in self.output_datasets: + dataset = dataset_assoc.dataset + dataset.deleted = True + dataset.state = dataset.states.DISCARDED + for dataset in dataset.dataset.history_associations: + # propagate info across shared datasets + dataset.deleted = True + dataset.blurb = 'deleted' + dataset.peek = 'Job deleted' + dataset.info = 'Job output deleted by user before job completed' + class JobParameter( object ): def __init__( self, name, value ): self.name = name diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index dbe3b873785..66031b2e972 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -356,7 +356,7 @@ StoredWorkflowMenuEntry.table = Table( "stored_workflow_menu_entry", metadata, MetadataFile.table = Table( "metadata_file", metadata, Column( "id", Integer, primary_key=True ), - Column( "name", String ), + Column( "name", TEXT ), Column( "hda_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True, nullable=True ), Column( "lda_id", Integer, ForeignKey( "library_folder_dataset_association.id" ), index=True, nullable=True ), Column( "create_time", DateTime, default=now ), diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 0e04a0b4d86..8a05fdb8084 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -676,7 +676,7 @@ class Tool: # on the standard run form) or "URL" (a parameter provided by # external data source tools). if "runtool_btn" not in incoming and "URL" not in incoming: - return "tool_form.tmpl", dict( errors={}, tool_state=state, param_values={}, incoming={} ) + return "tool_form.mako", dict( errors={}, tool_state=state, param_values={}, incoming={} ) # Process incoming data if not( self.check_values ): # If `self.check_values` is false we don't do any checking or @@ -702,20 +702,20 @@ class Tool: # error messages if errors: error_message = "One or more errors were found in the input you provided. The specific errors are marked below." - return "tool_form.tmpl", dict( errors=errors, tool_state=state, incoming=incoming, error_message=error_message ) + return "tool_form.mako", dict( errors=errors, tool_state=state, incoming=incoming, error_message=error_message ) # If we've completed the last page we can execute the tool elif state.page == self.last_page: out_data = self.execute( trans, incoming=params ) - return 'tool_executed.tmpl', dict( out_data=out_data ) + return 'tool_executed.mako', dict( out_data=out_data ) # Otherwise move on to the next page else: state.page += 1 # Fill in the default values for the next page self.fill_in_new_state( trans, self.inputs_by_page[ state.page ], state.inputs ) - return 'tool_form.tmpl', dict( errors=errors, tool_state=state ) + return 'tool_form.mako', dict( errors=errors, tool_state=state ) else: # Just a refresh, render the form with updated state and errors. - return 'tool_form.tmpl', dict( errors=errors, tool_state=state ) + return 'tool_form.mako', dict( errors=errors, tool_state=state ) def update_state( self, trans, inputs, state, incoming, prefix="", context=None, update_only=False, old_errors={}, changed_dependencies={} ): @@ -797,7 +797,7 @@ class Tool: # Deal with the 'test' element and see if it's value changed test_param_key = group_prefix + input.test_param.name test_param_error = None - test_incoming = incoming.get( test_param_key, None ) + test_incoming = get_incoming_value( incoming, test_param_key, None ) if test_param_key not in incoming \ and "__force_update__" + test_param_key not in incoming \ and update_only: @@ -878,7 +878,7 @@ class Tool: except: pass if not incoming_value_generated: - incoming_value = incoming.get( key, None ) + incoming_value = get_incoming_value( incoming, key, None ) value, error = check_param( trans, input, incoming_value, context ) if input.dependent_params and state[ input.name ] != value: # We need to keep track of changed dependency parametrs ( parameters @@ -948,7 +948,10 @@ class Tool: # Regular tool parameter value = input_values[ input.name ] if isinstance( value, UnvalidatedValue ): - value = input.from_html( value.value, None, context ) + if value.value is None: #if value.value is None, it could not have been submited via html form and therefore .from_html can't be guaranteed to work + value = None + else: + value = input.from_html( value.value, None, context ) # Then do any further validation on the value input.validate( value, None ) input_values[ input.name ] = value @@ -1361,3 +1364,12 @@ def json_fix( val ): else: return val +def get_incoming_value( incoming, key, default ): + if "__" + key + "__is_composite" in incoming: + composite_keys = incoming["__" + key + "__keys"].split() + value = dict() + for composite_key in composite_keys: + value[composite_key] = incoming[key + "_" + composite_key] + return value + else: + return incoming.get( key, default ) \ No newline at end of file diff --git a/lib/galaxy/tools/actions/upload.py b/lib/galaxy/tools/actions/upload.py index 90b98da39ce..332fb007607 100644 --- a/lib/galaxy/tools/actions/upload.py +++ b/lib/galaxy/tools/actions/upload.py @@ -26,12 +26,22 @@ class UploadToolAction( object ): temp_name = "" data_list = [] - if 'filename' in dir( data_file ): + if 'local_filename' in dir( data_file ): + # Use the existing file try: file_name = data_file.filename file_name = file_name.split( '\\' )[-1] file_name = file_name.split( '/' )[-1] - data_list.append( self.add_file( trans, data_file.file, file_name, file_type, dbkey, space_to_tab=space_to_tab ) ) + data_list.append( self.add_file( trans, data_file.local_filename, file_name, file_type, dbkey, space_to_tab=space_to_tab ) ) + except Exception, e: + return self.upload_empty( trans, "Error:", str( e ) ) + elif 'filename' in dir( data_file ): + try: + file_name = data_file.filename + file_name = file_name.split( '\\' )[-1] + file_name = file_name.split( '/' )[-1] + temp_name = sniff.stream_to_file( data_file.file ) + data_list.append( self.add_file( trans, temp_name, file_name, file_type, dbkey, space_to_tab=space_to_tab ) ) except Exception, e: return self.upload_empty( trans, "Error:", str( e ) ) if url_paste not in [ None, "" ]: @@ -53,7 +63,8 @@ class UploadToolAction( object ): if not NAME: NAME = line try: - data_list.append( self.add_file( trans, urllib.urlopen( line ), NAME, file_type, dbkey, info=INFO, space_to_tab=space_to_tab ) ) + temp_name = sniff.stream_to_file( urllib.urlopen( line ) ) + data_list.append( self.add_file( trans, temp_name, NAME, file_type, dbkey, info="uploaded url", space_to_tab=space_to_tab ) ) except Exception, e: return self.upload_empty( trans, "Error:", str( e ) ) else: @@ -65,7 +76,8 @@ class UploadToolAction( object ): break if is_valid: try: - data_list.append( self.add_file( trans, StringIO.StringIO( url_paste ), 'Pasted Entry', file_type, dbkey, info="pasted entry", space_to_tab=space_to_tab ) ) + temp_name = sniff.stream_to_file( StringIO.StringIO( url_paste ) ) + data_list.append( self.add_file( trans, temp_name, 'Pasted Entry', file_type, dbkey, info="pasted entry", space_to_tab=space_to_tab ) ) except Exception, e: return self.upload_empty( trans, "Error:", str( e ) ) else: @@ -90,9 +102,8 @@ class UploadToolAction( object ): trans.app.model.flush() return dict( output=data ) - def add_file( self, trans, file_obj, file_name, file_type, dbkey, info=None, space_to_tab=False ): + def add_file( self, trans, temp_name, file_name, file_type, dbkey, info=None, space_to_tab=False ): data_type = None - temp_name = sniff.stream_to_file( file_obj ) # See if we have an empty file if not os.path.getsize( temp_name ) > 0: diff --git a/lib/galaxy/tools/parameters/__init__.py b/lib/galaxy/tools/parameters/__init__.py index f8cda91c0de..a17fdcd5d5e 100644 --- a/lib/galaxy/tools/parameters/__init__.py +++ b/lib/galaxy/tools/parameters/__init__.py @@ -16,9 +16,7 @@ def check_param( trans, param, incoming_value, param_values ): value = incoming_value error = None try: - if param.name == 'file_data': - pass - elif value is not None or isinstance(param, DataToolParameter): + if value is not None or isinstance(param, DataToolParameter): # Convert value from HTML representation value = param.from_html( value, trans, param_values ) # Allow the value to be converted if neccesary diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 882849f7bf2..4a055e6dd0b 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -2,10 +2,11 @@ Basic tool parameters. """ -import logging, string, sys, os +import logging, string, sys, os, os.path from elementtree.ElementTree import XML, Element from galaxy import config, datatypes, util from galaxy.web import form_builder +from galaxy.util.bunch import Bunch import validation, dynamic_options # For BaseURLToolParameter from galaxy.web import url_for @@ -291,6 +292,23 @@ class FileToolParameter( ToolParameter ): self.name = elem.get( 'name' ) def get_html_field( self, trans=None, value=None, other_values={} ): return form_builder.FileField( self.name ) + def from_html( self, value, trans=None, other_values={} ): + # Middleware or proxies may encode files in special ways (TODO: this + # should be pluggable) + if type( value ) == dict: + upload_location = self.tool.app.config.nginx_upload_location + assert upload_location, \ + "Request appears to have been processed by nginx_upload_module \ + but Galaxy is not configured to recgonize it" + # Check that the file is in the right location + local_filename = os.path.abspath( value['path'] ) + assert local_filename.startswith( upload_location ), \ + "Filename provided by nginx is not in correct directory" + value = Bunch( + filename = value["name"], + local_filename = local_filename + ) + return value def get_required_enctype( self ): """ File upload elements require the multipart/form-data encoding diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py index ddaed52842c..2e080ab42ae 100644 --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -102,7 +102,7 @@ class DataMetaFilter( Filter ): if self.multiple: return dataset_value in file_value.split( self.separator ) return file_value == dataset_value - assert self.ref_name in other_values or trans.workflow_building_mode, "Required dependency '%s' not found in incoming values" % self.ref_name + assert self.ref_name in other_values or ( trans is not None and trans.workflow_building_mode), "Required dependency '%s' not found in incoming values" % self.ref_name ref = other_values.get( self.ref_name, None ) if not isinstance( ref, self.dynamic_option.tool_param.tool.app.model.HistoryDatasetAssociation ): return [] #not a valid dataset @@ -146,9 +146,9 @@ class ParamValueFilter( Filter ): def get_dependency_name( self ): return self.ref_name def filter_options( self, options, trans, other_values ): - if trans.workflow_building_mode: return [] + if trans is not None and trans.workflow_building_mode: return [] + assert self.ref_name in other_values, "Required dependency '%s' not found in incoming values" % self.ref_name ref = str( other_values.get( self.ref_name, None ) ) - assert ref is not None, "Required dependency '%s' not found in incoming values" % self.ref_name rval = [] for fields in options: if ( self.keep and fields[self.column] == ref ) or ( not self.keep and fields[self.column] != ref ): diff --git a/lib/galaxy/web/controllers/root.py b/lib/galaxy/web/controllers/root.py index 6a3046587ff..6e8140d7e8e 100644 --- a/lib/galaxy/web/controllers/root.py +++ b/lib/galaxy/web/controllers/root.py @@ -305,6 +305,28 @@ class RootController( BaseController ): else: return trans.show_error_message( "You do not have permission to edit this dataset's (%s) attributes." % id ) + def __delete_dataset( self, trans, id ): + data = self.app.model.HistoryDatasetAssociation.get( id ) + if data: + # Walk up parent datasets to find the containing history + topmost_parent = data + while topmost_parent.parent: + topmost_parent = topmost_parent.parent + assert topmost_parent in trans.history.datasets, "Data does not belong to current history" + # Mark deleted and cleanup + data.mark_deleted() + data.clear_associated_files() + trans.log_event( "Dataset id %s marked as deleted" % str(id) ) + if data.parent_id is None and len( data.creating_job_associations ) > 0: + # Mark associated job for deletion + job = data.creating_job_associations[0].job + if job.state not in [ model.Job.states.QUEUED, model.Job.states.RUNNING, model.Job.states.NEW ]: + return + # Are *all* of the job's other output datasets deleted? + if job.check_if_output_datasets_deleted(): + job.mark_deleted() + self.app.model.flush() + @web.expose def delete( self, trans, id = None, show_deleted_on_refresh = False, **kwd): if id: @@ -315,55 +337,20 @@ class RootController( BaseController ): history = trans.get_history() for id in dataset_ids: try: - int( id ) + id = int( id ) except: continue - data = self.app.model.HistoryDatasetAssociation.get( id ) - if data: - # Walk up parent datasets to find the containing history - topmost_parent = data - while topmost_parent.parent: - topmost_parent = topmost_parent.parent - assert topmost_parent in history.datasets, "Data does not belong to current history" - # Mark deleted and cleanup - data.mark_deleted() - data.clear_associated_files() - self.app.model.flush() - trans.log_event( "Dataset id %s marked as deleted" % str(id) ) - if data.parent_id is None: - try: - self.app.job_stop_queue.put( data.creating_job_associations[0].job ) - except IndexError: - pass # upload tool will cause this since it doesn't have a job + self.__delete_dataset( trans, id ) return self.history( trans, show_deleted = show_deleted_on_refresh ) @web.expose def delete_async( self, trans, id = None, **kwd): if id: try: - int( id ) + id = int( id ) except: return "Dataset id '%s' is invalid" %str( id ) - history = trans.get_history() - data = self.app.model.HistoryDatasetAssociation.get( id ) - if data: - # Walk up parent datasets to find the containing history - topmost_parent = data - while topmost_parent.parent: - topmost_parent = topmost_parent.parent - assert topmost_parent in history.datasets, "Data does not belong to current history" - # Mark deleted and cleanup - data.mark_deleted() - data.clear_associated_files() - self.app.model.flush() - trans.log_event( "Dataset id %s marked as deleted async" % str(id) ) - if data.parent_id is None: - try: - self.app.job_stop_queue.put( data.creating_job_associations[0].job ) - except IndexError: - pass # upload tool will cause this since it doesn't have a job - else: - return "Dataset id '%s' is invalid" %str( id ) + self.__delete_dataset( trans, id ) return "OK" ## ---- History management ----------------------------------------------- diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index 470fbdf9f30..6dff0719d2a 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -5,6 +5,7 @@ A simple WSGI application/framework. import socket import types import logging +import os.path import sys from Cookie import SimpleCookie @@ -132,16 +133,16 @@ class WebApplication( object ): if callable( body ): # Assume the callable is another WSGI application to run return body( environ, start_response ) + elif isinstance( body, types.FileType ): + # Stream the file back to the browser + return send_file( start_response, trans, body ) else: start_response( trans.response.wsgi_status(), trans.response.wsgi_headeritems() ) return self.make_body_iterable( trans, body ) def make_body_iterable( self, trans, body ): - if isinstance( body, types.FileType ): - # Stream the file back to the browser - return iterate_file( body ) - elif isinstance( body, ( types.GeneratorType, list, tuple ) ): + if isinstance( body, ( types.GeneratorType, list, tuple ) ): # Recursively stream the iterable return flatten( body ) elif isinstance( body, basestring ): @@ -302,6 +303,20 @@ class Response( object ): CHUNK_SIZE = 2**16 +def send_file( start_response, trans, body ): + # If configured use X-Accel-Redirect header for nginx + base = trans.app.config.nginx_x_accel_redirect_base + if base: + trans.response.headers['X-Accel-Redirect'] = \ + base + os.path.abspath( body.name ) + body = [ "" ] + # Fall back on sending the file in chunks + else: + body = iterate_file( body ) + start_response( trans.response.wsgi_status(), + trans.response.wsgi_headeritems() ) + return body + def iterate_file( file ): """ Progressively return chunks from `file`. diff --git a/scripts/cleanup_datasets/cleanup_datasets.py b/scripts/cleanup_datasets/cleanup_datasets.py index a465e0d3d61..68058b72a81 100644 --- a/scripts/cleanup_datasets/cleanup_datasets.py +++ b/scripts/cleanup_datasets/cleanup_datasets.py @@ -191,8 +191,6 @@ def purge_histories( h, d, m, cutoff_time, remove_from_disk ): if errmsg: errors = True print errmsg - else: - print "%s" % dataset.file_name else: dataset.purged = True dataset.flush() @@ -264,7 +262,6 @@ def purge_datasets( d, m, cutoff_time, remove_from_disk ): print errmsg else: dataset_count += 1 - print "%s" % dataset.file_name else: dataset.purged = True dataset.file_size = 0 @@ -314,6 +311,7 @@ def purge_dataset( dataset, m ): else: # Remove dataset file from disk os.unlink( dataset.file_name ) + print "%s" % dataset.file_name # Mark all associated MetadataFiles as deleted and purged and remove them from disk print "The following metadata files associated with dataset '%s' have been purged" % dataset.file_name for hda in dataset.history_associations: diff --git a/scripts/scramble/scripts/pbs_python.py b/scripts/scramble/scripts/pbs_python.py index 138546b8e32..6c898642564 100644 --- a/scripts/scramble/scripts/pbs_python.py +++ b/scripts/scramble/scripts/pbs_python.py @@ -27,6 +27,9 @@ for dir in [ "build", "dist" ]: print "scramble(): removing dir:", dir shutil.rmtree( dir ) +# the build process doesn't set an rpath for libtorque +os.environ['LD_RUN_PATH'] = os.environ['LIBTORQUE_DIR'] + print "scramble(): Running pbs_python configure script" p = subprocess.Popen( args = "sh configure --with-pbsdir=%s" % os.environ['LIBTORQUE_DIR'], shell = True ) r = p.wait() diff --git a/templates/tool_executed.tmpl b/templates/tool_executed.mako similarity index 61% rename from templates/tool_executed.tmpl rename to templates/tool_executed.mako index fa9e98eaa18..9fd7ed1bc08 100644 --- a/templates/tool_executed.tmpl +++ b/templates/tool_executed.mako @@ -4,37 +4,31 @@ Galaxy - + @@ -43,14 +37,13 @@ -

The following job has been succesfully added to the queue:

-#for $data in $out_data.values -
$data.hid: $data.name
-#end for +%for data in out_data.values(): +
${data.hid}: ${data.name}
+%endfor

You can check the status of queued jobs and view the resulting @@ -59,9 +52,9 @@ the status will change from 'running' to 'finished' if completed succesfully or 'error' if problems were encountered.

-#if $tool.options.refresh -
-#end if +%if tool.options.refresh: +
+
${input.title_plural}
+ <% repeat_state = tool_state[input.name] %> + %for i in range( len( repeat_state ) ): +
+ <% + if input.name in errors: + rep_errors = errors[input.name][i] + else: + rep_errors = dict() + index = repeat_state[i]['__index__'] + %> +
${input.title} ${i + 1}
+ ${do_inputs( input.inputs, repeat_state[i], rep_errors, prefix + input.name + "_" + str(index) + "|", other_values )} +
+
+ %endfor +
+
+ %elif input.type == "conditional": + <% + group_state = tool_state[input.name] + group_errors = errors.get( input.name, {} ) + current_case = group_state['__current_case__'] + group_prefix = prefix + input.name + "|" + %> + ${row_for_param( group_prefix, input.test_param, group_state, group_errors, other_values )} + ${do_inputs( input.cases[current_case].inputs, group_state, group_errors, group_prefix, other_values )} + %else: + ${row_for_param( prefix, input, tool_state, errors, other_values )} + %endif + %endfor + + +<%def name="row_for_param( prefix, param, parent_state, parent_errors, other_values )"> + <% + if parent_errors.has_key( param.name ): + cls = "form-row form-row-error" + else: + cls = "form-row" + %> +
+ <% label = param.get_label() %> + %if label: + + %endif + <% + field = param.get_html_field( trans, parent_state[ param.name ], other_values ) + field.refresh_on_change = param.refresh_on_change + %> +
${field.get_html( prefix )}
+ %if parent_errors.has_key( param.name ): +
+
 ${parent_errors[param.name]}
+
+ %endif + + %if param.help: +
+ ${param.help} +
+ %endif + +
+ +
+ + +%if add_frame.from_noframe: +
+ Welcome to Galaxy +
+ It appears that you found this tool from a link outside of Galaxy. + If you're not familiar with Galaxy, please consider visiting the + welcome page. + To learn more about what Galaxy is and what it can do for you, please visit + the Galaxy wiki. +
+
+%endif + +
+ %if tool.has_multiple_pages: +
${tool.name} (step ${tool_state.page+1} of ${tool.npages})
+ %else: +
${tool.name}
+ %endif +
+
+ + + + %if tool.display_by_page[tool_state.page]: + ${trans.fill_template_string( tool.display_by_page[tool_state.page], other_values=tool.get_param_html_map( trans, tool_state.page, tool_state.inputs ) )} + + + %else: + +
+ %if tool_state.page == tool.last_page: + + %else: + + %endif +
+ ${do_inputs( tool.inputs_by_page[ tool_state.page ], tool_state.inputs, errors, "" )} +
+ %if tool_state.page == tool.last_page: + + %else: + + %endif +
+ + %endif + +
+
+
+ +%if tool.help: +
+
+ %if tool.has_multiple_pages: + ${tool.help_by_page[tool_state.page]} + %else: + ${tool.help} + %endif +
+
+%endif + + + + + + diff --git a/templates/tool_form.tmpl b/templates/tool_form.tmpl deleted file mode 100644 index 86f359e0849..00000000000 --- a/templates/tool_form.tmpl +++ /dev/null @@ -1,191 +0,0 @@ - - - -#from galaxy.util.expressions import ExpressionContext - - - - -Galaxy - - - - - - - - -## #if $getVar( 'error_message', None ) -##
$error_message
-##

-## #end if - -#def do_inputs( $inputs, $tool_state, $errors, $prefix, $context=None ) - #set $context = ExpressionContext( $tool_state, $context ) - #for $input_index, $input in enumerate( $inputs.itervalues() ) - #if $input.type == "repeat" -
-
${input.title_plural}
- #set repeat_state = $tool_state[$input.name] - #for i in range( len( $repeat_state ) ): -
- #if $input.name in errors - #set rep_errors = $errors[$input.name][$i] - #else - #set rep_errors = dict() - #end if - #set index = $repeat_state[$i]['__index__'] -
${input.title} ${i + 1}
- $do_inputs( $input.inputs, $repeat_state[$i], $rep_errors, $prefix + $input.name + "_" + str($index) + "|", $context ) -
-
- #end for -
-
- #elif $input.type == "conditional" - #set group_state = $tool_state[$input.name] - #set group_errors = $errors.get( $input.name, {} ) - #set current_case = $group_state['__current_case__'] - #set group_prefix = $prefix + $input.name + "|" - $row_for_param( $group_prefix, $input.test_param, $group_state, $group_errors, $context ) - $do_inputs( $input.cases[$current_case].inputs, $group_state, $group_errors, $group_prefix, $context ) - #else - $row_for_param( $prefix, $input, $tool_state, $errors, $context ) - #end if - #end for -#end def - -#def row_for_param( $prefix, $param, $parent_state, $parent_errors, $context ) - #if $parent_errors.has_key( $param.name ): - #set cls = "form-row form-row-error" - #else - #set cls = "form-row" - #end if -
- #set label = $param.get_label() - #if $label: - - #end if - #set field = $param.get_html_field( $caller, $parent_state[ $param.name ], $context ) - #set $field.refresh_on_change = $param.refresh_on_change -
$field.get_html( $prefix )
- #if $parent_errors.has_key( $param.name ): -
-
 $parent_errors[$param.name]
-
- #elif $param.help - ##
$param.help
- #end if - - #if $param.help -
- $param.help -
- - #end if - -
- -
-#end def - -#if $add_frame.from_noframe -
- Welcome to Galaxy -
- It appears that you found this tool from a link outside of Galaxy. If you're not familiar with Galaxy, please consider visiting the welcome page. To learn more about what Galaxy is and what it can do for you, please visit the Galaxy wiki. -
-
-#end if - -
- #if $tool.has_multiple_pages -
$tool.name (step #echo $tool_state.page+1 # of $tool.npages)
- #else -
$tool.name
- #end if -
-
- - - - #if $tool.display_by_page[$tool_state.page] - - $caller.fill_template_string( $tool.display_by_page[$tool_state.page], context=$tool.get_param_html_map( $caller, $tool_state.page, $tool_state.inputs ) ) - - - #else - -
- #if $tool_state.page == $tool.last_page - - #else - - #end if -
- $do_inputs( $tool.inputs_by_page[ $tool_state.page ], $tool_state.inputs, $errors, "" ) -
- #if $tool_state.page == $tool.last_page - - #else - - #end if -
- - #end if - -
-
-
- -#if $tool.help -
-
- #if $tool.has_multiple_pages - $tool.help_by_page[$tool_state.page] - #else - $tool.help - #end if -
-
-#end if - - - - - - diff --git a/tools/data_source/microbial_import_code.py b/tools/data_source/microbial_import_code.py index 7c220566055..a1d926ba58e 100644 --- a/tools/data_source/microbial_import_code.py +++ b/tools/data_source/microbial_import_code.py @@ -80,7 +80,7 @@ def load_microbial_data( GALAXY_DATA_INDEX_DIR, sep='\t' ): return microbe_info #post processing, set build for data and add additional data to history -from galaxy import datatypes, config, jobs +from galaxy import datatypes, config, jobs, tools from shutil import copyfile def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr): @@ -96,7 +96,12 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr #if not (kingdom or group or org): if not (kingdom or org): print "Parameters are not available." - + #workflow passes galaxy.tools.parameters.basic.UnvalidatedValue instead of values + if isinstance( kingdom, tools.parameters.basic.UnvalidatedValue ): + kingdom = kingdom.value + if isinstance( org, tools.parameters.basic.UnvalidatedValue ): + org = org.value + GALAXY_DATA_INDEX_DIR = app.config.tool_data_path microbe_info = load_microbial_data( GALAXY_DATA_INDEX_DIR, sep='\t' ) new_stdout = "" diff --git a/tools/new_operations/complement.xml b/tools/new_operations/complement.xml index 84ca368c712..be661ee6f4a 100644 --- a/tools/new_operations/complement.xml +++ b/tools/new_operations/complement.xml @@ -18,6 +18,11 @@ + + + + + diff --git a/tools/new_operations/coverage.xml b/tools/new_operations/coverage.xml index 63475e98fc5..979db1e6533 100644 --- a/tools/new_operations/coverage.xml +++ b/tools/new_operations/coverage.xml @@ -19,6 +19,11 @@ + + + + + diff --git a/tools/new_operations/intersect.xml b/tools/new_operations/intersect.xml index a2975178c63..5a755548a42 100644 --- a/tools/new_operations/intersect.xml +++ b/tools/new_operations/intersect.xml @@ -21,13 +21,27 @@ - + + + + + + + + - + - - + + + + + + + + + diff --git a/tools/new_operations/merge.xml b/tools/new_operations/merge.xml index ce7a349292c..c0118db9d25 100644 --- a/tools/new_operations/merge.xml +++ b/tools/new_operations/merge.xml @@ -21,6 +21,11 @@ + + + + + diff --git a/tools/new_operations/subtract.xml b/tools/new_operations/subtract.xml index 9a6ca6e3868..2a58e1654ee 100644 --- a/tools/new_operations/subtract.xml +++ b/tools/new_operations/subtract.xml @@ -32,6 +32,13 @@ + + + + + + + diff --git a/universe_wsgi.ini.sample b/universe_wsgi.ini.sample index 4930c93ec6e..68c3cd99820 100644 --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -34,7 +34,7 @@ job_queue_cleanup_interval = 30 # Database connection database_file = database/universe.sqlite # You may use a SQLAlchemy connection string to specify an external database instead -## database_connection = postgres:///galaxy_test +## database_connection = postgres:///galaxy ## database_engine_option_echo = true ## database_engine_option_echo_pool = true ## database_engine_option_pool_size = 10 @@ -93,12 +93,15 @@ mailing_join_addr = galaxy-user-join@bx.psu.edu # Write thread status periodically to 'heartbeat.log' (careful, uses disk space rapidly!) ## use_heartbeat = True +# Enable the memory debugging interface (careful, negatively impacts server performance) +## use_memdump = True + # Profiling middleware (cProfile based) ## use_profile = True -# Mail -smtp_server = coltrane.bx.psu.edu -error_email_to = galaxy-bugs@bx.psu.edu +# For use by 'report this error' link on error-state datasets +#smtp_server = smtp.example.org +#error_email_to = galaxy-bugs@example.org # Use the new iframe / javascript based layout use_new_layout = true @@ -124,29 +127,34 @@ static_style_dir = %(here)s/static/june_2007_style/blue ## wiki_url: replaces the default galaxy main wiki ## bugs_email: replaces the default galaxy bugs email list #brand = Private local mirror -#wiki_url=/path/to/my/local/wiki -#bugs_email=mailto:bugmaster@this.site.com +#wiki_url = /path/to/my/local/wiki +#bugs_email = mailto:galaxy-bugs@example.org # ---- Job Runners ---------------------------------------------------------- # Clustering Galaxy is not a straightforward process and requires a lot of -# pre-configuration. See the ClusteringGalaxy Wiki before attempting to set any -# of these options. If running normally (without a cluster), do not change -# anything in this section. +# pre-configuration. See the ClusteringGalaxy Wiki before attempting to set +# any of these options: +# +# http://g2.trac.bx.psu.edu/wiki/ClusteringGalaxy +# +# If running normally (without a cluster), do not change anything in this +# section. # start_job_runners: Comma-separated list of job runners to start. local is # always started. If left commented, no jobs will be run on the cluster, even # if a cluster URL is explicitly defined in the [galaxy:tool_runners] section -# below. The only runner currently available is 'pbs'. +# below. The runners currently available are 'pbs' and 'sge'. #start_job_runners = pbs # default_cluster_job_runner: The URL for the default runner to use when a tool # doesn't explicity define a runner below. For help on the cluster URL format, -# see the ClusteringGalaxy Wiki. Leave commented if not using a cluster job runner. +# see the ClusteringGalaxy Wiki. Leave commented if not using a cluster job +# runner. #default_cluster_job_runner = pbs:/// # The PBS options are described in detail in the Galaxy Configuration section of -# the ClusteringGalaxy Wiki +# the ClusteringGalaxy Wiki, and are only necessary when using file staging. #pbs_application_server = #pbs_stage_path = #pbs_dataset_server = @@ -156,8 +164,6 @@ static_style_dir = %(here)s/static/june_2007_style/blue [galaxy:tool_runners] biomart = local:/// -blat2wig = pbs:///blast -blat_wrapper = pbs:///blast encode_db1 = local:/// encode_import_all_latest_datasets1 = local:/// encode_import_chromatin_and_chromosomes1 = local:/// @@ -165,13 +171,8 @@ encode_import_gencode1 = local:/// encode_import_genes_and_transcripts1 = local:/// encode_import_multi-species_sequence_analysis1 = local:/// encode_import_transcription_regulation1 = local:/// -generate_coverage_report = pbs:///blast hbvar = local:/// -hist_high_quality_score = pbs:///blast -megablast_wrapper = pbs:///blast -megablast_xml_parser = pbs:///blast microbial_import1 = local:/// -quality_score_distribution = pbs:///blast ucsc_table_direct1 = local:/// ucsc_table_direct_archaea1 = local:/// ucsc_table_direct_test1 = local:///