From e35fa87fbd422d94bb4585f2f8d3cf8c6baae3f3 Mon Sep 17 00:00:00 2001 From: Guruprasad Anada Date: Thu, 30 Oct 2008 16:03:11 -0400 Subject: [PATCH 1/6] Added new functional tests to all gops tools using BitsetSafeReaderWrapper. These tests check cases where datasets have the chr,start and end values in non-standard columns. --- tools/new_operations/complement.xml | 5 +++++ tools/new_operations/coverage.xml | 5 +++++ tools/new_operations/intersect.xml | 22 ++++++++++++++++++---- tools/new_operations/merge.xml | 5 +++++ tools/new_operations/subtract.xml | 7 +++++++ 5 files changed, 40 insertions(+), 4 deletions(-) diff --git a/tools/new_operations/complement.xml b/tools/new_operations/complement.xml index 84ca368c712..be661ee6f4a 100644 --- a/tools/new_operations/complement.xml +++ b/tools/new_operations/complement.xml @@ -18,6 +18,11 @@ + + + + + diff --git a/tools/new_operations/coverage.xml b/tools/new_operations/coverage.xml index 63475e98fc5..979db1e6533 100644 --- a/tools/new_operations/coverage.xml +++ b/tools/new_operations/coverage.xml @@ -19,6 +19,11 @@ + + + + + diff --git a/tools/new_operations/intersect.xml b/tools/new_operations/intersect.xml index a2975178c63..5a755548a42 100644 --- a/tools/new_operations/intersect.xml +++ b/tools/new_operations/intersect.xml @@ -21,13 +21,27 @@ - + + + + + + + + - + - - + + + + + + + + + diff --git a/tools/new_operations/merge.xml b/tools/new_operations/merge.xml index ce7a349292c..c0118db9d25 100644 --- a/tools/new_operations/merge.xml +++ b/tools/new_operations/merge.xml @@ -21,6 +21,11 @@ + + + + + diff --git a/tools/new_operations/subtract.xml b/tools/new_operations/subtract.xml index 9a6ca6e3868..2a58e1654ee 100644 --- a/tools/new_operations/subtract.xml +++ b/tools/new_operations/subtract.xml @@ -32,6 +32,13 @@ + + + + + + + From aafccaa3679acea7c892cfa6b1f909373c24d079 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 30 Oct 2008 16:11:44 -0400 Subject: [PATCH 2/6] Use LD_RUN_PATH when building pbs_python --- scripts/scramble/scripts/pbs_python.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/scramble/scripts/pbs_python.py b/scripts/scramble/scripts/pbs_python.py index 138546b8e32..6c898642564 100644 --- a/scripts/scramble/scripts/pbs_python.py +++ b/scripts/scramble/scripts/pbs_python.py @@ -27,6 +27,9 @@ for dir in [ "build", "dist" ]: print "scramble(): removing dir:", dir shutil.rmtree( dir ) +# the build process doesn't set an rpath for libtorque +os.environ['LD_RUN_PATH'] = os.environ['LIBTORQUE_DIR'] + print "scramble(): Running pbs_python configure script" p = subprocess.Popen( args = "sh configure --with-pbsdir=%s" % os.environ['LIBTORQUE_DIR'], shell = True ) r = p.wait() From 5be8fefaf5c7d987b779b801ea0da26ac71308ab Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Thu, 30 Oct 2008 16:17:46 -0400 Subject: [PATCH 3/6] Migrate central repo to alchemy 4. --- eggs.ini | 4 +- lib/galaxy/app.py | 2 +- lib/galaxy/jobs/__init__.py | 7 +- lib/galaxy/model/custom_types.py | 17 +++-- lib/galaxy/model/mapping.py | 66 +++++++++---------- lib/galaxy/model/mapping_tests.py | 6 +- lib/galaxy/model/orm/__init__.py | 7 ++ lib/galaxy/model/orm/ext/__init__.py | 3 + lib/galaxy/model/orm/ext/assignmapper.py | 62 +++++++++++++++++ lib/galaxy/web/controllers/root.py | 8 +-- lib/galaxy/web/controllers/user.py | 8 +-- lib/galaxy/web/controllers/workflow.py | 2 +- lib/galaxy/web/framework/__init__.py | 14 ++-- .../webapps/reports/controllers/jobs.py | 2 +- .../webapps/reports/controllers/system.py | 5 +- .../webapps/reports/controllers/users.py | 2 +- scripts/cleanup_datasets/cleanup_datasets.py | 8 +-- tools/stats/grouping.py | 2 +- 18 files changed, 147 insertions(+), 78 deletions(-) create mode 100644 lib/galaxy/model/orm/__init__.py create mode 100644 lib/galaxy/model/orm/ext/__init__.py create mode 100644 lib/galaxy/model/orm/ext/assignmapper.py diff --git a/eggs.ini b/eggs.ini index ce76ee07353..ef17e4f8239 100644 --- a/eggs.ini +++ b/eggs.ini @@ -40,7 +40,7 @@ PasteDeploy = 1.3.1 PasteScript = 1.3.6 Routes = 1.6.3 simplejson = 1.5 -SQLAlchemy = 0.3.11 +SQLAlchemy = 0.4.7p1 Tempita = 0.1 twill = 0.9 WebError = 0.8a @@ -85,7 +85,7 @@ PasteDeploy = http://cheeseshop.python.org/packages/source/P/PasteDeploy/PasteDe PasteScript = http://cheeseshop.python.org/packages/source/P/PasteScript/PasteScript-1.3.6.tar.gz Routes = http://pypi.python.org/packages/source/R/Routes/Routes-1.6.3.tar.gz simplejson = http://cheeseshop.python.org/packages/source/s/simplejson/simplejson-1.5.tar.gz -SQLAlchemy = http://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-0.3.11.tar.gz +SQLAlchemy = http://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-0.4.7p1.tar.gz Tempita = http://pypi.python.org/packages/source/T/Tempita/Tempita-0.1.tar.gz twill = http://darcs.idyll.org/~t/projects/twill-0.9.tar.gz WebError = http://pypi.python.org/packages/source/W/WebError/WebError-0.8a.tar.gz diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 0c35c12a4be..f59ae433613 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -20,7 +20,7 @@ class UniverseApplication( object ): if self.config.database_connection: db_url = self.config.database_connection else: - db_url = "sqlite://%s?isolation_level=IMMEDIATE" % self.config.database + db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database # Setup the database engine and ORM self.model = galaxy.model.mapping.init( self.config.file_path, db_url, diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 9579e97e67a..90a40d6a2d7 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -97,11 +97,10 @@ class JobQueue( object ): model = self.app.model # Jobs in the NEW state won't be requeued unless we're tracking in the database if not self.track_jobs_in_database: - for job in model.Job.select( model.Job.c.state == model.Job.states.NEW ): + for job in model.Job.filter( model.Job.c.state==model.Job.states.NEW ).all(): log.debug( "no runner: %s is still in new state, adding to the jobs queue" %job.id ) self.queue.put( ( job.id, job.tool_id ) ) - for job in model.Job.select( (model.Job.c.state == model.Job.states.RUNNING) - | (model.Job.c.state == model.Job.states.QUEUED) ): + for job in model.Job.filter( (model.Job.c.state == model.Job.states.RUNNING) | (model.Job.c.state == model.Job.states.QUEUED) ).all(): if job.job_runner_name is not None: # why are we passing the queue to the wrapper? job_wrapper = JobWrapper( job.id, self.app.toolbox.tools_by_id[ job.tool_id ], self ) @@ -136,7 +135,7 @@ class JobQueue( object ): new_jobs = [] if self.track_jobs_in_database: model = self.app.model - for j in model.Job.select( model.Job.c.state == model.Job.states.NEW ): + for j in model.Job.filter( model.Job.c.state==model.Job.states.NEW ).all(): job = JobWrapper( j.id, self.app.toolbox.tools_by_id[ j.tool_id ], self ) new_jobs.append( job ) else: diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index 77b80bfeab5..0b6ffa2a57e 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -18,16 +18,15 @@ class JSONType( TypeDecorator ): self.mutable = mutable super( JSONType, self).__init__() - def convert_result_value( self, value, dialect ): + def process_bind_param( self, value, dialect ): if value is None: return None - buf = self.impl.convert_result_value( value, dialect ) - return self.jsonifyer.loads( str(buf) ) - - def convert_bind_param( self, value, dialect ): + return self.jsonifyer.dumps( value ) + + def process_result_value( self, value, dialect ): if value is None: return None - return self.impl.convert_bind_param( self.jsonifyer.dumps(value), dialect ) + return self.jsonifyer.loads( str( value ) ) def copy_value( self, value ): if self.mutable: @@ -60,10 +59,10 @@ class MetadataType( JSONType ): self.mutable = mutable super( MetadataType, self).__init__() - def convert_result_value( self, value, dialect ): + def process_result_value( self, value, dialect ): if value is None: return None - buf = self.impl.convert_result_value( value, dialect ) + buf = value ret = None try: ret = self.pickler.loads( str(buf) ) @@ -77,7 +76,7 @@ class MetadataType( JSONType ): class TrimmedString( TypeDecorator ): impl = String - def convert_bind_param( self, value, dialect ): + def process_bind_param( self, value, dialect ): """Automatically truncate string values""" if self.impl.length and value is not None: value = value[0:self.impl.length] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 7fc8aa957a4..30ec3c420c6 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -5,23 +5,20 @@ are encapsulated here. import logging log = logging.getLogger( __name__ ) -import pkg_resources -pkg_resources.require( "sqlalchemy>=0.3" ) - import sys import datetime -from sqlalchemy.ext.sessioncontext import SessionContext -from sqlalchemy.ext.assignmapper import assign_mapper -from sqlalchemy.ext.orderinglist import ordering_list - -from sqlalchemy import * from galaxy.model import * +from galaxy.model.orm import * +from galaxy.model.orm.ext.assignmapper import * from galaxy.model.custom_types import * from galaxy.util.bunch import Bunch -metadata = DynamicMetaData( threadlocal=False ) -context = SessionContext( create_session ) +metadata = MetaData() +context = Session = scoped_session( sessionmaker( autoflush=False, transactional=False ) ) + +# For backward compatibility with "context.current" +context.current = Session dialect_to_egg = { "sqlite" : "pysqlite>=2", @@ -120,15 +117,15 @@ Job.table = Table( "job", metadata, Column( "update_time", DateTime, default=now, onupdate=now ), Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ), Column( "tool_id", String( 255 ) ), - Column( "tool_version", String, default="1.0.0" ), + Column( "tool_version", TEXT, default="1.0.0" ), Column( "state", String( 64 ) ), Column( "info", TrimmedString( 255 ) ), - Column( "command_line", String() ), + Column( "command_line", TEXT ), Column( "param_filename", String( 1024 ) ), Column( "runner_name", String( 255 ) ), - Column( "stdout", String() ), - Column( "stderr", String() ), - Column( "traceback", String() ), + Column( "stdout", TEXT ), + Column( "stderr", TEXT ), + Column( "traceback", TEXT ), Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True, nullable=True ), Column( "job_runner_name", String( 255 ) ), Column( "job_runner_external_id", String( 255 ) ) ) @@ -188,7 +185,7 @@ StoredWorkflow.table = Table( "stored_workflow", metadata, Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), Column( "latest_workflow_id", Integer, ForeignKey( "workflow.id", use_alter=True, name='stored_workflow_latest_workflow_id_fk' ), index=True ), - Column( "name", String ), + Column( "name", TEXT ), Column( "deleted", Boolean, default=False ), ) @@ -197,7 +194,7 @@ Workflow.table = Table( "workflow", metadata, Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "stored_workflow_id", Integer, ForeignKey( "stored_workflow.id" ), index=True, nullable=False ), - Column( "name", String ), + Column( "name", TEXT ), Column( "has_cycles", Boolean ), Column( "has_errors", Boolean ) ) @@ -208,8 +205,8 @@ WorkflowStep.table = Table( "workflow_step", metadata, Column( "update_time", DateTime, default=now, onupdate=now ), Column( "workflow_id", Integer, ForeignKey( "workflow.id" ), index=True, nullable=False ), Column( "type", String(64) ), - Column( "tool_id", String ), - Column( "tool_version", String ), # Reserved for future + Column( "tool_id", TEXT ), + Column( "tool_version", TEXT ), # Reserved for future Column( "tool_inputs", JSONType ), Column( "tool_errors", JSONType ), Column( "position", JSONType ), @@ -222,8 +219,8 @@ WorkflowStepConnection.table = Table( "workflow_step_connection", metadata, Column( "id", Integer, primary_key=True ), Column( "output_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True ), Column( "input_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True ), - Column( "output_name", String ), - Column( "input_name", String) + Column( "output_name", TEXT ), + Column( "input_name", TEXT) ) StoredWorkflowUserShareAssociation.table = Table( "stored_workflow_user_share_connection", metadata, @@ -240,7 +237,7 @@ StoredWorkflowMenuEntry.table = Table( "stored_workflow_menu_entry", metadata, MetadataFile.table = Table( "metadata_file", metadata, Column( "id", Integer, primary_key=True ), - Column( "name", String ), + Column( "name", TEXT ), Column( "hda_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True, nullable=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, index=True, default=now, onupdate=now ), @@ -257,9 +254,7 @@ assign_mapper( context, HistoryDatasetAssociation, HistoryDatasetAssociation.tab dataset=relation( Dataset, primaryjoin=( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ), lazy=False ), - history=relation( - History, - primaryjoin=( History.table.c.id == HistoryDatasetAssociation.table.c.history_id ) ), + # .history defined in History mapper copied_to_history_dataset_associations=relation( HistoryDatasetAssociation, primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id == HistoryDatasetAssociation.table.c.id ), @@ -380,11 +375,12 @@ def db_next_hid( self ): Override __next_hid to generate from the database in a concurrency safe way. """ - conn = self.table.engine.contextual_connect() + conn = object_session( self ).connection() + table = self.table trans = conn.begin() try: - next_hid = select( [self.c.hid_counter], self.c.id == self.id, for_update=True ).scalar() - self.table.update( self.c.id == self.id ).execute( hid_counter = ( next_hid + 1 ) ) + next_hid = select( [table.c.hid_counter], table.c.id == self.id, for_update=True ).scalar() + table.update( table.c.id == self.id ).execute( hid_counter = ( next_hid + 1 ) ) trans.commit() return next_hid except: @@ -413,17 +409,21 @@ def init( file_path, url, engine_options={}, create_tables=False ): # Create the database engine engine = create_engine( url, **engine_options ) # Connect the metadata to the database. - metadata.connect( engine ) - ## metadata.engine.echo = True + metadata.bind = engine + # Clear any existing contextual sessions and reconfigure + Session.remove() + Session.configure( bind=engine ) # Create tables if needed if create_tables: metadata.create_all() # metadata.engine.commit() # Pack everything into a bunch result = Bunch( **globals() ) - result.engine = metadata.engine - result.flush = lambda *args, **kwargs: context.current.flush( *args, **kwargs ) - result.context = context + result.engine = engine + result.flush = lambda *args, **kwargs: Session.flush( *args, **kwargs ) + result.session = Session + # For backward compatibility with "model.context.current" + result.context = Session result.create_tables = create_tables return result diff --git a/lib/galaxy/model/mapping_tests.py b/lib/galaxy/model/mapping_tests.py index d0734a418a5..2a1c62b2328 100644 --- a/lib/galaxy/model/mapping_tests.py +++ b/lib/galaxy/model/mapping_tests.py @@ -22,13 +22,13 @@ class MappingTests( unittest.TestCase ): model.context.current.flush() model.context.current.clear() # Check - users = model.User.select() + users = model.User.query().all() assert len( users ) == 1 assert users[0].email == "james@foo.bar.baz" assert users[0].password == "password" assert len( users[0].histories ) == 1 assert users[0].histories[0].name == "History 1" - hists = model.History.select() + hists = model.History.query().all() assert hists[0].name == "History 1" assert hists[1].name == ( "H" * 255 ) assert hists[0].user == users[0] @@ -40,7 +40,7 @@ class MappingTests( unittest.TestCase ): hists[1].name = "History 2b" model.context.current.flush() model.context.current.clear() - hists = model.History.select() + hists = model.History.query().all() assert hists[0].name == "History 1" assert hists[1].name == "History 2b" # gvk TODO need to ad test for GalaxySessions, but not yet sure what they should look like. diff --git a/lib/galaxy/model/orm/__init__.py b/lib/galaxy/model/orm/__init__.py new file mode 100644 index 00000000000..433e1e26f45 --- /dev/null +++ b/lib/galaxy/model/orm/__init__.py @@ -0,0 +1,7 @@ +import pkg_resources +pkg_resources.require( "SQLAlchemy >= 0.4" ) + +from sqlalchemy import * +from sqlalchemy.orm import * + +from sqlalchemy.ext.orderinglist import ordering_list diff --git a/lib/galaxy/model/orm/ext/__init__.py b/lib/galaxy/model/orm/ext/__init__.py new file mode 100644 index 00000000000..230b8aec908 --- /dev/null +++ b/lib/galaxy/model/orm/ext/__init__.py @@ -0,0 +1,3 @@ +""" +Galaxy specific SQLAlchemy extensions. +""" \ No newline at end of file diff --git a/lib/galaxy/model/orm/ext/assignmapper.py b/lib/galaxy/model/orm/ext/assignmapper.py new file mode 100644 index 00000000000..d2b7e2ca7b4 --- /dev/null +++ b/lib/galaxy/model/orm/ext/assignmapper.py @@ -0,0 +1,62 @@ +""" +This is similar to the assignmapper extensions in SQLAclhemy 0.3 and 0.4 but +with some compatibility fixes. It assumes that the session is a ScopedSession, +and thus has the "mapper" method to attach contextual mappers to a class. It +adds additional query and session methods to the class to support the +SQLAlchemy 0.3 style of access. The following methods which would normally be +accessed through "Object.query().method()" are available directly through the +object: + + 'get', 'filter', 'filter_by', 'select', 'select_by', + 'selectfirst', 'selectfirst_by', 'selectone', 'selectone_by', + 'get_by', 'join_to', 'join_via', 'count', 'count_by', + 'options', 'instances' + +Additionally, the following Session methods, which normally accept an instance +or list of instances, are available directly through the objects, e.g. +"Session.flush( [instance] )" can be performed as "instance.flush()": + + 'refresh', 'expire', 'delete', 'expunge', 'update' +""" + +__all__ = [ 'assign_mapper' ] + +from sqlalchemy import util, exceptions +import types +from sqlalchemy.orm import mapper, Query + +def _monkeypatch_query_method( name, session, class_ ): + def do(self, *args, **kwargs): + ## util.warn_deprecated('Query methods on the class are deprecated; use %s.query.%s instead' % (class_.__name__, name)) + return getattr( class_.query, name)(*args, **kwargs) + try: + do.__name__ = name + except: + pass + if not hasattr(class_, name): + setattr(class_, name, classmethod(do)) + +def _monkeypatch_session_method(name, session, class_, make_list=False): + def do(self, *args, **kwargs): + if make_list: + self = [ self ] + return getattr(session, name)( self, *args, **kwargs ) + try: + do.__name__ = name + except: + pass + if not hasattr(class_, name): + setattr(class_, name, do) + +def assign_mapper( session, class_, *args, **kwargs ): + m = class_.mapper = session.mapper( class_, *args, **kwargs ) + for name in ('get', 'filter', 'filter_by', 'select', 'select_by', + 'selectfirst', 'selectfirst_by', 'selectone', 'selectone_by', + 'get_by', 'join_to', 'join_via', 'count', 'count_by', + 'options', 'instances'): + _monkeypatch_query_method(name, session, class_) + for name in ('refresh', 'expire', 'delete', 'expunge', 'update'): + _monkeypatch_session_method(name, session, class_) + for name in ( 'flush', ): + _monkeypatch_session_method( name, session, class_, make_list=True ) + return m diff --git a/lib/galaxy/web/controllers/root.py b/lib/galaxy/web/controllers/root.py index a0b54983ad6..2f52aa90b0f 100644 --- a/lib/galaxy/web/controllers/root.py +++ b/lib/galaxy/web/controllers/root.py @@ -429,7 +429,7 @@ class RootController( BaseController ): if not email: return trans.fill_template("/history/share.mako", histories=histories, email=email, send_to_err=send_to_err) user = trans.get_user() - send_to_user = trans.app.model.User.get_by( email = email ) + send_to_user = trans.app.model.User.filter_by( email=email ).first() if not send_to_user: send_to_err = "No such user" elif user.email == email: @@ -488,7 +488,7 @@ class RootController( BaseController ): new_history.user_id = user.id galaxy_session = trans.get_galaxy_session() try: - association = trans.app.model.GalaxySessionToHistoryAssociation.selectone_by( session_id=galaxy_session.id, history_id=new_history.id ) + association = trans.app.model.GalaxySessionToHistoryAssociation.filter_by( session_id=galaxy_session.id, history_id=new_history.id ).first() except: association = None new_history.add_galaxy_session( galaxy_session, association=association ) @@ -505,7 +505,7 @@ class RootController( BaseController ): new_history.user_id = None galaxy_session = trans.get_galaxy_session() try: - association = trans.app.model.GalaxySessionToHistoryAssociation.selectone_by( session_id=galaxy_session.id, history_id=new_history.id ) + association = trans.app.model.GalaxySessionToHistoryAssociation.filter_by( session_id=galaxy_session.id, history_id=new_history.id ).first() except: association = None new_history.add_galaxy_session( galaxy_session, association=association ) @@ -530,7 +530,7 @@ class RootController( BaseController ): if new_history: galaxy_session = trans.get_galaxy_session() try: - association = trans.app.model.GalaxySessionToHistoryAssociation.selectone_by( session_id=galaxy_session.id, history_id=new_history.id ) + association = trans.app.model.GalaxySessionToHistoryAssociation.filter_by( session_id=galaxy_session.id, history_id=new_history.id ).first() except: association = None new_history.add_galaxy_session( galaxy_session, association=association ) diff --git a/lib/galaxy/web/controllers/user.py b/lib/galaxy/web/controllers/user.py index 41890611abc..c4088534035 100644 --- a/lib/galaxy/web/controllers/user.py +++ b/lib/galaxy/web/controllers/user.py @@ -53,7 +53,7 @@ class User( BaseController ): email_err = "Please enter a real email address" elif len( email) > 255: email_err = "Email address exceeds maximum allowable length" - elif len( trans.app.model.User.select_by( email=email ) ) > 0: + elif trans.app.model.User.filter_by( email=email ).first(): email_err = "User with that email already exists" elif email != conf_email: conf_email_err = "Email addresses do not match." @@ -73,7 +73,7 @@ class User( BaseController ): email_error = password_error = None # Attempt login if email or password: - user = trans.app.model.User.get_by( email = email ) + user = trans.app.model.User.filter_by( email=email ).first() if not user: email_error = "No such user" elif user.external: @@ -108,7 +108,7 @@ class User( BaseController ): email_error = "Please enter a real email address" elif len( email) > 255: email_error = "Email address exceeds maximum allowable length" - elif len( trans.app.model.User.select_by( email=email ) ) > 0: + elif trans.app.model.User.filter_by( email=email ).first(): email_error = "User with that email already exists" elif len( password ) < 6: password_error = "Please use a password of at least 6 characters" @@ -143,7 +143,7 @@ class User( BaseController ): @web.expose def reset_password(self, trans, email=None, **kwd): error = '' - reset_user = trans.app.model.User.get_by( email = email ) + reset_user = trans.app.model.User.filter_by( email=email ).first() user = trans.get_user() if reset_user: if user and user.id != reset_user.id: diff --git a/lib/galaxy/web/controllers/workflow.py b/lib/galaxy/web/controllers/workflow.py index 5719202814e..0e51e2352d1 100644 --- a/lib/galaxy/web/controllers/workflow.py +++ b/lib/galaxy/web/controllers/workflow.py @@ -44,7 +44,7 @@ class WorkflowController( BaseController ): # Load workflow from database stored = get_stored_workflow( trans, id ) if email: - other = model.User.get_by( email=email ) + other = model.User.filter_by( email=email ).first() if not other: mtype = "error" msg = ( "User '%s' does not exist" % email ) diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py index a8f837dbf5c..6c284915624 100644 --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -24,7 +24,7 @@ pkg_resources.require( "Mako" ) import mako.template import mako.lookup -pkg_resources.require( "sqlalchemy>=0.3" ) +pkg_resources.require( "SQLAlchemy >= 0.4" ) from sqlalchemy import desc import logging @@ -172,7 +172,7 @@ class UniverseWebTransaction( base.DefaultWebTransaction ): if secure_id: session_key = self.security.decode_session_key( secure_id ) try: - galaxy_session = self.app.model.GalaxySession.selectone_by( session_key=session_key ) + galaxy_session = self.app.model.GalaxySession.filter_by( session_key=session_key ).first() if galaxy_session and galaxy_session.is_valid and galaxy_session.current_history_id: history = self.app.model.History.get( galaxy_session.current_history_id ) if history and not history.deleted: @@ -216,7 +216,7 @@ class UniverseWebTransaction( base.DefaultWebTransaction ): galaxy_session.user_id = self.user.id try: # See if we have already associated the history with the session - association = self.app.model.GalaxySessionToHistoryAssociation.select_by( session_id=galaxy_session.id, history_id=history.id )[0] + association = self.app.model.GalaxySessionToHistoryAssociation.filter_by( session_id=galaxy_session.id, history_id=history.id ).first() except: association = None history.add_galaxy_session( galaxy_session, association=association ) @@ -265,7 +265,7 @@ class UniverseWebTransaction( base.DefaultWebTransaction ): """Return the user in $HTTP_REMOTE_USER and create if necessary""" # remote_user middleware ensures HTTP_REMOTE_USER exists try: - user = self.app.model.User.selectone_by( email=self.environ[ 'HTTP_REMOTE_USER' ] ) + user = self.app.model.User.filter_by( email=self.environ[ 'HTTP_REMOTE_USER' ] ).first() except: user = self.app.model.User( email=self.environ[ 'HTTP_REMOTE_USER' ] ) user.set_password_cleartext( 'external' ) @@ -281,7 +281,7 @@ class UniverseWebTransaction( base.DefaultWebTransaction ): if secure_id: session_key = self.security.decode_session_key( secure_id ) try: - galaxy_session = self.app.model.GalaxySession.selectone_by( session_key=session_key ) + galaxy_session = self.app.model.GalaxySession.filter_by( session_key=session_key ).first() if galaxy_session and galaxy_session.is_valid and galaxy_session.user_id: user = self.app.model.User.get( galaxy_session.user_id ) if user: @@ -321,7 +321,7 @@ class UniverseWebTransaction( base.DefaultWebTransaction ): session_key = self.security.decode_session_key( secure_id ) try: # Retrive the galaxy_session id via the unique session_key - galaxy_session = self.app.model.GalaxySession.selectone_by( session_key=session_key ) + galaxy_session = self.app.model.GalaxySession.filter_by( session_key=session_key ).first() if galaxy_session and galaxy_session.is_valid: self.__galaxy_session = galaxy_session except: @@ -382,7 +382,7 @@ class UniverseWebTransaction( base.DefaultWebTransaction ): if self.history is not None: # See if we have already associated the session with the history try: - association = self.app.model.GalaxySessionToHistoryAssociation.select_by( session_id=galaxy_session.id, history_id=self.history.id )[0] + association = self.app.model.GalaxySessionToHistoryAssociation.filter_by( session_id=galaxy_session.id, history_id=self.history.id ).first() except: association = None galaxy_session.add_history( self.history, association=association ) diff --git a/lib/galaxy/webapps/reports/controllers/jobs.py b/lib/galaxy/webapps/reports/controllers/jobs.py index bf7935d9647..a502ca48532 100644 --- a/lib/galaxy/webapps/reports/controllers/jobs.py +++ b/lib/galaxy/webapps/reports/controllers/jobs.py @@ -5,7 +5,7 @@ import calendar from galaxy.webapps.reports.base.controller import * import galaxy.model import pkg_resources -pkg_resources.require( "sqlalchemy>=0.3" ) +pkg_resources.require( "SQLAlchemy >= 0.4" ) import sqlalchemy as sa import logging log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/webapps/reports/controllers/system.py b/lib/galaxy/webapps/reports/controllers/system.py index 33e7cc8f84b..e4017544460 100644 --- a/lib/galaxy/webapps/reports/controllers/system.py +++ b/lib/galaxy/webapps/reports/controllers/system.py @@ -2,8 +2,9 @@ import operator, os from datetime import datetime, timedelta from galaxy.webapps.reports.base.controller import * import pkg_resources -pkg_resources.require( "sqlalchemy>=0.3" ) -from sqlalchemy import eagerload, desc +pkg_resources.require( "SQLAlchemy >= 0.4" ) +from sqlalchemy.orm import eagerload +from sqlalchemy import desc import logging log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/webapps/reports/controllers/users.py b/lib/galaxy/webapps/reports/controllers/users.py index b99546f14dd..185a18107c3 100644 --- a/lib/galaxy/webapps/reports/controllers/users.py +++ b/lib/galaxy/webapps/reports/controllers/users.py @@ -3,7 +3,7 @@ import calendar from galaxy.webapps.reports.base.controller import * import galaxy.model import pkg_resources -pkg_resources.require( "sqlalchemy>=0.3" ) +pkg_resources.require( "SQLAlchemy >= 0.4" ) import sqlalchemy as sa import logging log = logging.getLogger( __name__ ) diff --git a/scripts/cleanup_datasets/cleanup_datasets.py b/scripts/cleanup_datasets/cleanup_datasets.py index 32d96bd913c..8cc12e6f33b 100644 --- a/scripts/cleanup_datasets/cleanup_datasets.py +++ b/scripts/cleanup_datasets/cleanup_datasets.py @@ -13,8 +13,8 @@ from galaxy import eggs import galaxy.model.mapping import pkg_resources -pkg_resources.require( "sqlalchemy>=0.3" ) -from sqlalchemy import eagerload +pkg_resources.require( "SQLAlchemy >= 0.4" ) +from sqlalchemy.orm import eagerload assert sys.version_info[:2] >= ( 2, 4 ) @@ -191,8 +191,6 @@ def purge_histories( h, d, m, cutoff_time, remove_from_disk ): if errmsg: errors = True print errmsg - else: - print "%s" % dataset.file_name else: dataset.purged = True dataset.flush() @@ -258,7 +256,6 @@ def purge_datasets( d, m, cutoff_time, remove_from_disk ): print errmsg else: dataset_count += 1 - print "%s" % dataset.file_name else: dataset.purged = True dataset.file_size = 0 @@ -302,6 +299,7 @@ def purge_dataset( dataset, m ): else: # Remove dataset file from disk os.unlink( dataset.file_name ) + print "%s" % dataset.file_name # Mark all associated MetadataFiles as deleted and purged and remove them from disk print "The following metadata files associated with dataset '%s' have been purged" % dataset.file_name for hda in dataset.history_associations: diff --git a/tools/stats/grouping.py b/tools/stats/grouping.py index d21a6da6f4e..a26fb033f0b 100644 --- a/tools/stats/grouping.py +++ b/tools/stats/grouping.py @@ -90,7 +90,7 @@ def main(): for ii, line in enumerate( file( tmpfile.name )): if line and not line.startswith( '#' ): - line = line.strip() + line = line.rstrip( '\r\n' ) try: fields = line.split("\t") item = fields[group_col] From 952c12bd60ce7912375ee12aeb4a118713b298fa Mon Sep 17 00:00:00 2001 From: James Taylor Date: Thu, 30 Oct 2008 17:34:46 -0400 Subject: [PATCH 4/6] Convert tool_form/tool_executed to mako (for performance) --- lib/galaxy/tools/__init__.py | 10 +- ...{tool_executed.tmpl => tool_executed.mako} | 39 ++-- templates/tool_form.mako | 194 ++++++++++++++++++ templates/tool_form.tmpl | 191 ----------------- 4 files changed, 215 insertions(+), 219 deletions(-) rename templates/{tool_executed.tmpl => tool_executed.mako} (61%) create mode 100644 templates/tool_form.mako delete mode 100644 templates/tool_form.tmpl diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 3c0eb30631c..81574a3986f 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -676,7 +676,7 @@ class Tool: # on the standard run form) or "URL" (a parameter provided by # external data source tools). if "runtool_btn" not in incoming and "URL" not in incoming: - return "tool_form.tmpl", dict( errors={}, tool_state=state, param_values={}, incoming={} ) + return "tool_form.mako", dict( errors={}, tool_state=state, param_values={}, incoming={} ) # Process incoming data if not( self.check_values ): # If `self.check_values` is false we don't do any checking or @@ -702,20 +702,20 @@ class Tool: # error messages if errors: error_message = "One or more errors were found in the input you provided. The specific errors are marked below." - return "tool_form.tmpl", dict( errors=errors, tool_state=state, incoming=incoming, error_message=error_message ) + return "tool_form.mako", dict( errors=errors, tool_state=state, incoming=incoming, error_message=error_message ) # If we've completed the last page we can execute the tool elif state.page == self.last_page: out_data = self.execute( trans, incoming=params ) - return 'tool_executed.tmpl', dict( out_data=out_data ) + return 'tool_executed.mako', dict( out_data=out_data ) # Otherwise move on to the next page else: state.page += 1 # Fill in the default values for the next page self.fill_in_new_state( trans, self.inputs_by_page[ state.page ], state.inputs ) - return 'tool_form.tmpl', dict( errors=errors, tool_state=state ) + return 'tool_form.mako', dict( errors=errors, tool_state=state ) else: # Just a refresh, render the form with updated state and errors. - return 'tool_form.tmpl', dict( errors=errors, tool_state=state ) + return 'tool_form.mako', dict( errors=errors, tool_state=state ) def update_state( self, trans, inputs, state, incoming, prefix="", context=None, update_only=False, old_errors={}, changed_dependencies={} ): diff --git a/templates/tool_executed.tmpl b/templates/tool_executed.mako similarity index 61% rename from templates/tool_executed.tmpl rename to templates/tool_executed.mako index fa9e98eaa18..9fd7ed1bc08 100644 --- a/templates/tool_executed.tmpl +++ b/templates/tool_executed.mako @@ -4,37 +4,31 @@ Galaxy - + @@ -43,14 +37,13 @@ -

The following job has been succesfully added to the queue:

-#for $data in $out_data.values -
$data.hid: $data.name
-#end for +%for data in out_data.values(): +
${data.hid}: ${data.name}
+%endfor

You can check the status of queued jobs and view the resulting @@ -59,9 +52,9 @@ the status will change from 'running' to 'finished' if completed succesfully or 'error' if problems were encountered.

-#if $tool.options.refresh -
-#end if +%if tool.options.refresh: +
+
${input.title_plural}
+ <% repeat_state = tool_state[input.name] %> + %for i in range( len( repeat_state ) ): +
+ <% + if input.name in errors: + rep_errors = errors[input.name][i] + else: + rep_errors = dict() + index = repeat_state[i]['__index__'] + %> +
${input.title} ${i + 1}
+ ${do_inputs( input.inputs, repeat_state[i], rep_errors, prefix + input.name + "_" + str(index) + "|", other_values )} +
+
+ %endfor +
+
+ %elif input.type == "conditional": + <% + group_state = tool_state[input.name] + group_errors = errors.get( input.name, {} ) + current_case = group_state['__current_case__'] + group_prefix = prefix + input.name + "|" + %> + ${row_for_param( group_prefix, input.test_param, group_state, group_errors, other_values )} + ${do_inputs( input.cases[current_case].inputs, group_state, group_errors, group_prefix, other_values )} + %else: + ${row_for_param( prefix, input, tool_state, errors, other_values )} + %endif + %endfor + + +<%def name="row_for_param( prefix, param, parent_state, parent_errors, other_values )"> + <% + if parent_errors.has_key( param.name ): + cls = "form-row form-row-error" + else: + cls = "form-row" + %> +
+ <% label = param.get_label() %> + %if label: + + %endif + <% + field = param.get_html_field( trans, parent_state[ param.name ], other_values ) + field.refresh_on_change = param.refresh_on_change + %> +
${field.get_html( prefix )}
+ %if parent_errors.has_key( param.name ): +
+
 ${parent_errors[param.name]}
+
+ %endif + + %if param.help: +
+ ${param.help} +
+ %endif + +
+ +
+ + +%if add_frame.from_noframe: +
+ Welcome to Galaxy +
+ It appears that you found this tool from a link outside of Galaxy. + If you're not familiar with Galaxy, please consider visiting the + welcome page. + To learn more about what Galaxy is and what it can do for you, please visit + the Galaxy wiki. +
+
+%endif + +
+ %if tool.has_multiple_pages: +
${tool.name} (step ${tool_state.page+1} of ${tool.npages})
+ %else: +
${tool.name}
+ %endif +
+
+ + + + %if tool.display_by_page[tool_state.page]: + ${trans.fill_template_string( tool.display_by_page[tool_state.page], other_values=tool.get_param_html_map( trans, tool_state.page, tool_state.inputs ) )} + + + %else: + +
+ %if tool_state.page == tool.last_page: + + %else: + + %endif +
+ ${do_inputs( tool.inputs_by_page[ tool_state.page ], tool_state.inputs, errors, "" )} +
+ %if tool_state.page == tool.last_page: + + %else: + + %endif +
+ + %endif + +
+
+
+ +%if tool.help: +
+
+ %if tool.has_multiple_pages: + ${tool.help_by_page[tool_state.page]} + %else: + ${tool.help} + %endif +
+
+%endif + + + + + + diff --git a/templates/tool_form.tmpl b/templates/tool_form.tmpl deleted file mode 100644 index 86f359e0849..00000000000 --- a/templates/tool_form.tmpl +++ /dev/null @@ -1,191 +0,0 @@ - - - -#from galaxy.util.expressions import ExpressionContext - - - - -Galaxy - - - - - - - - -## #if $getVar( 'error_message', None ) -##
$error_message
-##

-## #end if - -#def do_inputs( $inputs, $tool_state, $errors, $prefix, $context=None ) - #set $context = ExpressionContext( $tool_state, $context ) - #for $input_index, $input in enumerate( $inputs.itervalues() ) - #if $input.type == "repeat" -
-
${input.title_plural}
- #set repeat_state = $tool_state[$input.name] - #for i in range( len( $repeat_state ) ): -
- #if $input.name in errors - #set rep_errors = $errors[$input.name][$i] - #else - #set rep_errors = dict() - #end if - #set index = $repeat_state[$i]['__index__'] -
${input.title} ${i + 1}
- $do_inputs( $input.inputs, $repeat_state[$i], $rep_errors, $prefix + $input.name + "_" + str($index) + "|", $context ) -
-
- #end for -
-
- #elif $input.type == "conditional" - #set group_state = $tool_state[$input.name] - #set group_errors = $errors.get( $input.name, {} ) - #set current_case = $group_state['__current_case__'] - #set group_prefix = $prefix + $input.name + "|" - $row_for_param( $group_prefix, $input.test_param, $group_state, $group_errors, $context ) - $do_inputs( $input.cases[$current_case].inputs, $group_state, $group_errors, $group_prefix, $context ) - #else - $row_for_param( $prefix, $input, $tool_state, $errors, $context ) - #end if - #end for -#end def - -#def row_for_param( $prefix, $param, $parent_state, $parent_errors, $context ) - #if $parent_errors.has_key( $param.name ): - #set cls = "form-row form-row-error" - #else - #set cls = "form-row" - #end if -
- #set label = $param.get_label() - #if $label: - - #end if - #set field = $param.get_html_field( $caller, $parent_state[ $param.name ], $context ) - #set $field.refresh_on_change = $param.refresh_on_change -
$field.get_html( $prefix )
- #if $parent_errors.has_key( $param.name ): -
-
 $parent_errors[$param.name]
-
- #elif $param.help - ##
$param.help
- #end if - - #if $param.help -
- $param.help -
- - #end if - -
- -
-#end def - -#if $add_frame.from_noframe -
- Welcome to Galaxy -
- It appears that you found this tool from a link outside of Galaxy. If you're not familiar with Galaxy, please consider visiting the welcome page. To learn more about what Galaxy is and what it can do for you, please visit the Galaxy wiki. -
-
-#end if - -
- #if $tool.has_multiple_pages -
$tool.name (step #echo $tool_state.page+1 # of $tool.npages)
- #else -
$tool.name
- #end if -
-
- - - - #if $tool.display_by_page[$tool_state.page] - - $caller.fill_template_string( $tool.display_by_page[$tool_state.page], context=$tool.get_param_html_map( $caller, $tool_state.page, $tool_state.inputs ) ) - - - #else - -
- #if $tool_state.page == $tool.last_page - - #else - - #end if -
- $do_inputs( $tool.inputs_by_page[ $tool_state.page ], $tool_state.inputs, $errors, "" ) -
- #if $tool_state.page == $tool.last_page - - #else - - #end if -
- - #end if - -
-
-
- -#if $tool.help -
-
- #if $tool.has_multiple_pages - $tool.help_by_page[$tool_state.page] - #else - $tool.help - #end if -
-
-#end if - - - - - - From edca52e0f247efb97f8968f72dfb648e2d160dfa Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 31 Oct 2008 10:22:46 -0400 Subject: [PATCH 5/6] James' changes allowing the server to run w/o a job runner, and nginx-specific performance improvements. --- lib/galaxy/app.py | 11 ++-- lib/galaxy/config.py | 10 ++++ lib/galaxy/jobs/__init__.py | 68 +++++++++++-------------- lib/galaxy/model/__init__.py | 30 ++++++++++- lib/galaxy/tools/__init__.py | 13 ++++- lib/galaxy/tools/actions/upload.py | 23 ++++++--- lib/galaxy/tools/parameters/__init__.py | 4 +- lib/galaxy/tools/parameters/basic.py | 20 +++++++- lib/galaxy/web/controllers/root.py | 61 +++++++++------------- lib/galaxy/web/framework/base.py | 23 +++++++-- universe_wsgi.ini.sample | 39 +++++++------- 11 files changed, 187 insertions(+), 115 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index f59ae433613..ed10d43396f 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -31,9 +31,11 @@ class UniverseApplication( object ): #Load datatype converters self.datatypes_registry.load_datatype_converters( self.toolbox ) # Start the job queue - job_dispatcher = jobs.DefaultJobDispatcher( self ) - self.job_queue = jobs.JobQueue( self, job_dispatcher ) - self.job_stop_queue = jobs.JobStopQueue( self, job_dispatcher ) + self.job_manager = jobs.JobManager( self ) + # FIXME: These are exposed directly for backward compatibility + self.job_queue = self.job_manager.job_queue + self.job_stop_queue = self.job_manager.job_stop_queue + # Heartbeat and memdump for thread / heap profiling self.heartbeat = None self.memdump = None # Start the heartbeat process if configured and available @@ -48,7 +50,6 @@ class UniverseApplication( object ): if memdump.Memdump: self.memdump = memdump.Memdump() def shutdown( self ): - self.job_stop_queue.shutdown() - self.job_queue.shutdown() + self.job_manager.shutdown() if self.heartbeat: self.heartbeat.shutdown() diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 7e6b32088c1..58dc3a87b9a 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -64,6 +64,11 @@ class Configuration( object ): self.bugs_email = kwargs.get( 'bugs_email', None ) self.blog_url = kwargs.get( 'blog_url', None ) self.screencasts_url = kwargs.get( 'screencasts_url', None ) + # Configuration options for taking advantage of nginx features + self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False ) + self.nginx_upload_location = kwargs.get( 'nginx_upload_store', False ) + if self.nginx_upload_location: + self.nginx_upload_location = os.path.abspath( self.nginx_upload_location ) # Parse global_conf and save the parser global_conf = kwargs.get( 'global_conf', None ) global_conf_parser = ConfigParser.ConfigParser() @@ -78,6 +83,11 @@ class Configuration( object ): self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' ) def get( self, key, default ): return self.config_dict.get( key, default ) + def get_bool( self, key, default ): + if key in self.config_dict: + return string_as_bool( key ) + else: + return default def check( self ): # Check that required directories exist for path in self.root, self.file_path, self.tool_path, self.tool_data_path, self.template_path, self.job_working_directory: diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 90a40d6a2d7..2b0f6705252 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -17,6 +17,27 @@ log = logging.getLogger( __name__ ) # States for running a job. These are NOT the same as data states JOB_WAIT, JOB_ERROR, JOB_INPUT_ERROR, JOB_INPUT_DELETED, JOB_OK, JOB_READY, JOB_DELETED = 'wait', 'error', 'input_error', 'input_deleted', 'ok', 'ready', 'deleted' +class JobManager( object ): + """ + Highest level interface to job management. + + TODO: Currently the app accesses "job_queue" and "job_stop_queue" directly. + This should be decoupled. + """ + def __init__( self, app ): + self.app = app + if self.app.config.get_bool( "enable_job_running", True ): + # The dispatcher launches the underlying job runners + self.dispatcher = DefaultJobDispatcher( app ) + # Queues for starting and stopping jobs + self.job_queue = JobQueue( app, self.dispatcher ) + self.job_stop_queue = JobStopQueue( app, self.dispatcher ) + else: + self.job_queue = self.job_stop_queue = NoopQueue() + def shutdown( self ): + self.job_queue.shutdown() + self.job_stop_queue.shutdown() + class Sleeper( object ): """ Provides a 'sleep' method that sleeps for a number of seconds *unless* @@ -594,50 +615,12 @@ class JobStopQueue( object ): pass for job in jobs: - # jobs in a non queued/running/new state do not need to be stopped - if job.state not in [ model.Job.states.QUEUED, model.Job.states.RUNNING, model.Job.states.NEW ]: - return - # job has multiple datasets that aren't parent/child and not all of them are deleted. - if not self.check_if_output_datasets_deleted( job.id ): - return - self.mark_deleted( job.id ) # job is in JobQueue or FooJobRunner, will be dequeued due to state change above if job.job_runner_name is None: return # tell the dispatcher to stop the job self.dispatcher.stop( job ) - def check_if_output_datasets_deleted( self, job_id ): - job = model.Job.get( job_id ) - for dataset_assoc in job.output_datasets: - dataset = dataset_assoc.dataset - dataset.refresh() - #only the originator of the job can delete a dataset to cause - #cancellation of the job, no need to loop through history_associations - if not dataset.deleted: - return False - return True - - def mark_deleted( self, job_id ): - job = model.Job.get( job_id ) - job.refresh() - job.state = job.states.DELETED - job.info = "Job output deleted by user before job completed." - job.flush() - for dataset_assoc in job.output_datasets: - dataset = dataset_assoc.dataset - dataset.refresh() - dataset.deleted = True - dataset.state = dataset.states.DISCARDED - dataset.dataset.flush() - for dataset in dataset.dataset.history_associations: - #propagate info across shared datasets - dataset.deleted = True - dataset.blurb = 'deleted' - dataset.peek = 'Job deleted' - dataset.info = 'Job output deleted by user before job completed' - dataset.flush() - def put( self, job ): self.queue.put( job ) @@ -652,3 +635,12 @@ class JobStopQueue( object ): self.queue.put( self.STOP_SIGNAL ) self.sleeper.wake() log.info( "job stopper stopped" ) + +class NoopQueue( object ): + """ + Implements the JobQueue / JobStopQueue interface but does nothing + """ + def put( self, *args ): + return + def shutdown( self ): + return \ No newline at end of file diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index de157193e13..5e9b8291dac 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -85,7 +85,35 @@ class Job( object ): tool = app.toolbox.tools_by_id[self.tool_id] param_dict = tool.params_from_strings( param_dict, app ) return param_dict - + def check_if_output_datasets_deleted( self ): + """ + Return true if all of the output datasets associated with this job are + in the deleted state + """ + for dataset_assoc in self.output_datasets: + dataset = dataset_assoc.dataset + # only the originator of the job can delete a dataset to cause + # cancellation of the job, no need to loop through history_associations + if not dataset.deleted: + return False + return True + def mark_deleted( self ): + """ + Mark this job as deleted, and mark any output datasets as discarded. + """ + self.state = Job.states.DELETED + self.info = "Job output deleted by user before job completed." + for dataset_assoc in self.output_datasets: + dataset = dataset_assoc.dataset + dataset.deleted = True + dataset.state = dataset.states.DISCARDED + for dataset in dataset.dataset.history_associations: + # propagate info across shared datasets + dataset.deleted = True + dataset.blurb = 'deleted' + dataset.peek = 'Job deleted' + dataset.info = 'Job output deleted by user before job completed' + class JobParameter( object ): def __init__( self, name, value ): self.name = name diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 81574a3986f..50f273d0042 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -797,7 +797,7 @@ class Tool: # Deal with the 'test' element and see if it's value changed test_param_key = group_prefix + input.test_param.name test_param_error = None - test_incoming = incoming.get( test_param_key, None ) + test_incoming = get_incoming_value( incoming, test_param_key, None ) if test_param_key not in incoming \ and "__force_update__" + test_param_key not in incoming \ and update_only: @@ -878,7 +878,7 @@ class Tool: except: pass if not incoming_value_generated: - incoming_value = incoming.get( key, None ) + incoming_value = get_incoming_value( incoming, key, None ) value, error = check_param( trans, input, incoming_value, context ) if input.dependent_params and state[ input.name ] != value: # We need to keep track of changed dependency parametrs ( parameters @@ -1362,3 +1362,12 @@ def json_fix( val ): else: return val +def get_incoming_value( incoming, key, default ): + if "__" + key + "__is_composite" in incoming: + composite_keys = incoming["__" + key + "__keys"].split() + value = dict() + for composite_key in composite_keys: + value[composite_key] = incoming[key + "_" + composite_key] + return value + else: + return incoming.get( key, default ) \ No newline at end of file diff --git a/lib/galaxy/tools/actions/upload.py b/lib/galaxy/tools/actions/upload.py index 25bdb29baed..ae1f3b39c2a 100644 --- a/lib/galaxy/tools/actions/upload.py +++ b/lib/galaxy/tools/actions/upload.py @@ -26,12 +26,22 @@ class UploadToolAction( object ): temp_name = "" data_list = [] - if 'filename' in dir( data_file ): + if 'local_filename' in dir( data_file ): + # Use the existing file try: file_name = data_file.filename file_name = file_name.split( '\\' )[-1] file_name = file_name.split( '/' )[-1] - data_list.append( self.add_file( trans, data_file.file, file_name, file_type, dbkey, space_to_tab=space_to_tab ) ) + data_list.append( self.add_file( trans, data_file.local_filename, file_name, file_type, dbkey, space_to_tab=space_to_tab ) ) + except Exception, e: + return self.upload_empty( trans, "Error:", str( e ) ) + elif 'filename' in dir( data_file ): + try: + file_name = data_file.filename + file_name = file_name.split( '\\' )[-1] + file_name = file_name.split( '/' )[-1] + temp_name = sniff.stream_to_file( data_file.file ) + data_list.append( self.add_file( trans, temp_name, file_name, file_type, dbkey, space_to_tab=space_to_tab ) ) except Exception, e: return self.upload_empty( trans, "Error:", str( e ) ) if url_paste not in [ None, "" ]: @@ -41,7 +51,8 @@ class UploadToolAction( object ): line = line.rstrip( '\r\n' ) if line: try: - data_list.append( self.add_file( trans, urllib.urlopen( line ), line, file_type, dbkey, info="uploaded url", space_to_tab=space_to_tab ) ) + temp_name = sniff.stream_to_file( urllib.urlopen( line ) ) + data_list.append( self.add_file( trans, temp_name, line, file_type, dbkey, info="uploaded url", space_to_tab=space_to_tab ) ) except Exception, e: return self.upload_empty( trans, "Error:", str( e ) ) else: @@ -53,7 +64,8 @@ class UploadToolAction( object ): break if is_valid: try: - data_list.append( self.add_file( trans, StringIO.StringIO( url_paste ), 'Pasted Entry', file_type, dbkey, info="pasted entry", space_to_tab=space_to_tab ) ) + temp_name = sniff.stream_to_file( StringIO.StringIO( url_paste ) ) + data_list.append( self.add_file( trans, temp_name, 'Pasted Entry', file_type, dbkey, info="pasted entry", space_to_tab=space_to_tab ) ) except Exception, e: return self.upload_empty( trans, "Error:", str( e ) ) else: @@ -77,9 +89,8 @@ class UploadToolAction( object ): trans.app.model.flush() return dict( output=data ) - def add_file( self, trans, file_obj, file_name, file_type, dbkey, info=None, space_to_tab=False ): + def add_file( self, trans, temp_name, file_name, file_type, dbkey, info=None, space_to_tab=False ): data_type = None - temp_name = sniff.stream_to_file( file_obj ) # See if we have an empty file if not os.path.getsize( temp_name ) > 0: diff --git a/lib/galaxy/tools/parameters/__init__.py b/lib/galaxy/tools/parameters/__init__.py index f8cda91c0de..a17fdcd5d5e 100644 --- a/lib/galaxy/tools/parameters/__init__.py +++ b/lib/galaxy/tools/parameters/__init__.py @@ -16,9 +16,7 @@ def check_param( trans, param, incoming_value, param_values ): value = incoming_value error = None try: - if param.name == 'file_data': - pass - elif value is not None or isinstance(param, DataToolParameter): + if value is not None or isinstance(param, DataToolParameter): # Convert value from HTML representation value = param.from_html( value, trans, param_values ) # Allow the value to be converted if neccesary diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 2b1cbb5abb0..b43a066e22e 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -2,12 +2,13 @@ Basic tool parameters. """ -import logging, string, sys, os +import logging, string, sys, os, os.path from elementtree.ElementTree import XML, Element from galaxy import config, datatypes, util from galaxy.web import form_builder +from galaxy.util.bunch import Bunch import validation, dynamic_options @@ -294,6 +295,23 @@ class FileToolParameter( ToolParameter ): self.name = elem.get( 'name' ) def get_html_field( self, trans=None, value=None, other_values={} ): return form_builder.FileField( self.name ) + def from_html( self, value, trans=None, other_values={} ): + # Middleware or proxies may encode files in special ways (TODO: this + # should be pluggable) + if type( value ) == dict: + upload_location = self.tool.app.config.nginx_upload_location + assert upload_location, \ + "Request appears to have been processed by nginx_upload_module \ + but Galaxy is not configured to recgonize it" + # Check that the file is in the right location + local_filename = os.path.abspath( value['path'] ) + assert local_filename.startswith( upload_location ), \ + "Filename provided by nginx is not in correct directory" + value = Bunch( + filename = value["name"], + local_filename = local_filename + ) + return value def get_required_enctype( self ): """ File upload elements require the multipart/form-data encoding diff --git a/lib/galaxy/web/controllers/root.py b/lib/galaxy/web/controllers/root.py index 2f52aa90b0f..b262836f18e 100644 --- a/lib/galaxy/web/controllers/root.py +++ b/lib/galaxy/web/controllers/root.py @@ -266,6 +266,28 @@ class RootController( BaseController ): return trans.fill_template( "/dataset/edit_attributes.mako", data=data, datatypes=ldatatypes, err=None ) + def __delete_dataset( self, trans, id ): + data = self.app.model.HistoryDatasetAssociation.get( id ) + if data: + # Walk up parent datasets to find the containing history + topmost_parent = data + while topmost_parent.parent: + topmost_parent = topmost_parent.parent + assert topmost_parent in trans.history.datasets, "Data does not belong to current history" + # Mark deleted and cleanup + data.mark_deleted() + data.clear_associated_files() + trans.log_event( "Dataset id %s marked as deleted" % str(id) ) + if data.parent_id is None and len( data.creating_job_associations ) > 0: + # Mark associated job for deletion + job = data.creating_job_associations[0].job + if job.state not in [ model.Job.states.QUEUED, model.Job.states.RUNNING, model.Job.states.NEW ]: + return + # Are *all* of the job's other output datasets deleted? + if job.check_if_output_datasets_deleted(): + job.mark_deleted() + self.app.model.flush() + @web.expose def delete( self, trans, id = None, show_deleted_on_refresh = False, **kwd): if id: @@ -276,26 +298,10 @@ class RootController( BaseController ): history = trans.get_history() for id in dataset_ids: try: - int( id ) + id = int( id ) except: continue - data = self.app.model.HistoryDatasetAssociation.get( id ) - if data: - # Walk up parent datasets to find the containing history - topmost_parent = data - while topmost_parent.parent: - topmost_parent = topmost_parent.parent - assert topmost_parent in history.datasets, "Data does not belong to current history" - # Mark deleted and cleanup - data.mark_deleted() - data.clear_associated_files() - self.app.model.flush() - trans.log_event( "Dataset id %s marked as deleted" % str(id) ) - if data.parent_id is None: - try: - self.app.job_stop_queue.put( data.creating_job_associations[0].job ) - except IndexError: - pass # upload tool will cause this since it doesn't have a job + self.__delete_dataset( trans, id ) return self.history( trans, show_deleted = show_deleted_on_refresh ) @web.expose @@ -305,24 +311,7 @@ class RootController( BaseController ): int( id ) except: return "Dataset id '%s' is invalid" %str( id ) - history = trans.get_history() - data = self.app.model.HistoryDatasetAssociation.get( id ) - if data: - # Walk up parent datasets to find the containing history - topmost_parent = data - while topmost_parent.parent: - topmost_parent = topmost_parent.parent - assert topmost_parent in history.datasets, "Data does not belong to current history" - # Mark deleted and cleanup - data.mark_deleted() - data.clear_associated_files() - self.app.model.flush() - trans.log_event( "Dataset id %s marked as deleted async" % str(id) ) - if data.parent_id is None: - try: - self.app.job_stop_queue.put( data.creating_job_associations[0].job ) - except IndexError: - pass # upload tool will cause this since it doesn't have a job + self.__delete_dataset( trans, id ) return "OK" ## ---- History management ----------------------------------------------- diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index 470fbdf9f30..6dff0719d2a 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -5,6 +5,7 @@ A simple WSGI application/framework. import socket import types import logging +import os.path import sys from Cookie import SimpleCookie @@ -132,16 +133,16 @@ class WebApplication( object ): if callable( body ): # Assume the callable is another WSGI application to run return body( environ, start_response ) + elif isinstance( body, types.FileType ): + # Stream the file back to the browser + return send_file( start_response, trans, body ) else: start_response( trans.response.wsgi_status(), trans.response.wsgi_headeritems() ) return self.make_body_iterable( trans, body ) def make_body_iterable( self, trans, body ): - if isinstance( body, types.FileType ): - # Stream the file back to the browser - return iterate_file( body ) - elif isinstance( body, ( types.GeneratorType, list, tuple ) ): + if isinstance( body, ( types.GeneratorType, list, tuple ) ): # Recursively stream the iterable return flatten( body ) elif isinstance( body, basestring ): @@ -302,6 +303,20 @@ class Response( object ): CHUNK_SIZE = 2**16 +def send_file( start_response, trans, body ): + # If configured use X-Accel-Redirect header for nginx + base = trans.app.config.nginx_x_accel_redirect_base + if base: + trans.response.headers['X-Accel-Redirect'] = \ + base + os.path.abspath( body.name ) + body = [ "" ] + # Fall back on sending the file in chunks + else: + body = iterate_file( body ) + start_response( trans.response.wsgi_status(), + trans.response.wsgi_headeritems() ) + return body + def iterate_file( file ): """ Progressively return chunks from `file`. diff --git a/universe_wsgi.ini.sample b/universe_wsgi.ini.sample index 6dcdf061f94..b0ed9136041 100644 --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -34,7 +34,7 @@ job_queue_cleanup_interval = 30 # Database connection database_file = database/universe.sqlite # You may use a SQLAlchemy connection string to specify an external database instead -## database_connection = postgres:///galaxy_test +## database_connection = postgres:///galaxy ## database_engine_option_echo = true ## database_engine_option_echo_pool = true ## database_engine_option_pool_size = 10 @@ -89,12 +89,15 @@ mailing_join_addr = galaxy-user-join@bx.psu.edu # Write thread status periodically to 'heartbeat.log' (careful, uses disk space rapidly!) ## use_heartbeat = True +# Enable the memory debugging interface (careful, negatively impacts server performance) +## use_memdump = True + # Profiling middleware (cProfile based) ## use_profile = True -# Mail -smtp_server = coltrane.bx.psu.edu -error_email_to = galaxy-bugs@bx.psu.edu +# For use by 'report this error' link on error-state datasets +#smtp_server = smtp.example.org +#error_email_to = galaxy-bugs@example.org # Use the new iframe / javascript based layout use_new_layout = true @@ -120,29 +123,34 @@ static_style_dir = %(here)s/static/june_2007_style/blue ## wiki_url: replaces the default galaxy main wiki ## bugs_email: replaces the default galaxy bugs email list #brand = Private local mirror -#wiki_url=/path/to/my/local/wiki -#bugs_email=mailto:bugmaster@this.site.com +#wiki_url = /path/to/my/local/wiki +#bugs_email = mailto:galaxy-bugs@example.org # ---- Job Runners ---------------------------------------------------------- # Clustering Galaxy is not a straightforward process and requires a lot of -# pre-configuration. See the ClusteringGalaxy Wiki before attempting to set any -# of these options. If running normally (without a cluster), do not change -# anything in this section. +# pre-configuration. See the ClusteringGalaxy Wiki before attempting to set +# any of these options: +# +# http://g2.trac.bx.psu.edu/wiki/ClusteringGalaxy +# +# If running normally (without a cluster), do not change anything in this +# section. # start_job_runners: Comma-separated list of job runners to start. local is # always started. If left commented, no jobs will be run on the cluster, even # if a cluster URL is explicitly defined in the [galaxy:tool_runners] section -# below. The only runner currently available is 'pbs'. +# below. The runners currently available are 'pbs' and 'sge'. #start_job_runners = pbs # default_cluster_job_runner: The URL for the default runner to use when a tool # doesn't explicity define a runner below. For help on the cluster URL format, -# see the ClusteringGalaxy Wiki. Leave commented if not using a cluster job runner. +# see the ClusteringGalaxy Wiki. Leave commented if not using a cluster job +# runner. #default_cluster_job_runner = pbs:/// # The PBS options are described in detail in the Galaxy Configuration section of -# the ClusteringGalaxy Wiki +# the ClusteringGalaxy Wiki, and are only necessary when using file staging. #pbs_application_server = #pbs_stage_path = #pbs_dataset_server = @@ -152,8 +160,6 @@ static_style_dir = %(here)s/static/june_2007_style/blue [galaxy:tool_runners] biomart = local:/// -blat2wig = pbs:///blast -blat_wrapper = pbs:///blast encode_db1 = local:/// encode_import_all_latest_datasets1 = local:/// encode_import_chromatin_and_chromosomes1 = local:/// @@ -161,13 +167,8 @@ encode_import_gencode1 = local:/// encode_import_genes_and_transcripts1 = local:/// encode_import_multi-species_sequence_analysis1 = local:/// encode_import_transcription_regulation1 = local:/// -generate_coverage_report = pbs:///blast hbvar = local:/// -hist_high_quality_score = pbs:///blast -megablast_wrapper = pbs:///blast -megablast_xml_parser = pbs:///blast microbial_import1 = local:/// -quality_score_distribution = pbs:///blast ucsc_table_direct1 = local:/// ucsc_table_direct_archaea1 = local:/// ucsc_table_direct_test1 = local:/// From 35168f40a52713e6601f8f8ece56befbc50883cf Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 31 Oct 2008 11:00:50 -0400 Subject: [PATCH 6/6] Fixes for dealing with UnvalidatedValues (i.e. when running workflows) and also when passing None trans to dynamic options. --- lib/galaxy/jobs/__init__.py | 2 +- lib/galaxy/tools/__init__.py | 5 ++++- lib/galaxy/tools/parameters/dynamic_options.py | 6 +++--- tools/data_source/microbial_import_code.py | 9 +++++++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 2b0f6705252..e0135c75afc 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -464,7 +464,7 @@ class JobWrapper( object ): # custom post process setup inp_data = dict( [ ( da.name, da.dataset ) for da in job.input_datasets ] ) out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] ) - param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] ) # why not re-use self.param_dict here? + param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] ) # why not re-use self.param_dict here? ##dunno...probably should, this causes tools.parameters.basic.UnvalidatedValue to be used in following methods instead of validated and transformed values during i.e. running workflows param_dict = self.tool.params_from_strings( param_dict, self.app ) # Check for and move associated_files self.tool.collect_associated_files(out_data) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 50f273d0042..a79843d046a 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -948,7 +948,10 @@ class Tool: # Regular tool parameter value = input_values[ input.name ] if isinstance( value, UnvalidatedValue ): - value = input.from_html( value.value, None, context ) + if value.value is None: #if value.value is None, it could not have been submited via html form and therefore .from_html can't be guaranteed to work + value = None + else: + value = input.from_html( value.value, None, context ) # Then do any further validation on the value input.validate( value, None ) input_values[ input.name ] = value diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py index ddaed52842c..2e080ab42ae 100644 --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -102,7 +102,7 @@ class DataMetaFilter( Filter ): if self.multiple: return dataset_value in file_value.split( self.separator ) return file_value == dataset_value - assert self.ref_name in other_values or trans.workflow_building_mode, "Required dependency '%s' not found in incoming values" % self.ref_name + assert self.ref_name in other_values or ( trans is not None and trans.workflow_building_mode), "Required dependency '%s' not found in incoming values" % self.ref_name ref = other_values.get( self.ref_name, None ) if not isinstance( ref, self.dynamic_option.tool_param.tool.app.model.HistoryDatasetAssociation ): return [] #not a valid dataset @@ -146,9 +146,9 @@ class ParamValueFilter( Filter ): def get_dependency_name( self ): return self.ref_name def filter_options( self, options, trans, other_values ): - if trans.workflow_building_mode: return [] + if trans is not None and trans.workflow_building_mode: return [] + assert self.ref_name in other_values, "Required dependency '%s' not found in incoming values" % self.ref_name ref = str( other_values.get( self.ref_name, None ) ) - assert ref is not None, "Required dependency '%s' not found in incoming values" % self.ref_name rval = [] for fields in options: if ( self.keep and fields[self.column] == ref ) or ( not self.keep and fields[self.column] != ref ): diff --git a/tools/data_source/microbial_import_code.py b/tools/data_source/microbial_import_code.py index b6bc2f6bd85..7d2945c4985 100644 --- a/tools/data_source/microbial_import_code.py +++ b/tools/data_source/microbial_import_code.py @@ -80,7 +80,7 @@ def load_microbial_data( GALAXY_DATA_INDEX_DIR, sep='\t' ): return microbe_info #post processing, set build for data and add additional data to history -from galaxy import datatypes, config, jobs +from galaxy import datatypes, config, jobs, tools from shutil import copyfile def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr): @@ -95,7 +95,12 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr #if not (kingdom or group or org): if not (kingdom or org): print "Parameters are not available." - + #workflow passes galaxy.tools.parameters.basic.UnvalidatedValue instead of values + if isinstance( kingdom, tools.parameters.basic.UnvalidatedValue ): + kingdom = kingdom.value + if isinstance( org, tools.parameters.basic.UnvalidatedValue ): + org = org.value + GALAXY_DATA_INDEX_DIR = app.config.tool_data_path microbe_info = load_microbial_data( GALAXY_DATA_INDEX_DIR, sep='\t' ) new_stdout = ""