diff --git a/eggs.ini b/eggs.ini index 6b918a7f324..1950ce4aa05 100644 --- a/eggs.ini +++ b/eggs.ini @@ -39,7 +39,7 @@ PasteDeploy = 1.3.1 PasteScript = 1.3.6 Routes = 1.6.3 simplejson = 1.5 -SQLAlchemy = 0.3.11 +SQLAlchemy = 0.4.7p1 Tempita = 0.1 twill = 0.9 WebError = 0.8a @@ -84,7 +84,7 @@ PasteDeploy = http://cheeseshop.python.org/packages/source/P/PasteDeploy/PasteDe PasteScript = http://cheeseshop.python.org/packages/source/P/PasteScript/PasteScript-1.3.6.tar.gz Routes = http://pypi.python.org/packages/source/R/Routes/Routes-1.6.3.tar.gz simplejson = http://cheeseshop.python.org/packages/source/s/simplejson/simplejson-1.5.tar.gz -SQLAlchemy = http://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-0.3.11.tar.gz +SQLAlchemy = http://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-0.4.7p1.tar.gz Tempita = http://pypi.python.org/packages/source/T/Tempita/Tempita-0.1.tar.gz twill = http://darcs.idyll.org/~t/projects/twill-0.9.tar.gz WebError = http://pypi.python.org/packages/source/W/WebError/WebError-0.8a.tar.gz diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 1964945012e..eb26d6a9acd 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -21,7 +21,7 @@ class UniverseApplication( object ): if self.config.database_connection: db_url = self.config.database_connection else: - db_url = "sqlite://%s?isolation_level=IMMEDIATE" % self.config.database + db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database # Setup the database engine and ORM self.model = galaxy.model.mapping.init( self.config.file_path, db_url, diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 342ee4636ef..4ab41942f4e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -103,28 +103,9 @@ class JobToOutputDatasetAssociation( object ): self.name = name self.dataset = dataset -<<<<<<< local -class HistoryDatasetAssociation( object ): - def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None, - dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None, - parent_id=None, copied_from_history_dataset_association = None, validation_errors=None, visible=True, create_dataset = False ): - self.name = name or "Unnamed dataset" - self.id = id - self.hid = hid - self.info = info - self.blurb = blurb - self.peek = peek - self.extension = extension - self.designation = designation - self.metadata = metadata or dict() - self.dbkey = dbkey - self.deleted = deleted - self.visible = visible - # Relationships -======= class GroupDatasetAssociation( object ): def __init__( self, group, dataset, permitted_actions=[] ): - if isinstance( group, GroupDatasetAssociation ) or \ + if isinstance( group, GroupDatasetAssociation ) or \ isinstance( group, DefaultUserGroupAssociation ) or \ isinstance( group, DefaultHistoryGroupAssociation ): group = group.group @@ -182,16 +163,9 @@ class DefaultHistoryGroupAssociation( object ): isinstance( group, DefaultUserGroupAssociation ) or \ isinstance( group, DefaultHistoryGroupAssociation ): group = group.group ->>>>>>> other self.history = history -<<<<<<< local - if not dataset and create_dataset: - dataset = Dataset() - dataset.flush() - self.dataset = dataset - self.parent_id = parent_id - self.validation_errors = validation_errors - self.copied_from_history_dataset_association = copied_from_history_dataset_association + self.group = group + self.permitted_actions = permitted_actions @property def ext( self ): @@ -342,8 +316,6 @@ class DefaultHistoryGroupAssociation( object ): for child in self.children: child.mark_deleted() - - class History( object ): def __init__( self, id=None, name=None, user=None ): self.id = id @@ -415,10 +387,6 @@ class History( object ): # # Relationships # self.history = history # self.datasets = [] -======= - self.group = group - self.permitted_actions = permitted_actions ->>>>>>> other class Dataset( object ): states = Bunch( NEW = 'new', @@ -528,7 +496,7 @@ class DatasetInstance( object ): self.extension = extension self.dbkey = dbkey self.designation = designation - self._metadata = metadata or dict() + self.metadata = metadata or dict() self.deleted = deleted self.visible = visible # Relationships @@ -559,9 +527,9 @@ class DatasetInstance( object ): def datatype( self ): return datatypes_registry.get_datatype_by_extension( self.extension ) def get_metadata( self ): - if not self._metadata: - self._metadata = dict() - return MetadataCollection( self, self.datatype.metadata_spec ) + if not hasattr( self, '_metadata_collection' ): + self._metadata_collection = MetadataCollection( self, self.datatype.metadata_spec ) + return self._metadata_collection def set_metadata( self, bunch ): # Needs to accept a MetadataCollection, a bunch, or a dict self._metadata = dict( bunch.items() ) @@ -587,6 +555,8 @@ class DatasetInstance( object ): dbkey = property( get_dbkey, set_dbkey ) def change_datatype( self, new_ext ): self.clear_associated_files() + if hasattr( self, '_metadata_collection' ): + del self._metadata_collection datatypes_registry.change_datatype( self, new_ext ) def get_size( self ): """Returns the size of the data on disk""" diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index f537a93f463..72d6b79075c 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -16,16 +16,15 @@ class JSONType( TypeDecorator ): self.mutable = mutable super( JSONType, self).__init__() - def convert_result_value( self, value, dialect ): + def process_bind_param( self, value, dialect ): if value is None: return None - buf = self.impl.convert_result_value( value, dialect ) - return self.jsonifyer.loads( str(buf) ) - - def convert_bind_param( self, value, dialect ): + return self.jsonifyer.dumps( value ) + + def process_result_value( self, value, dialect ): if value is None: return None - return self.impl.convert_bind_param( self.jsonifyer.dumps(value), dialect ) + return self.jsonifyer.loads( str( value ) ) def copy_value( self, value ): if self.mutable: @@ -58,10 +57,10 @@ class MetadataType( JSONType ): self.mutable = mutable super( MetadataType, self).__init__() - def convert_result_value( self, value, dialect ): + def process_result_value( self, value, dialect ): if value is None: return None - buf = self.impl.convert_result_value( value, dialect ) + buf = value ret = None try: ret = self.pickler.loads( str(buf) ) @@ -75,7 +74,7 @@ class MetadataType( JSONType ): class TrimmedString( TypeDecorator ): impl = String - def convert_bind_param( self, value, dialect ): + def process_bind_param( self, value, dialect ): """Automatically truncate string values""" if self.impl.length and value is not None: value = value[0:self.impl.length] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 5beab407c4c..f0262731996 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -5,24 +5,21 @@ are encapsulated here. import logging log = logging.getLogger( __name__ ) -import pkg_resources -pkg_resources.require( "sqlalchemy>=0.3" ) - import sys import datetime -from sqlalchemy.ext.sessioncontext import SessionContext -from sqlalchemy.ext.assignmapper import assign_mapper -from sqlalchemy.ext.orderinglist import ordering_list - -from sqlalchemy import * from galaxy.model import * +from galaxy.model.orm import * +from galaxy.model.orm.ext.assignmapper import * from galaxy.model.custom_types import * from galaxy.util.bunch import Bunch from galaxy.security import GalaxyRBACAgent -metadata = DynamicMetaData( threadlocal=False ) -context = SessionContext( create_session ) +metadata = MetaData() +context = Session = scoped_session( sessionmaker( autoflush=False, transactional=False ) ) + +# For backward compatibility with "context.current" +context.current = Session dialect_to_egg = { "sqlite" : "pysqlite>=2", @@ -224,15 +221,15 @@ Job.table = Table( "job", metadata, Column( "update_time", DateTime, default=now, onupdate=now ), Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ), Column( "tool_id", String( 255 ) ), - Column( "tool_version", String, default="1.0.0" ), + Column( "tool_version", TEXT, default="1.0.0" ), Column( "state", String( 64 ) ), Column( "info", TrimmedString( 255 ) ), - Column( "command_line", String() ), + Column( "command_line", TEXT ), Column( "param_filename", String( 1024 ) ), Column( "runner_name", String( 255 ) ), - Column( "stdout", String() ), - Column( "stderr", String() ), - Column( "traceback", String() ), + Column( "stdout", TEXT ), + Column( "stderr", TEXT ), + Column( "traceback", TEXT ), Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True, nullable=True ), Column( "job_runner_name", String( 255 ) ), Column( "job_runner_external_id", String( 255 ) ) ) @@ -292,7 +289,7 @@ StoredWorkflow.table = Table( "stored_workflow", metadata, Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), Column( "latest_workflow_id", Integer, ForeignKey( "workflow.id", use_alter=True, name='stored_workflow_latest_workflow_id_fk' ), index=True ), - Column( "name", String ), + Column( "name", TEXT ), Column( "deleted", Boolean, default=False ), ) @@ -301,7 +298,7 @@ Workflow.table = Table( "workflow", metadata, Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "stored_workflow_id", Integer, ForeignKey( "stored_workflow.id" ), index=True, nullable=False ), - Column( "name", String ), + Column( "name", TEXT ), Column( "has_cycles", Boolean ), Column( "has_errors", Boolean ) ) @@ -312,8 +309,8 @@ WorkflowStep.table = Table( "workflow_step", metadata, Column( "update_time", DateTime, default=now, onupdate=now ), Column( "workflow_id", Integer, ForeignKey( "workflow.id" ), index=True, nullable=False ), Column( "type", String(64) ), - Column( "tool_id", String ), - Column( "tool_version", String ), # Reserved for future + Column( "tool_id", TEXT ), + Column( "tool_version", TEXT ), # Reserved for future Column( "tool_inputs", JSONType ), Column( "tool_errors", JSONType ), Column( "position", JSONType ), @@ -326,8 +323,8 @@ WorkflowStepConnection.table = Table( "workflow_step_connection", metadata, Column( "id", Integer, primary_key=True ), Column( "output_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True ), Column( "input_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True ), - Column( "output_name", String ), - Column( "input_name", String) + Column( "output_name", TEXT ), + Column( "input_name", TEXT) ) StoredWorkflowUserShareAssociation.table = Table( "stored_workflow_user_share_connection", metadata, @@ -352,9 +349,7 @@ assign_mapper( context, HistoryDatasetAssociation, HistoryDatasetAssociation.tab dataset=relation( Dataset, primaryjoin=( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ), lazy=False ), - history=relation( - History, - primaryjoin=( History.table.c.id == HistoryDatasetAssociation.table.c.history_id ) ), + # .history defined in History mapper copied_to_history_dataset_associations=relation( HistoryDatasetAssociation, primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id == HistoryDatasetAssociation.table.c.id ), @@ -415,11 +410,11 @@ assign_mapper( context, Group, Group.table, assign_mapper( context, UserGroupAssociation, UserGroupAssociation.table, properties=dict( user=relation( User, backref = "groups" ), - group=relation( Group, backref = "users" ) ) ) + group=relation( Group, backref = "members" ) ) ) assign_mapper( context, GroupDatasetAssociation, GroupDatasetAssociation.table, properties=dict( dataset=relation( Dataset, backref = "groups" ), - group=relation( Group, backref = "datasets" ) ) ) + group=relation( Group, backref = "group_datasets" ) ) ) assign_mapper( context, DefaultUserGroupAssociation, DefaultUserGroupAssociation.table, properties=dict( user=relation( User, backref = "default_groups" ), @@ -555,11 +550,12 @@ def db_next_hid( self ): Override __next_hid to generate from the database in a concurrency safe way. """ - conn = self.table.engine.contextual_connect() + conn = object_session( self ).connection() + table = self.table trans = conn.begin() try: - next_hid = select( [self.c.hid_counter], self.c.id == self.id, for_update=True ).scalar() - self.table.update( self.c.id == self.id ).execute( hid_counter = ( next_hid + 1 ) ) + next_hid = select( [table.c.hid_counter], table.c.id == self.id, for_update=True ).scalar() + table.update( table.c.id == self.id ).execute( hid_counter = ( next_hid + 1 ) ) trans.commit() return next_hid except: @@ -588,17 +584,21 @@ def init( file_path, url, engine_options={}, create_tables=False ): # Create the database engine engine = create_engine( url, **engine_options ) # Connect the metadata to the database. - metadata.connect( engine ) - ## metadata.engine.echo = True + metadata.bind = engine + # Clear any existing contextual sessions and reconfigure + Session.remove() + Session.configure( bind=engine ) # Create tables if needed if create_tables: metadata.create_all() # metadata.engine.commit() # Pack everything into a bunch result = Bunch( **globals() ) - result.engine = metadata.engine - result.flush = lambda *args, **kwargs: context.current.flush( *args, **kwargs ) - result.context = context + result.engine = engine + result.flush = lambda *args, **kwargs: Session.flush( *args, **kwargs ) + result.session = Session + # For backward compatibility with "model.context.current" + result.context = Session result.create_tables = create_tables #load local galaxy security policy result.security_agent = GalaxyRBACAgent( result ) diff --git a/lib/galaxy/model/orm/__init__.py b/lib/galaxy/model/orm/__init__.py new file mode 100644 index 00000000000..433e1e26f45 --- /dev/null +++ b/lib/galaxy/model/orm/__init__.py @@ -0,0 +1,7 @@ +import pkg_resources +pkg_resources.require( "SQLAlchemy >= 0.4" ) + +from sqlalchemy import * +from sqlalchemy.orm import * + +from sqlalchemy.ext.orderinglist import ordering_list diff --git a/lib/galaxy/model/orm/ext/__init__.py b/lib/galaxy/model/orm/ext/__init__.py new file mode 100644 index 00000000000..230b8aec908 --- /dev/null +++ b/lib/galaxy/model/orm/ext/__init__.py @@ -0,0 +1,3 @@ +""" +Galaxy specific SQLAlchemy extensions. +""" \ No newline at end of file diff --git a/lib/galaxy/model/orm/ext/assignmapper.py b/lib/galaxy/model/orm/ext/assignmapper.py new file mode 100644 index 00000000000..d2b7e2ca7b4 --- /dev/null +++ b/lib/galaxy/model/orm/ext/assignmapper.py @@ -0,0 +1,62 @@ +""" +This is similar to the assignmapper extensions in SQLAclhemy 0.3 and 0.4 but +with some compatibility fixes. It assumes that the session is a ScopedSession, +and thus has the "mapper" method to attach contextual mappers to a class. It +adds additional query and session methods to the class to support the +SQLAlchemy 0.3 style of access. The following methods which would normally be +accessed through "Object.query().method()" are available directly through the +object: + + 'get', 'filter', 'filter_by', 'select', 'select_by', + 'selectfirst', 'selectfirst_by', 'selectone', 'selectone_by', + 'get_by', 'join_to', 'join_via', 'count', 'count_by', + 'options', 'instances' + +Additionally, the following Session methods, which normally accept an instance +or list of instances, are available directly through the objects, e.g. +"Session.flush( [instance] )" can be performed as "instance.flush()": + + 'refresh', 'expire', 'delete', 'expunge', 'update' +""" + +__all__ = [ 'assign_mapper' ] + +from sqlalchemy import util, exceptions +import types +from sqlalchemy.orm import mapper, Query + +def _monkeypatch_query_method( name, session, class_ ): + def do(self, *args, **kwargs): + ## util.warn_deprecated('Query methods on the class are deprecated; use %s.query.%s instead' % (class_.__name__, name)) + return getattr( class_.query, name)(*args, **kwargs) + try: + do.__name__ = name + except: + pass + if not hasattr(class_, name): + setattr(class_, name, classmethod(do)) + +def _monkeypatch_session_method(name, session, class_, make_list=False): + def do(self, *args, **kwargs): + if make_list: + self = [ self ] + return getattr(session, name)( self, *args, **kwargs ) + try: + do.__name__ = name + except: + pass + if not hasattr(class_, name): + setattr(class_, name, do) + +def assign_mapper( session, class_, *args, **kwargs ): + m = class_.mapper = session.mapper( class_, *args, **kwargs ) + for name in ('get', 'filter', 'filter_by', 'select', 'select_by', + 'selectfirst', 'selectfirst_by', 'selectone', 'selectone_by', + 'get_by', 'join_to', 'join_via', 'count', 'count_by', + 'options', 'instances'): + _monkeypatch_query_method(name, session, class_) + for name in ('refresh', 'expire', 'delete', 'expunge', 'update'): + _monkeypatch_session_method(name, session, class_) + for name in ( 'flush', ): + _monkeypatch_session_method( name, session, class_, make_list=True ) + return m diff --git a/lib/galaxy/web/controllers/admin.py b/lib/galaxy/web/controllers/admin.py index 958e6234663..c7035dd98eb 100644 --- a/lib/galaxy/web/controllers/admin.py +++ b/lib/galaxy/web/controllers/admin.py @@ -6,7 +6,7 @@ from galaxy.security import RBACAgent import galaxy.model from xml.sax.saxutils import escape, unescape import pkg_resources -pkg_resources.require( "sqlalchemy>=0.3" ) +pkg_resources.require( "SQLAlchemy >= 0.4" ) import sqlalchemy as sa import logging @@ -251,7 +251,7 @@ class Admin( BaseController ): # simpler approach of deleting all existing members and creating # new records for user_ids in the received members param. # First remove existing members that are not in the received members param - for user_group_assoc in group.users: + for user_group_assoc in group.members: if user_group_assoc.user_id not in members: user = galaxy.model.User.get( user_group_assoc.user_id ) # Delete DefaultUserGroupAssociations @@ -272,7 +272,7 @@ class Admin( BaseController ): # Then add all new members to the group for user_id in members: user = galaxy.model.User.get( user_id ) - if user not in group.users: + if user not in group.members: user_group_association = galaxy.model.UserGroupAssociation( user, group ) user_group_association.flush() msg = "Group membership has been updated with a total of %s members" % len( members ) @@ -405,7 +405,7 @@ class Admin( BaseController ): group_id = params.group_id group = galaxy.model.Group.get( group_id ) # Remove members and all associations - for user_group_assoc in group.users: + for user_group_assoc in group.members: user = galaxy.model.User.get( user_group_assoc.user_id ) # Delete DefaultUserGroupAssociations for default_user_group_association in user.default_groups: diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py index 2be8810b68e..20bda3c2d8e 100644 --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -24,7 +24,7 @@ import mako.lookup pkg_resources.require( "simplejson" ) import simplejson -pkg_resources.require( "sqlalchemy>=0.3" ) +pkg_resources.require( "SQLAlchemy >= 0.4" ) from sqlalchemy import desc import logging diff --git a/lib/galaxy/webapps/reports/controllers/jobs.py b/lib/galaxy/webapps/reports/controllers/jobs.py index bf7935d9647..a502ca48532 100644 --- a/lib/galaxy/webapps/reports/controllers/jobs.py +++ b/lib/galaxy/webapps/reports/controllers/jobs.py @@ -5,7 +5,7 @@ import calendar from galaxy.webapps.reports.base.controller import * import galaxy.model import pkg_resources -pkg_resources.require( "sqlalchemy>=0.3" ) +pkg_resources.require( "SQLAlchemy >= 0.4" ) import sqlalchemy as sa import logging log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/webapps/reports/controllers/system.py b/lib/galaxy/webapps/reports/controllers/system.py index dc6d3b36b90..477b2084d03 100644 --- a/lib/galaxy/webapps/reports/controllers/system.py +++ b/lib/galaxy/webapps/reports/controllers/system.py @@ -2,8 +2,9 @@ import operator, os from datetime import datetime, timedelta from galaxy.webapps.reports.base.controller import * import pkg_resources -pkg_resources.require( "sqlalchemy>=0.3" ) -from sqlalchemy import eagerload, desc +pkg_resources.require( "SQLAlchemy >= 0.4" ) +from sqlalchemy.orm import eagerload +from sqlalchemy import desc import logging log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/webapps/reports/controllers/users.py b/lib/galaxy/webapps/reports/controllers/users.py index b99546f14dd..185a18107c3 100644 --- a/lib/galaxy/webapps/reports/controllers/users.py +++ b/lib/galaxy/webapps/reports/controllers/users.py @@ -3,7 +3,7 @@ import calendar from galaxy.webapps.reports.base.controller import * import galaxy.model import pkg_resources -pkg_resources.require( "sqlalchemy>=0.3" ) +pkg_resources.require( "SQLAlchemy >= 0.4" ) import sqlalchemy as sa import logging log = logging.getLogger( __name__ ) diff --git a/scripts/cleanup_datasets/cleanup_datasets.py b/scripts/cleanup_datasets/cleanup_datasets.py index 0d653981d00..97ca3f8e8f8 100644 --- a/scripts/cleanup_datasets/cleanup_datasets.py +++ b/scripts/cleanup_datasets/cleanup_datasets.py @@ -13,8 +13,8 @@ from galaxy import eggs import galaxy.model.mapping import pkg_resources -pkg_resources.require( "sqlalchemy>=0.3" ) -from sqlalchemy import eagerload +pkg_resources.require( "SQLAlchemy >= 0.4" ) +from sqlalchemy.orm import eagerload assert sys.version_info[:2] >= ( 2, 4 ) diff --git a/test/functional/test_history_functions.py b/test/functional/test_history_functions.py index 24663bf4e77..8b0b12819fa 100644 --- a/test/functional/test_history_functions.py +++ b/test/functional/test_history_functions.py @@ -48,7 +48,10 @@ class TestHistory( TwillTestCase ): """Testing sharing a history with another user""" self.upload_file('1.bed', dbkey='hg18') id, name, email = self.share_history() - self.check_page_for_string( 'History (%s) has been shared with: %s' %(name, email) ) + try: + self.check_page_for_string( 'History (%s) has been shared with: %s' %(name, email) ) + except TwillAssertionError: + self.check_page_for_string( "The history or histories you've chosen to share contain datasets that the user you're sharing with does not have permission to access." ) self.logout() self.login( email='test2@bx.psu.edu' ) self.view_stored_histories() diff --git a/test/functional/test_security_and_libraries.py b/test/functional/test_security_and_libraries.py index 748684e696f..031eef67b61 100644 --- a/test/functional/test_security_and_libraries.py +++ b/test/functional/test_security_and_libraries.py @@ -85,7 +85,7 @@ class TestHistory( TwillTestCase ): def test_20_create_library( self ): """Testing creating new library""" - self.create_library( name='New Test Library', description='New test Library Description' ) + self.create_library( name='New Test Library', description='New Test Library Description' ) self.visit_page( 'admin/libraries' ) self.check_page_for_string( "New Test Library" ) diff --git a/tools/stats/grouping.py b/tools/stats/grouping.py index d21a6da6f4e..a26fb033f0b 100644 --- a/tools/stats/grouping.py +++ b/tools/stats/grouping.py @@ -90,7 +90,7 @@ def main(): for ii, line in enumerate( file( tmpfile.name )): if line and not line.startswith( '#' ): - line = line.strip() + line = line.rstrip( '\r\n' ) try: fields = line.split("\t") item = fields[group_col]