imported patch alchemy05_fixes_02

This commit is contained in:
Nate Coraor
2009-11-03 12:52:01 -05:00
parent 7a21067ad8
commit 8d96c8d979
21 changed files with 69 additions and 58 deletions
+3 -3
View File
@@ -199,7 +199,7 @@ class JobQueue( object ):
try:
# Clear the session for each job so we get fresh states for
# job and all datasets
self.sa_session.clear()
self.sa_session.expunge_all()
# Get the real job entity corresponding to the wrapper (if we
# are tracking in the database this is probably cached in
# the session from the origianl query above)
@@ -346,7 +346,7 @@ class JobWrapper( object ):
Prepare the job to run by creating the working directory and the
config files.
"""
self.sa_session.clear() #this prevents the metadata reverting that has been seen in conjunction with the PBS job runner
self.sa_session.expunge_all() #this prevents the metadata reverting that has been seen in conjunction with the PBS job runner
if not os.path.exists( self.working_directory ):
os.mkdir( self.working_directory )
# Restore parameters from the database
@@ -477,7 +477,7 @@ class JobWrapper( object ):
the contents of the output files.
"""
# default post job setup
self.sa_session.clear()
self.sa_session.expunge_all()
job = self.sa_session.query( model.Job ).get( self.job_id )
# if the job was deleted, don't finish it
if job.state == job.states.DELETED:
+2 -2
View File
@@ -817,7 +817,7 @@ assign_mapper( context, UserRoleAssociation, UserRoleAssociation.table,
user=relation( User, backref="roles" ),
non_private_roles=relation( User,
backref="non_private_roles",
primaryjoin=( ( User.table.c.id == UserRoleAssociation.table.c.user_id ) & ( UserRoleAssociation.table.c.role_id == Role.table.c.id ) & not_( Role.table.c.name == User.table.c.email & Role.table.c.type == 'private' ) ) ),
primaryjoin=( ( User.table.c.id == UserRoleAssociation.table.c.user_id ) & ( UserRoleAssociation.table.c.role_id == Role.table.c.id ) & not_( Role.table.c.name == User.table.c.email ) ) ),
role=relation( Role )
)
)
@@ -1134,7 +1134,7 @@ def init( file_path, url, engine_options={}, create_tables=False ):
# Pack everything into a bunch
result = Bunch( **globals() )
result.engine = engine
result.flush = lambda *args, **kwargs: Session.flush( *args, **kwargs )
# model.flush() has been removed.
result.session = Session
# For backward compatibility with "model.context.current"
result.context = Session
+8 -7
View File
@@ -19,16 +19,17 @@ class MappingTests( unittest.TestCase ):
d1 = model.HistoryDatasetAssociation( extension="interval", metadata=dict(chromCol=1,startCol=2,endCol=3 ), history=h2, create_dataset=True )
#h2.queries.append( q1 )
#h2.queries.append( model.Query( "h2->q2" ) )
model.context.current.flush()
model.context.current.clear()
model.session.add_all( ( u, h1, h2, d1 ) )
model.session.flush()
model.session.expunge_all()
# Check
users = model.context.current.query( model.User ).all()
users = model.session.query( model.User ).all()
assert len( users ) == 1
assert users[0].email == "james@foo.bar.baz"
assert users[0].password == "password"
assert len( users[0].histories ) == 1
assert users[0].histories[0].name == "History 1"
hists = model.context.current.query( model.History ).all()
hists = model.session.query( model.History ).all()
assert hists[0].name == "History 1"
assert hists[1].name == ( "H" * 255 )
assert hists[0].user == users[0]
@@ -38,9 +39,9 @@ class MappingTests( unittest.TestCase ):
assert hists[1].datasets[0].file_name == os.path.join( "/tmp", *directory_hash_id( id ) ) + ( "/dataset_%d.dat" % id )
# Do an update and check
hists[1].name = "History 2b"
model.context.current.flush()
model.context.current.clear()
hists = model.context.current.query( model.History ).all()
model.session.flush()
model.session.expunge_all()
hists = model.session.query( model.History ).all()
assert hists[0].name == "History 1"
assert hists[1].name == "History 2b"
# gvk TODO need to ad test for GalaxySessions, but not yet sure what they should look like.
@@ -21,7 +21,7 @@ handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, transactional=False ) )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
def display_migration_details():
print "========================================"
@@ -59,4 +59,4 @@ def upgrade():
except Exception, e:
log.debug( "Adding foreign key constraint 'user_form_values_id_fk' to table 'galaxy_user' failed: %s" % ( str( e ) ) )
def downgrade():
pass
pass
+4 -3
View File
@@ -235,7 +235,7 @@ class DefaultToolAction( object ):
# Store output
out_data[ name ] = data
# Store all changes to database
trans.app.model.flush()
trans.sa_session.flush()
# Add all the top-level (non-child) datasets to the history
for name in out_data.keys():
if name not in child_dataset_names and name not in incoming: #don't add children; or already existing datasets, i.e. async created
@@ -248,7 +248,7 @@ class DefaultToolAction( object ):
child_dataset = out_data[ child_name ]
parent_dataset.children.append( child_dataset )
# Store data after custom code runs
trans.app.model.flush()
trans.sa_session.flush()
# Create the job object
job = trans.app.model.Job()
job.session_id = trans.get_galaxy_session().id
@@ -274,7 +274,8 @@ class DefaultToolAction( object ):
job.add_input_dataset( name, None )
for name, dataset in out_data.iteritems():
job.add_output_dataset( name, dataset )
trans.app.model.flush()
trans.sa_session.add( job )
trans.sa_session.flush()
# Some tools are not really executable, but jobs are still created for them ( for record keeping ).
# Examples include tools that redirect to other applications ( epigraph ). These special tools must
# include something that can be retrieved from the params ( e.g., REDIRECT_URL ) to keep the job
+3 -2
View File
@@ -26,7 +26,8 @@ class SetMetadataToolAction( ToolAction ):
job.tool_version = tool.version
except:
job.tool_version = "1.0.0"
job.flush() #ensure job.id is available
trans.sa_session.add( job )
trans.sa_session.flush() #ensure job.id is available
#add parameters to job_parameter table
# Store original dataset state, so we can restore it. A separate table might be better (no chance of 'losing' the original state)?
@@ -49,7 +50,7 @@ class SetMetadataToolAction( ToolAction ):
#Need a special state here to show that metadata is being set and also allow the job to run
# i.e. if state was set to 'running' the set metadata job would never run, as it would wait for input (the dataset to set metadata on) to be in a ready state
dataset.state = dataset.states.SETTING_METADATA
trans.app.model.flush()
trans.sa_session.flush()
# Queue the job for execution
trans.app.job_queue.put( job.id, tool )
+3 -1
View File
@@ -121,6 +121,7 @@ def new_history_upload( trans, uploaded_dataset, state=None ):
trans.history.add_dataset( hda, genome_build = uploaded_dataset.dbkey )
permissions = trans.app.security_agent.history_get_default_permissions( trans.history )
trans.app.security_agent.set_all_dataset_permissions( hda.dataset, permissions )
trans.sa_session.flush()
return hda
def new_library_upload( trans, uploaded_dataset, library_bunch, state=None ):
@@ -291,7 +292,8 @@ def create_job( trans, params, tool, json_file_path, data_list, folder=None ):
for i, dataset in enumerate( data_list ):
job.add_output_dataset( 'output%i' % i, dataset )
job.state = job.states.NEW
trans.app.model.flush()
trans.sa_session.add( job )
trans.sa_session.flush()
# Queue the job for execution
trans.app.job_queue.put( job.id, tool )
+3 -3
View File
@@ -77,7 +77,7 @@ class ASync( BaseController ):
data.state = data.blurb = jobs.JOB_ERROR
data.info = "Error -> %s" % STATUS
trans.model.flush()
trans.sa_session.flush()
return "Data %s with status %s received. OK" % (data_id, STATUS)
@@ -112,7 +112,7 @@ class ASync( BaseController ):
data.flush()
open( data.file_name, 'wb' ).close() #create the file
trans.history.add_dataset( data, genome_build=GALAXY_BUILD )
trans.model.flush()
trans.sa_session.flush()
trans.log_event( "Added dataset %d to history %d" %(data.id, trans.history.id ), tool_id=tool_id )
try:
@@ -132,6 +132,6 @@ class ASync( BaseController ):
data.info = str(e)
data.state = data.blurb = data.states.ERROR
trans.model.flush()
trans.sa_session.flush()
return trans.fill_template('tool_executed.tmpl', out_data={}, tool=tool, config=self.app.config )
+2 -2
View File
@@ -335,7 +335,7 @@ class DatasetInterface( BaseController ):
assert topmost_parent in history.datasets, "Data does not belong to current history"
# Mark undeleted
data.mark_undeleted()
self.app.model.flush()
trans.sa_session.flush()
trans.log_event( "Dataset id %s has been undeleted" % str(id) )
return True
return False
@@ -407,7 +407,7 @@ class DatasetInterface( BaseController ):
hist.add_dataset( data.copy( copy_children = True ) )
if history in target_histories:
refresh_frames = ['history']
trans.app.model.flush()
trans.sa_session.flush()
done_msg = "%i datasets copied to %i histories." % ( len( source_dataset_ids ) - invalid_datasets, len( target_histories ) )
trans.sa_session.refresh( history )
elif create_new_history:
+3 -3
View File
@@ -421,7 +421,7 @@ class Library( BaseController ):
if trans.app.security_agent.can_modify_library_item( user, roles, ldda ):
if ldda.datatype.allow_datatype_change and trans.app.datatypes_registry.get_datatype_by_extension( params.datatype ).allow_datatype_change:
trans.app.datatypes_registry.change_datatype( ldda, params.datatype )
trans.app.model.flush()
trans.sa_session.flush()
msg = "Data type changed for library dataset '%s'" % ldda.name
messagetype = 'done'
else:
@@ -463,7 +463,7 @@ class Library( BaseController ):
setattr( ldda.metadata, name, spec.unwrap( params.get ( name, None ) ) )
ldda.metadata.dbkey = dbkey
ldda.datatype.after_edit( ldda )
trans.app.model.flush()
trans.sa_session.flush()
msg = 'Attributes updated for library dataset %s' % ldda.name
messagetype = 'done'
else:
@@ -486,7 +486,7 @@ class Library( BaseController ):
setattr( ldda.metadata, name, spec.unwrap( spec.get( 'default' ) ) )
ldda.datatype.set_meta( ldda )
ldda.datatype.after_edit( ldda )
trans.app.model.flush()
trans.sa_session.flush()
msg = 'Attributes updated for library dataset %s' % ldda.name
messagetype = 'done'
else:
+8 -4
View File
@@ -432,7 +432,7 @@ class LibraryAdmin( BaseController ):
# The user clicked the Save button on the 'Change data type' form
if ldda.datatype.allow_datatype_change and trans.app.datatypes_registry.get_datatype_by_extension( params.datatype ).allow_datatype_change:
trans.app.datatypes_registry.change_datatype( ldda, params.datatype )
trans.app.model.flush()
trans.sa_session.flush()
msg = "Data type changed for library dataset '%s'" % ldda.name
return trans.fill_template( "/admin/library/ldda_edit_info.mako",
ldda=ldda,
@@ -469,7 +469,7 @@ class LibraryAdmin( BaseController ):
setattr( ldda.metadata, name, spec.unwrap( params.get ( name, None ) ) )
ldda.metadata.dbkey = dbkey
ldda.datatype.after_edit( ldda )
trans.app.model.flush()
trans.sa_session.flush()
msg = 'Attributes updated for library dataset %s' % ldda.name
messagetype = 'done'
return trans.fill_template( "/admin/library/ldda_edit_info.mako",
@@ -488,7 +488,7 @@ class LibraryAdmin( BaseController ):
setattr( ldda.metadata, name, spec.unwrap( spec.get( 'default' ) ) )
ldda.datatype.set_meta( ldda )
ldda.datatype.after_edit( ldda )
trans.app.model.flush()
trans.sa_session.flush()
msg = 'Attributes updated for library dataset %s' % ldda.name
return trans.fill_template( "/admin/library/ldda_edit_info.mako",
ldda=ldda,
@@ -674,6 +674,10 @@ class LibraryAdmin( BaseController ):
replace_id = params.get( 'replace_id', None )
if replace_id not in [ None, 'None' ]:
replace_dataset = trans.sa_session.query( trans.app.model.LibraryDataset ).get( int( replace_id ) )
# The name is separately - by the time the new ldda is created,
# replace_dataset.name will point to the new ldda, not the one it's
# replacing.
replace_dataset_name = replace_dataset.name
if not last_used_build:
last_used_build = replace_dataset.library_dataset_dataset_association.dbkey
# Don't allow multiple datasets to be uploaded when replacing a dataset with a new version
@@ -701,7 +705,7 @@ class LibraryAdmin( BaseController ):
if created_outputs:
total_added = len( created_outputs.values() )
if replace_dataset:
msg = "Added %d dataset versions to the library dataset '%s' in the folder '%s'." % ( total_added, replace_dataset.name, folder.name )
msg = "Added %d dataset versions to the library dataset '%s' in the folder '%s'." % ( total_added, replace_dataset_name, folder.name )
else:
if not folder.parent:
# Libraries have the same name as their root_folder
+1 -1
View File
@@ -294,7 +294,7 @@ class Requests( BaseController ):
# save all the new/unsaved samples entered by the user
if edit_mode == 'False':
for index in range(len(current_samples)-len(request.samples)):
sample_index = index + len(request.samples)
sample_index = len(request.samples)
sample_name = util.restore_text( params.get( 'sample_%i_name' % sample_index, '' ) )
sample_values = []
for field_index in range(len(request.type.sample_form.fields)):
+5 -5
View File
@@ -277,7 +277,7 @@ class RootController( BaseController ):
if not __ok_to_edit_metadata( data.id ):
return trans.show_error_message( "This dataset is currently being used as input or output. You cannot change datatype until the jobs have completed or you have canceled them." )
trans.app.datatypes_registry.change_datatype( data, params.datatype )
trans.app.model.flush()
trans.sa_session.flush()
else:
return trans.show_error_message( "You are unable to change datatypes in this manner. Changing %s to %s is not allowed." % ( data.extension, params.datatype ) )
elif params.save:
@@ -303,7 +303,7 @@ class RootController( BaseController ):
data.datatype.after_edit( data )
else:
msg = ' (Metadata could not be changed because this dataset is currently being used as input or output. You must cancel or wait for these jobs to complete before changing metadata.)'
trans.app.model.flush()
trans.sa_session.flush()
return trans.show_ok_message( "Attributes updated%s" % msg, refresh_frames=['history'] )
elif params.detect:
# The user clicked the Auto-detect button on the 'Edit Attributes' form
@@ -322,7 +322,7 @@ class RootController( BaseController ):
msg = 'Attributes updated'
data.set_meta()
data.datatype.after_edit( data )
trans.app.model.flush()
trans.sa_session.flush()
return trans.show_ok_message( msg, refresh_frames=['history'] )
elif params.convert_data:
target_type = kwd.get("target_type", None)
@@ -383,7 +383,7 @@ class RootController( BaseController ):
if job.check_if_output_datasets_deleted():
job.mark_deleted()
self.app.job_manager.job_stop_queue.put( job.id )
self.app.model.flush()
trans.sa_session.flush()
@web.expose
def delete( self, trans, id = None, show_deleted_on_refresh = False, **kwd):
@@ -432,7 +432,7 @@ class RootController( BaseController ):
for dataset in history.datasets:
dataset.deleted = True
dataset.clear_associated_files()
self.app.model.flush()
trans.sa_session.flush()
trans.log_event( "History id %s cleared" % (str(history.id)) )
trans.response.send_redirect( url_for("/index" ) )
@web.expose
@@ -198,8 +198,6 @@ class ToolRunner( BaseController ):
# pasted data
datasets.append( create_dataset( 'Pasted Entry' ) )
break
if datasets:
trans.model.flush()
return [ d.id for d in datasets ]
@web.expose
+14 -9
View File
@@ -139,7 +139,7 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
self.__galaxy_session = NOT_SET
base.DefaultWebTransaction.__init__( self, environ )
self.setup_i18n()
self.sa_session.clear()
self.sa_session.expunge_all()
self.debug = asbool( self.app.config.get( 'debug', False ) )
# Flag indicating whether we are in workflow building mode (means
# that the current history should not be used for parameter values
@@ -302,12 +302,12 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
self.galaxy_session = galaxy_session
# Do we need to flush the session?
if galaxy_session_requires_flush:
objects_to_flush = [ galaxy_session ]
sa_session.add( galaxy_session )
# FIXME: If prev_session is a proper relation this would not
# be needed.
if prev_galaxy_session:
objects_to_flush.append( prev_galaxy_session )
sa_session.flush( objects_to_flush )
sa_session.add( prev_galaxy_session )
sa_session.flush()
# If the old session was invalid, get a new history with our new session
if invalidate_existing_session:
self.new_history()
@@ -427,7 +427,8 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
if not last_accessed:
# Only set default history permissions if current history is not from a previous session
self.app.security_agent.history_set_default_permissions( history, dataset=True, bypass_manage_permission=True )
self.sa_session.flush( [ prev_galaxy_session, self.galaxy_session, history ] )
self.sa_session.add_all( ( prev_galaxy_session, self.galaxy_session, history ) )
self.sa_session.flush()
# This method is not called from the Galaxy reports, so the cookie will always be galaxysession
self.__update_session_cookie( name='galaxysession' )
def handle_user_logout( self ):
@@ -439,7 +440,8 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
prev_galaxy_session = self.galaxy_session
prev_galaxy_session.is_valid = False
self.galaxy_session = self.__create_new_session( prev_galaxy_session )
self.sa_session.flush( [ prev_galaxy_session, self.galaxy_session ] )
self.sa_session.add_all( ( prev_galaxy_session, self.galaxy_session ) )
self.sa_session.flush()
# This method is not called from the Galaxy reports, so the cookie will always be galaxysession
self.__update_session_cookie( name='galaxysession' )
@@ -466,7 +468,8 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
def set_history( self, history ):
if history and not history.deleted:
self.galaxy_session.current_history = history
self.sa_session.flush( [ self.galaxy_session ] )
self.sa_session.add( self.galaxy_session )
self.sa_session.flush()
history = property( get_history, set_history )
def new_history( self, name=None ):
"""
@@ -489,7 +492,8 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
# Set the user's default history permissions
self.app.security_agent.history_set_default_permissions( history )
# Save
self.sa_session.flush( [ self.galaxy_session, history ] )
self.sa_session.add_all( ( self.galaxy_session, history ) )
self.sa_session.flush()
return history
def get_user( self ):
@@ -498,7 +502,8 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
def set_user( self, user ):
"""Set the current user."""
self.galaxy_session.user = user
self.sa_session.flush( [ self.galaxy_session ] )
self.sa_session.add( self.galaxy_session )
self.sa_session.flush()
user = property( get_user, set_user )
def get_user_and_roles( self ):
@@ -29,6 +29,7 @@ def get_latest_form(form_name):
.filter( galaxy.model.FormDefinitionCurrent.table.c.deleted==False ) \
.order_by( galaxy.model.FormDefinitionCurrent.table.c.create_time.desc() )
for fdc in fdc_list:
sa_session.refresh( fdc.latest_form )
if form_name == fdc.latest_form.name:
return fdc.latest_form
return None
@@ -156,7 +156,7 @@ class TestSecurityAndLibraries( TwillTestCase ):
raise AssertionError( '%s not in history id %d default_permissions after they were changed' % ( value.action, latest_history.id ) )
# Add a dataset to the history
self.upload_file( '1.bed' )
latest_dataset = galaxy.model.Dataset.query().order_by( desc( galaxy.model.Dataset.table.c.create_time ) ).first()
latest_dataset = sa_session.query( galaxy.model.Dataset ).order_by( desc( galaxy.model.Dataset.table.c.create_time ) ).first()
# Make sure DatasetPermissionss are correct
if len( latest_dataset.actions ) != len( latest_history.default_permissions ):
raise AssertionError( '%d DatasetPermissionss were created for dataset id %d when it was created ( should have been %d )' % \
+2 -1
View File
@@ -14,6 +14,7 @@ def get_latest_form(form_name):
.filter( galaxy.model.FormDefinitionCurrent.table.c.deleted==False ) \
.order_by( galaxy.model.FormDefinitionCurrent.table.c.create_time.desc() )
for fdc in fdc_list:
sa_session.refresh( fdc.latest_form )
if form_name == fdc.latest_form.name:
return fdc.latest_form
return None
@@ -146,4 +147,4 @@ class TestUserInfo( TwillTestCase ):
self.visit_page('forms/manage?show_filter=Deleted')
self.check_page_for_string(form_one_latest.name)
self.logout()
+3 -3
View File
@@ -123,7 +123,7 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
data = app.datatypes_registry.change_datatype( data, file_type )
data.init_meta()
data.set_peek()
app.model.flush()
data.flush()
elif fields[0] == "#NewFile":
description = fields[1]
chr = fields[2]
@@ -137,7 +137,7 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
newdata.flush()
app.security_agent.copy_dataset_permissions( base_dataset.dataset, newdata.dataset )
history.add_dataset( newdata )
app.model.flush()
history.flush()
try:
copyfile(filepath,newdata.file_name)
newdata.info = newdata.name
@@ -148,4 +148,4 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
newdata.dbkey = dbkey
newdata.init_meta()
newdata.set_peek()
app.model.flush()
newdata.flush()
-1
View File
@@ -16,4 +16,3 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
data.flush()
except:
continue
app.model.flush()
+1 -3
View File
@@ -21,7 +21,6 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
output_data.dbkey = dbkey
output_data.name = basic_name + " (" + dbkey + ")"
output_data.flush()
app.model.flush()
output_data_list.append(output_data)
elif line.startswith("#FILE"):
fields = line.split("\t")
@@ -36,7 +35,6 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
app.security_agent.copy_dataset_permissions( output_data.dataset, newdata.dataset )
newdata.flush()
history.flush()
app.model.flush()
try:
move(filepath,newdata.file_name)
newdata.info = newdata.name
@@ -47,7 +45,7 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
newdata.dbkey = dbkey
newdata.init_meta()
newdata.set_peek()
app.model.flush()
newdata.flush()
output_data_list.append(newdata)
else:
new_stdout = new_stdout + line