diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index 01127a988a1..ba1033751a9 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -369,6 +369,7 @@ class JobWrapper( object ):
idata = dataset_assoc.dataset
if not idata: continue
idata.refresh()
+ idata.dataset.refresh() #we need to refresh the base Dataset, since that is where 'state' is stored
# don't run jobs for which the input dataset was deleted
if idata.deleted == True:
self.fail( "input data %d was deleted before this job ran" % idata.hid )
@@ -398,30 +399,36 @@ class JobWrapper( object ):
if job.state == job.states.DELETED:
self.cleanup()
return
- job.state = 'ok'
+ if stderr:
+ job.state = "error"
+ else:
+ job.state = 'ok'
for dataset_assoc in job.output_datasets:
- dataset = dataset_assoc.dataset
- dataset.refresh()
- dataset.state = model.Dataset.states.OK
- dataset.blurb = 'done'
- dataset.peek = 'no peek'
- dataset.info = stdout + stderr
- dataset.set_size()
- if dataset.has_data():
- # Only set metadata values if they are missing...
- if dataset.missing_meta():
- dataset.set_meta()
- else:
- # ...however, some tools add / remove columns,
- # so we have to reset the readonly metadata values
- dataset.set_readonly_meta()
- dataset.set_peek()
- else:
- dataset.blurb = "empty"
if stderr:
- dataset.state = model.Dataset.states.ERROR
- dataset.blurb = "error"
- job.state = "error"
+ dataset_assoc.dataset.dataset.state = model.Dataset.states.ERROR
+ else:
+ dataset_assoc.dataset.dataset.state = model.Dataset.states.OK
+ dataset_assoc.dataset.dataset.flush()
+ for dataset in dataset_assoc.dataset.dataset.history_associations: #need to update all associated output hdas, i.e. history was shared with job running
+ dataset.blurb = 'done'
+ dataset.peek = 'no peek'
+ dataset.info = stdout + stderr
+ dataset.set_size()
+ if stderr:
+ dataset.blurb = "error"
+ elif dataset.has_data():
+ # Only set metadata values if they are missing...
+ if dataset.missing_meta():
+ dataset.set_meta()
+ else:
+ # ...however, some tools add / remove columns,
+ # so we have to reset the readonly metadata values
+ dataset.set_readonly_meta()
+ dataset.set_peek()
+ else:
+ dataset.blurb = "empty"
+ dataset.flush()
+
# Save stdout and stderr
if len( stdout ) > 32768:
log.error( "stdout for job %d is greater than 32K, only first part will be logged to database" % job.id )
@@ -588,6 +595,8 @@ class JobStopQueue( object ):
for dataset_assoc in job.output_datasets:
dataset = dataset_assoc.dataset
dataset.refresh()
+ #only the originator of the job can delete a dataset to cause
+ #cancellation of the job, no need to loop through history_associations
if not dataset.deleted:
return False
return True
@@ -601,11 +610,16 @@ class JobStopQueue( object ):
for dataset_assoc in job.output_datasets:
dataset = dataset_assoc.dataset
dataset.refresh()
- dataset.state = model.Dataset.states.DELETED
- dataset.blurb = 'deleted'
- dataset.peek = 'Job deleted'
- dataset.info = 'Job deleted by user before it completed'
- dataset.flush()
+ dataset.deleted = True
+ dataset.state = dataset.states.DISCARDED
+ dataset.dataset.flush()
+ for dataset in dataset.dataset.history_associations:
+ #propagate info across shared datasets
+ dataset.deleted = True
+ dataset.blurb = 'deleted'
+ dataset.peek = 'Job deleted'
+ dataset.info = 'Job deleted by user before it completed'
+ dataset.flush()
def put( self, job ):
self.queue.put( job )
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 74370a8a6fa..983948aaa72 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -99,92 +99,11 @@ class JobToOutputDatasetAssociation( object ):
def __init__( self, name, dataset ):
self.name = name
self.dataset = dataset
-
-class History( object ):
- def __init__( self, id=None, name=None, user=None ):
- self.id = id
- self.name = name or "Unnamed history"
- self.deleted = False
- self.purged = False
- self.genome_build = None
- # Relationships
- self.user = user
- self.datasets = []
- self.galaxy_sessions = []
-
- def _next_hid( self ):
- # TODO: override this with something in the database that ensures
- # better integrity
- if len( self.datasets ) == 0:
- return 1
- else:
- last_hid = 0
- for dataset in self.datasets:
- if dataset.hid > last_hid:
- last_hid = dataset.hid
- return last_hid + 1
- def add_galaxy_session( self, galaxy_session, association=None ):
- if association is None:
- self.galaxy_sessions.append( GalaxySessionToHistoryAssociation( galaxy_session, self ) )
- else:
- self.galaxy_sessions.append( association )
-
- def add_dataset( self, dataset, parent_id=None, genome_build=None, set_hid = True ):
- if parent_id:
- for data in self.datasets:
- if data.id == parent_id:
- dataset.hid = data.hid
- break
- else:
- if set_hid: dataset.hid = self._next_hid()
- else:
- if set_hid: dataset.hid = self._next_hid()
- self.genome_build = genome_build
- self.datasets.append( dataset )
-
- def copy(self):
- des = History()
- des.flush()
- des.name = self.name
- des.user_id = self.user_id
- for data in self.datasets:
- new_data = data.copy()
- des.add_dataset(new_data)
- new_data.hid = data.hid
- new_data.flush()
- for child_assoc in data.children:
- new_child = child_assoc.child.copy()
- new_assoc = DatasetChildAssociation( child_assoc.designation )
- new_assoc.child = new_child
- new_assoc.parent = new_data
- new_child.flush()
- des.hid_counter = self.hid_counter
- des.flush()
- return des
-
-# class Query( object ):
-# def __init__( self, name=None, state=None, tool_parameters=None, history=None ):
-# self.name = name or "Unnamed query"
-# self.state = state
-# self.tool_parameters = tool_parameters
-# # Relationships
-# self.history = history
-# self.datasets = []
-
-class Dataset( object ):
- states = Bunch( NEW = 'new',
- QUEUED = 'queued',
- RUNNING = 'running',
- OK = 'ok',
- EMPTY = 'empty',
- ERROR = 'error',
- DELETED = 'deleted')
- file_path = "/tmp/"
- engine = None
+class HistoryDatasetAssociation( object ):
def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None,
- dbkey=None, state=None, metadata=None, history=None, parent_id=None, designation=None,
- validation_errors=None, visible=True, filename_id = None, file_size=None ):
+ dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None,
+ parent_id=None, validation_errors=None, visible=True, create_dataset = False ):
self.name = name or "Unnamed dataset"
self.id = id
self.hid = hid
@@ -193,71 +112,44 @@ class Dataset( object ):
self.peek = peek
self.extension = extension
self.dbkey = dbkey
- self.state = state
- self._metadata = metadata or dict()
- self.parent_id = parent_id
self.designation = designation
- self.deleted = False
- self.purged = False
+ self._metadata = metadata or dict()
+ self.deleted = deleted
self.visible = visible
- self.filename_id = filename_id
- self.file_size = file_size
# Relationships
self.history = history
+ if not dataset and create_dataset:
+ dataset = Dataset()
+ dataset.flush()
+ self.dataset = dataset
+ self.parent_id = parent_id
self.validation_errors = validation_errors
-
+
@property
def ext( self ):
return self.extension
+ @property
+ def states( self ):
+ return self.dataset.states
+
+ def get_dataset_state( self ):
+ return self.dataset.state
+ def set_dataset_state ( self, state ):
+ self.dataset.state = state
+ state = property( get_dataset_state, set_dataset_state )
+
def get_file_name( self ):
- if self.filename_id is None:
- assert self.id is not None, "ID must be set before filename used (commit the object)"
- # First try filename directly under file_path
- filename = os.path.join( self.file_path, "dataset_%d.dat" % self.id )
- # Only use that filename if it already exists (backward compatibility),
- # otherwise construct hashed path
- if not os.path.exists( filename ):
- dir = os.path.join( self.file_path, *directory_hash_id( self.id ) )
- # Create directory if it does not exist
- try:
- os.makedirs( dir )
- except OSError, e:
- # File Exists is okay, otherwise reraise
- if e.errno != errno.EEXIST:
- raise
- # Return filename inside hashed directory
- return os.path.abspath( os.path.join( dir, "dataset_%d.dat" % self.id ) )
- else:
- filename = self.dataset_file.filename
- # Make filename absolute
- return os.path.abspath( filename )
+ return self.dataset.get_file_name()
def set_file_name (self, filename):
- if filename is None:
- self.filename_id = None
- else:
- filename_obj = DatasetFileName.get_by(filename=filename)
- if filename_obj is None:
- filename_obj = DatasetFileName(filename=filename, extra_files_path=self.extra_files_path)
- filename_obj.flush()
- self.filename_id = filename_obj.id
- self.flush()
- self.refresh()
+ return self.dataset.set_file_name( filename )
file_name = property( get_file_name, set_file_name )
@property
def extra_files_path( self ):
- if self.dataset_file and self.dataset_file.extra_files_path:
- path = self.dataset_file.extra_files_path
- else:
- path = os.path.join( self.file_path, "dataset_%d_files" % self.id )
- #only use path directly under self.file_path if it exists
- if not os.path.exists( path ):
- path = os.path.join( os.path.join( self.file_path, *directory_hash_id( self.id ) ), "dataset_%d_files" % self.id )
- # Make path absolute
- return os.path.abspath( path )
+ return self.dataset.extra_files_path
@property
def datatype( self ):
@@ -279,7 +171,7 @@ class Dataset( object ):
def get_dbkey( self ):
dbkey = self.metadata.dbkey
if not isinstance(dbkey, list): dbkey = [dbkey]
- if dbkey in [["?"], [None], []]: dbkey = [self.old_dbkey]
+ #if dbkey in [["?"], [None], []]: dbkey = [self.old_dbkey]
if dbkey in [[None], []]: return "?"
return dbkey[0]
def set_dbkey( self, value ):
@@ -288,10 +180,10 @@ class Dataset( object ):
self.metadata.dbkey = [value]
else:
self.metadata.dbkey = value
- if isinstance(value, list):
- self.old_dbkey = value[0]
- else:
- self.old_dbkey = value
+ #if isinstance(value, list):
+ # self.old_dbkey = value[0]
+ #else:
+ # self.old_dbkey = value
dbkey = property( get_dbkey, set_dbkey )
def change_datatype( self, new_ext ):
@@ -299,22 +191,13 @@ class Dataset( object ):
datatypes_registry.change_datatype( self, new_ext )
def get_size( self ):
"""Returns the size of the data on disk"""
- if self.file_size:
- return self.file_size
- else:
- try:
- return os.path.getsize( self.file_name )
- except OSError:
- return 0
+ return self.dataset.get_size()
def set_size( self ):
"""Returns the size of the data on disk"""
- try:
- self.file_size = os.path.getsize( self.file_name )
- except OSError:
- self.file_size = 0
+ return self.dataset.set_size()
def has_data( self ):
"""Detects whether there is any data"""
- return self.get_size() > 0
+ return self.dataset.has_data()
def get_raw_data( self ):
"""Returns the full data. To stream it open the file_name and read/write as needed"""
return self.datatype.get_raw_data( self )
@@ -346,55 +229,33 @@ class Dataset( object ):
return self.datatype.display_name( self )
def display_info( self ):
return self.datatype.display_info( self )
- def get_associated_files_by_type( self, file_type ):
+ def get_converted_files_by_type( self, file_type ):
valid = []
- for assoc in self.associated_files:
+ for assoc in self.implicitly_converted_datasets:
if not assoc.deleted and assoc.type == file_type:
- valid.append( assoc )
+ valid.append( assoc.dataset )
return valid
def clear_associated_files( self, metadata_safe = False, purge = False ):
#metadata_safe = True means to only clear when assoc.metadata_safe == False
- for assoc in self.associated_files:
+ for assoc in self.implicitly_converted_datasets:
if not metadata_safe or not assoc.metadata_safe:
assoc.clear( purge = purge )
def get_child_by_designation(self, designation):
- # if self.history:
- # for data in self.history.datasets:
- # if data.parent_id and data.parent_id == self.id:
- # if designation == data.designation:
- # return data
- for child_association in self.children:
- if child_association.designation == designation:
- return child_association.child
+ for child in self.children:
+ if child.designation == designation:
+ return child
return None
+
def get_converter_types(self):
return self.datatype.get_converter_types( self, datatypes_registry)
- def copy(self, parent_id=None):
- des = Dataset(extension=self.ext)
+ def copy( self, copy_children = False, parent_id = None ):
+ des = HistoryDatasetAssociation( hid=self.hid, name=self.name, info=self.info, blurb=self.blurb, peek=self.peek, extension=self.extension, dbkey=self.dbkey, metadata=self._metadata, dataset = self.dataset, visible=self.visible, deleted=self.deleted, parent_id=parent_id )
des.flush()
- des.name = self.name
- des.info = self.info
- des.blurb = self.blurb
- des.peek = self.peek
- des.extension = self.extension
- des.dbkey = str( self.dbkey )
- des.state = self.state
- des.metadata = self.metadata
- des.hid = self.hid
- des.deleted = self.deleted
- des.purged = self.purged
- # Make sure source is using filename table, so purge works properly
- if not self.dataset_file:
- self.set_file_name(self.file_name)
- self.flush()
- self.refresh()
- self.dataset_file.extra_files_path = self.extra_files_path
- self.flush()
- # Don't copy file contents, share original file
- des.file_name = self.file_name
- des.hid = self.hid
- des.designation = self.designation
+ if copy_children:
+ for child in self.children:
+ child_copy = child.copy( copy_children = copy_children, parent_id = des.id )
+ des.set_peek() #in some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
des.flush()
return des
@@ -407,9 +268,166 @@ class Dataset( object ):
def mark_deleted( self, include_children=True ):
self.deleted = True
if include_children:
- for child_assoc in self.children:
- child_assoc.child.mark_deleted()
+ for child in self.children:
+ child.mark_deleted()
+
+
+
+class History( object ):
+ def __init__( self, id=None, name=None, user=None ):
+ self.id = id
+ self.name = name or "Unnamed history"
+ self.deleted = False
+ self.purged = False
+ self.genome_build = None
+ # Relationships
+ self.user = user
+ self.datasets = []
+ self.galaxy_sessions = []
+
+ def _next_hid( self ):
+ # TODO: override this with something in the database that ensures
+ # better integrity
+ if len( self.datasets ) == 0:
+ return 1
+ else:
+ last_hid = 0
+ for dataset in self.datasets:
+ if dataset.hid > last_hid:
+ last_hid = dataset.hid
+ return last_hid + 1
+
+ def add_galaxy_session( self, galaxy_session, association=None ):
+ if association is None:
+ self.galaxy_sessions.append( GalaxySessionToHistoryAssociation( galaxy_session, self ) )
+ else:
+ self.galaxy_sessions.append( association )
+
+ def add_dataset( self, dataset, parent_id=None, genome_build=None, set_hid = True ):
+ if isinstance( dataset, Dataset ):
+ dataset = HistoryDatasetAssociation( dataset = dataset )
+ dataset.flush()
+ elif not isinstance( dataset, HistoryDatasetAssociation ):
+ raise TypeError, "You can only add Dataset and HistoryDatasetAssociation instances to a history."
+ if parent_id:
+ for data in self.datasets:
+ if data.id == parent_id:
+ dataset.hid = data.hid
+ break
+ else:
+ if set_hid: dataset.hid = self._next_hid()
+ else:
+ if set_hid: dataset.hid = self._next_hid()
+ dataset.history = self
+ if genome_build not in [None, '?']:
+ self.genome_build = genome_build
+ self.datasets.append( dataset )
+
+ def copy(self):
+ des = History()
+ des.flush()
+ des.name = self.name
+ des.user_id = self.user_id
+ for data in self.datasets:
+ new_data = data.copy( copy_children = True )
+ des.add_dataset( new_data )
+ new_data.flush()
+ des.hid_counter = self.hid_counter
+ des.flush()
+ return des
+
+# class Query( object ):
+# def __init__( self, name=None, state=None, tool_parameters=None, history=None ):
+# self.name = name or "Unnamed query"
+# self.state = state
+# self.tool_parameters = tool_parameters
+# # Relationships
+# self.history = history
+# self.datasets = []
+
+class Dataset( object ):
+ states = Bunch( NEW = 'new',
+ QUEUED = 'queued',
+ RUNNING = 'running',
+ OK = 'ok',
+ EMPTY = 'empty',
+ ERROR = 'error',
+ DISCARDED = 'discarded' )
+ file_path = "/tmp/"
+ engine = None
+ def __init__( self, id=None, state=None, external_filename=None, extra_files_path=None, file_size=None, purgable=True ):
+ self.id = id
+ self.state = state
+ self.deleted = False
+ self.purged = False
+ self.purgable = purgable
+ self.external_filename = external_filename
+ self._extra_files_path = extra_files_path
+ self.file_size = file_size
+
+ def get_file_name( self ):
+ if not self.external_filename:
+ assert self.id is not None, "ID must be set before filename used (commit the object)"
+ # First try filename directly under file_path
+ filename = os.path.join( self.file_path, "dataset_%d.dat" % self.id )
+ # Only use that filename if it already exists (backward compatibility),
+ # otherwise construct hashed path
+ if not os.path.exists( filename ):
+ dir = os.path.join( self.file_path, *directory_hash_id( self.id ) )
+ # Create directory if it does not exist
+ try:
+ os.makedirs( dir )
+ except OSError, e:
+ # File Exists is okay, otherwise reraise
+ if e.errno != errno.EEXIST:
+ raise
+ # Return filename inside hashed directory
+ return os.path.abspath( os.path.join( dir, "dataset_%d.dat" % self.id ) )
+ else:
+ filename = self.external_filename
+ # Make filename absolute
+ return os.path.abspath( filename )
+ def set_file_name ( self, filename ):
+ if not filename:
+ self.external_filename = None
+ else:
+ self.external_filename = filename
+
+ file_name = property( get_file_name, set_file_name )
+
+ @property
+ def extra_files_path( self ):
+ if self._extra_files_path:
+ path = self._extra_files_path
+ else:
+ path = os.path.join( self.file_path, "dataset_%d_files" % self.id )
+ #only use path directly under self.file_path if it exists
+ if not os.path.exists( path ):
+ path = os.path.join( os.path.join( self.file_path, *directory_hash_id( self.id ) ), "dataset_%d_files" % self.id )
+ # Make path absolute
+ return os.path.abspath( path )
+
+ def get_size( self ):
+ """Returns the size of the data on disk"""
+ if self.file_size:
+ return self.file_size
+ else:
+ try:
+ return os.path.getsize( self.file_name )
+ except OSError:
+ return 0
+ def set_size( self ):
+ """Returns the size of the data on disk"""
+ try:
+ self.file_size = os.path.getsize( self.file_name )
+ except OSError:
+ self.file_size = 0
+ def has_data( self ):
+ """Detects whether there is any data"""
+ return self.get_size() > 0
+ def mark_deleted( self, include_children=True ):
+ self.deleted = True
# FIXME: sqlalchemy will replace this
def _delete(self):
@@ -419,12 +437,6 @@ class Dataset( object ):
except OSError, e:
log.critical('%s delete error %s' % (self.__class__.__name__, e))
-class DatasetFileName( object ):
- def __init__( self, filename=None, readonly=False, extra_files_path=None ):
- self.filename = filename
- self.readonly = readonly
- self.extra_files_path = extra_files_path
-
class Old_Dataset( Dataset ):
pass
@@ -439,47 +451,22 @@ class DatasetToValidationErrorAssociation( object ):
self.dataset = dataset
self.validation_error = validation_error
-class DatasetChildAssociation( object ):
- def __init__( self, designation=None ):
- self.designation = designation
- self.parent = None
- self.child = None
-
-class DatasetAssociatedFile( object ):
- def __init__( self, id = None, dataset_id = None, file_type = None, parent_id = None, filename = None, deleted = False, purged = False, metadata_safe = True ):
+class ImplicitlyConvertedDatasetAssociation( object ):
+ def __init__( self, id = None, parent = None, dataset = None, file_type = None, deleted = False, purged = False, metadata_safe = True ):
self.id = id
- self.dataset_id = dataset_id
+ self.dataset = dataset
+ self.parent = parent
self.type = file_type
- self.parent_id = parent_id
- self.filename = filename
self.deleted = deleted
self.purged = purged
self.metadata_safe = metadata_safe
- def get_file_name( self ):
- #return absolute path of the filename
- if self.filename:
- return os.path.abspath( self.filename )
- if self.dataset_id is not None:
- return self.dataset.file_name
- else:
- assert self.id is not None, "ID must be set before filename used (commit the object)"
- assert self.parent_id is not None, "Parent ID must be set before filename used"
- return os.path.abspath( "%s_accociated_%s" % ( self.parent.file_name, self.id ) )
- def set_file_name ( self, filename ):
- self.filename = filename
- if self.dataset:
- self.dataset.deleted = True
- self.dataset = None
- self.dataset_id = None
- file_name = property( get_file_name, set_file_name )
-
def clear( self, purge = False ):
self.deleted = True
if self.dataset:
self.dataset.deleted = True
self.dataset.purged = purge
- if purge:
+ if purge: #do something with purging
self.purged = True
try: os.unlink( self.file_name )
except Exception, e: print "Failed to purge associated file (%s) from disk: %s" % ( self.file_name, e )
diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py
index 4b2011aec51..27d8d4342f6 100644
--- a/lib/galaxy/model/mapping.py
+++ b/lib/galaxy/model/mapping.py
@@ -63,62 +63,54 @@ History.table = Table( "history", metadata,
# Column( "state", String( 64 ) ),
# Column( "tool_parameters", Pickle() ) )
-Dataset.table = Table( "dataset", metadata,
+
+HistoryDatasetAssociation.table = Table( "history_dataset_association", metadata,
Column( "id", Integer, primary_key=True ),
- Column( "create_time", DateTime, default=now ),
- Column( "update_time", DateTime, index=True, default=now, onupdate=now ),
- Column( "hid", Integer ),
Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ),
+ Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
+ Column( "create_time", DateTime, default=now ),
+ Column( "update_time", DateTime, default=now, onupdate=now ),
+ Column( "hid", Integer ),
Column( "name", TrimmedString( 255 ) ),
Column( "info", TrimmedString( 255 ) ),
Column( "blurb", TrimmedString( 255 ) ),
Column( "peek" , TEXT ),
Column( "extension", TrimmedString( 64 ) ),
- Column( "dbkey", TrimmedString( 64 ), key="old_dbkey" ), # maps to old_dbkey, see __init__.py
- Column( "state", TrimmedString( 64 ) ),
Column( "metadata", MetadataType(), key="_metadata" ),
- Column( "parent_id", Integer, nullable=True ),
+ Column( "parent_id", Integer, ForeignKey( "history_dataset_association.id" ), nullable=True ),
Column( "designation", TrimmedString( 255 ) ),
Column( "deleted", Boolean, index=True, default=False ),
- Column( "purged", Boolean, index=True, default=False ),
- Column( "visible", Boolean ),
- Column( "filename_id", Integer, ForeignKey( "dataset_filename.id" ), index=True, nullable=True ),
- Column( 'file_size', Numeric( 15, 0 ) ),
- ForeignKeyConstraint(['parent_id'],['dataset.id'], ondelete="CASCADE") )
+ Column( "visible", Boolean ) )
-DatasetAssociatedFile.table = Table( "dataset_associated_file", metadata,
+Dataset.table = Table( "dataset", metadata,
+ Column( "id", Integer, primary_key=True ),
+ Column( "create_time", DateTime, default=now ),
+ Column( "update_time", DateTime, index=True, default=now, onupdate=now ),
+ Column( "state", TrimmedString( 64 ) ),
+ Column( "deleted", Boolean, index=True, default=False ),
+ Column( "purged", Boolean, index=True, default=False ),
+ Column( "purgable", Boolean, default=True ),
+ Column( "external_filename" , TEXT ),
+ Column( "_extra_files_path", TEXT ),
+ Column( 'file_size', Numeric( 15, 0 ) ) )
+
+ImplicitlyConvertedDatasetAssociation.table = Table( "implicitly_converted_dataset_association", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
- Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True, nullable=True ),
- Column( "parent_id", Integer, ForeignKey( "dataset.id" ), index=True ),
- Column( "filename", TEXT ),
+ Column( "hda_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True, nullable=True ),
+ Column( "hda_parent_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
Column( "deleted", Boolean, index=True, default=False ),
- Column( "purged", Boolean, index=True, default=False ),
Column( "metadata_safe", Boolean, index=True, default=True ),
Column( "type", TrimmedString( 255 ) ) )
-DatasetFileName.table = Table( "dataset_filename", metadata,
- Column( "id", Integer, primary_key=True ),
- Column( "create_time", DateTime, default=now ),
- Column( "update_time", DateTime, default=now, onupdate=now ),
- Column( "filename", TEXT ),
- Column( "extra_files_path", TEXT, nullable=True, default=None ),
- Column( "readonly", Boolean, default=False ) )
-
ValidationError.table = Table( "validation_error", metadata,
Column( "id", Integer, primary_key=True ),
- Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
+ Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
Column( "message", TrimmedString( 255 ) ),
Column( "err_type", TrimmedString( 64 ) ),
Column( "attributes", TEXT ) )
-DatasetChildAssociation.table = Table( "dataset_child_association", metadata,
- Column( "id", Integer, primary_key=True ),
- Column( "parent_dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
- Column( "child_dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
- Column( "designation", TrimmedString( 255 ) ) )
-
Job.table = Table( "job", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
@@ -147,13 +139,13 @@ JobParameter.table = Table( "job_parameter", metadata,
JobToInputDatasetAssociation.table = Table( "job_to_input_dataset", metadata,
Column( "id", Integer, primary_key=True ),
Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ),
- Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
+ Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
Column( "name", String(255) ) )
JobToOutputDatasetAssociation.table = Table( "job_to_output_dataset", metadata,
Column( "id", Integer, primary_key=True ),
Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ),
- Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
+ Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
Column( "name", String(255) ) )
Event.table = Table( "event", metadata,
@@ -236,57 +228,59 @@ WorkflowStepConnection.table = Table( "workflow_step_connection", metadata,
assign_mapper( context, ValidationError, ValidationError.table )
-# assign_mapper( context, Dataset, Dataset.table,
-# properties=dict( children=relation( DatasetChildAssociation, primaryjoin=( DatasetChildAssociation.table.c.parent_dataset_id == Dataset.table.c.id ),
-# lazy=False ),
-# validation_errors=relation( ValidationError, lazy=False ) ) )
-
-assign_mapper( context, Dataset, Dataset.table,
+assign_mapper( context, HistoryDatasetAssociation, HistoryDatasetAssociation.table,
properties=dict(
+ dataset=relation(
+ Dataset,
+ primaryjoin=( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) ),
+ history=relation(
+ History,
+ primaryjoin=( History.table.c.id == HistoryDatasetAssociation.table.c.history_id ) ),
+ implicitly_converted_datasets=relation(
+ ImplicitlyConvertedDatasetAssociation,
+ primaryjoin=( ImplicitlyConvertedDatasetAssociation.table.c.hda_parent_id == HistoryDatasetAssociation.table.c.id ) ),
children=relation(
- DatasetChildAssociation,
- primaryjoin=( DatasetChildAssociation.table.c.parent_dataset_id == Dataset.table.c.id ),
- lazy=False,
- backref="parent" ),
- dataset_file=relation(
- DatasetFileName,
- primaryjoin=( DatasetFileName.table.c.id == Dataset.table.c.filename_id ) ),
- associated_files=relation(
- DatasetAssociatedFile,
- primaryjoin=( DatasetAssociatedFile.table.c.parent_id == Dataset.table.c.id ) )
+ HistoryDatasetAssociation,
+ primaryjoin=( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ),
+ backref=backref( "parent", remote_side=[HistoryDatasetAssociation.table.c.id] ) )
) )
-assign_mapper( context, DatasetFileName, DatasetFileName.table )
+assign_mapper( context, Dataset, Dataset.table,
+ properties=dict(
+ history_associations=relation(
+ HistoryDatasetAssociation,
+ primaryjoin=( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) )
+ ) )
-assign_mapper( context, DatasetChildAssociation, DatasetChildAssociation.table,
- properties=dict( child=relation( Dataset, backref="parent", primaryjoin=( DatasetChildAssociation.table.c.child_dataset_id == Dataset.table.c.id ) ) ) )
-
-assign_mapper( context, DatasetAssociatedFile, DatasetAssociatedFile.table,
- properties=dict( parent=relation(
- Dataset,
- primaryjoin=( DatasetAssociatedFile.table.c.parent_id == Dataset.table.c.id ) ),
-
- dataset=relation(
- Dataset,
- primaryjoin=( DatasetAssociatedFile.table.c.dataset_id == Dataset.table.c.id ) ) ) )
# assign_mapper( model.Query, model.Query.table,
# properties=dict( datasets=relation( model.Dataset.mapper, backref="query") ) )
+
+assign_mapper( context, ImplicitlyConvertedDatasetAssociation, ImplicitlyConvertedDatasetAssociation.table,
+ properties=dict( parent=relation(
+ HistoryDatasetAssociation,
+ primaryjoin=( ImplicitlyConvertedDatasetAssociation.table.c.hda_parent_id == HistoryDatasetAssociation.table.c.id ) ),
+
+ dataset=relation(
+ HistoryDatasetAssociation,
+ primaryjoin=( ImplicitlyConvertedDatasetAssociation.table.c.hda_id == HistoryDatasetAssociation.table.c.id ) ) ) )
+
assign_mapper( context, History, History.table,
properties=dict( galaxy_sessions=relation( GalaxySessionToHistoryAssociation ),
- datasets=relation( Dataset, backref="history", order_by=asc(Dataset.table.c.hid) ),
- active_datasets=relation( Dataset, primaryjoin=( ( Dataset.c.history_id == History.table.c.id ) & ( not_( Dataset.c.deleted ) ) ), order_by=asc( Dataset.table.c.hid ), lazy=False, viewonly=True ) ) )
+ datasets=relation( HistoryDatasetAssociation, backref="history", order_by=asc(HistoryDatasetAssociation.table.c.hid) ),
+ active_datasets=relation( HistoryDatasetAssociation, primaryjoin=( ( HistoryDatasetAssociation.table.c.history_id == History.table.c.id ) & ( not_( HistoryDatasetAssociation.table.c.deleted ) ) ), order_by=asc( HistoryDatasetAssociation.table.c.hid ), lazy=False, viewonly=True ) ) )
+
assign_mapper( context, User, User.table,
properties=dict( histories=relation( History, backref="user",
order_by=desc(History.table.c.update_time) ) ) )
assign_mapper( context, JobToInputDatasetAssociation, JobToInputDatasetAssociation.table,
- properties=dict( job=relation( Job ), dataset=relation( Dataset ) ) )
+ properties=dict( job=relation( Job ), dataset=relation( HistoryDatasetAssociation ) ) )
assign_mapper( context, JobToOutputDatasetAssociation, JobToOutputDatasetAssociation.table,
- properties=dict( job=relation( Job ), dataset=relation( Dataset ) ) )
+ properties=dict( job=relation( Job ), dataset=relation( HistoryDatasetAssociation ) ) )
assign_mapper( context, JobParameter, JobParameter.table )
@@ -310,7 +304,7 @@ assign_mapper( context, GalaxySessionToHistoryAssociation, GalaxySessionToHistor
properties=dict( galaxy_session=relation( GalaxySession ),
history=relation( History ) ) )
-Dataset.mapper.add_property( "creating_job_associations", relation( JobToOutputDatasetAssociation ) )
+HistoryDatasetAssociation.mapper.add_property( "creating_job_associations", relation( JobToOutputDatasetAssociation ) )
assign_mapper( context, Workflow, Workflow.table,
properties=dict( steps=relation( WorkflowStep, backref='workflow', order_by=asc(WorkflowStep.table.c.order_index), cascade="all, delete-orphan" ) ) )
diff --git a/lib/galaxy/model/mapping_tests.py b/lib/galaxy/model/mapping_tests.py
index b267bcf1195..e040c59e393 100644
--- a/lib/galaxy/model/mapping_tests.py
+++ b/lib/galaxy/model/mapping_tests.py
@@ -16,7 +16,7 @@ class MappingTests( unittest.TestCase ):
#h1.queries.append( model.Query( "h1->q2" ) )
h2 = model.History( name=( "H" * 1024 ) )
#q1 = model.Query( "h2->q1" )
- d1 = model.Dataset( metadata=dict(chromCol=1,startCol=2,endCol=3 ), history=h2 )
+ d1 = model.HistoryDatasetAssociation( metadata=dict(chromCol=1,startCol=2,endCol=3 ), history=h2, create_dataset=True )
#h2.queries.append( q1 )
#h2.queries.append( model.Query( "h2->q2" ) )
model.context.current.flush()
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index d7ad1e9fd6b..2e3d3dda6b0 100644
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -970,20 +970,15 @@ class Tool:
for name, data in input_datasets.items():
param_dict[name] = DatasetFilenameWrapper( data, datatypes_registry = self.app.datatypes_registry, tool = self, name = name )
if data:
- for child_association in data.children:
- child = child_association.child
- key = "_CHILD___%s___%s" % ( name, child.designation )
- param_dict[ key ] = DatasetFilenameWrapper( child )
+ for child in data.children:
+ param_dict[ "_CHILD___%s___%s" % ( name, child.designation ) ] = DatasetFilenameWrapper( child )
for name, data in output_datasets.items():
param_dict[name] = DatasetFilenameWrapper( data )
# Provide access to a path to store additional files
# TODO: path munging for cluster/dataset server relocatability
param_dict[name].files_path = os.path.abspath(os.path.join(self.app.config.new_file_path, "dataset_%s_files" % (data.id) ))
-
- for child_association in data.children:
- child = child_association.child
- key = "_CHILD___%s___%s" % ( name, child.designation )
- param_dict[ key ] = DatasetFilenameWrapper( child )
+ for child in data.children:
+ param_dict[ "_CHILD___%s___%s" % ( name, child.designation ) ] = DatasetFilenameWrapper( child )
# We add access to app here, this allows access to app.config, etc
param_dict['__app__'] = RawObjectWrapper( self.app )
# More convienent access to app.config.new_file_path; we don't need to wrap a string
@@ -1092,25 +1087,24 @@ class Tool:
if visible == "visible": visible = True
else: visible = False
ext = fields.pop(0).lower()
- # Create new child dataset
- child_data = self.app.model.Dataset(extension=ext, parent_id=parent_id, designation=designation, visible=visible, dbkey=outdata.dbkey)
- child_data.flush()
+ child_dataset = self.app.model.HistoryDatasetAssociation( extension=ext, parent_id=outdata.id, designation=designation, visible=visible, dbkey=outdata.dbkey, create_dataset=True )
# Move data from temp location to dataset location
- shutil.move(filename, child_data.file_name)
- child_data.name = "Secondary Dataset (%s)" % (designation)
- child_data.state = child_data.states.OK
- child_data.init_meta()
- child_data.set_meta()
- child_data.set_peek()
- child_data.set_size()
- child_data.flush()
- # Add to child accociation table
- assoc = self.app.model.DatasetChildAssociation()
- assoc.child = child_data
- assoc.designation = child_data.designation
- outdata.children.append( assoc )
+ shutil.move( filename, child_dataset.file_name )
+ child_dataset.flush()
+ child_dataset.name = "Secondary Dataset (%s)" % ( designation )
+ child_dataset.state = child_dataset.states.OK
+ child_dataset.init_meta()
+ child_dataset.set_meta()
+ child_dataset.set_peek()
+ child_dataset.set_size()
+ child_dataset.flush()
# Add child to return dict
- children[name][designation] = child_data
+ children[name][designation] = child_dataset
+ for dataset in outdata.dataset.history_associations: #need to update all associated output hdas, i.e. history was shared with job running
+ if outdata == dataset: continue
+ # Create new child dataset
+ child_data = child_dataset.copy( parent_id = dataset.id )
+ child_data.flush()
return children
def collect_primary_datasets( self, output):
@@ -1129,20 +1123,25 @@ class Tool:
else: visible = False
ext = fields.pop(0).lower()
# Create new primary dataset
- primary_data = self.app.model.Dataset(extension=ext, designation=designation, visible=visible, dbkey=outdata.dbkey)
+ primary_data = self.app.model.HistoryDatasetAssociation( extension=ext, designation=designation, visible=visible, dbkey=outdata.dbkey, create_dataset=True )
primary_data.flush()
- self.app.model.History.get(outdata.history_id).add_dataset(primary_data)
# Move data from temp location to dataset location
- shutil.move(filename, primary_data.file_name)
- primary_data.name = outdata.name
- primary_data.info = outdata.info
+ shutil.move( filename, primary_data.file_name )
+ primary_data.name = dataset.name
+ primary_data.info = dataset.info
primary_data.state = primary_data.states.OK
- primary_data.init_meta(copy_from=outdata)
+ primary_data.init_meta( copy_from=dataset )
primary_data.set_peek()
primary_data.set_size()
primary_data.flush()
+ outdata.history.add_dataset( primary_data )
# Add dataset to return dict
primary_datasets[name][designation] = primary_data
+ for dataset in outdata.dataset.history_associations: #need to update all associated output hdas, i.e. history was shared with job running
+ if outdata == dataset: continue
+ new_data = primary_data.copy()
+ dataset.history.add( new_data )
+ new_data.flush()
return primary_datasets
diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py
index ca0ba568354..595997b9719 100644
--- a/lib/galaxy/tools/actions/__init__.py
+++ b/lib/galaxy/tools/actions/__init__.py
@@ -30,17 +30,18 @@ class DefaultToolAction( object ):
for target_ext in input.extensions:
if target_ext in data.get_converter_types():
data.refresh() #need to refresh incase this conversion just took place, i.e. input above in tool performed the same conversion
- assoc = data.get_associated_files_by_type( "CONVERTED_%s" % target_ext )
- if assoc: data = assoc[0].dataset
+ datasets = data.get_converted_files_by_type( target_ext )
+ if datasets: data = datasets[0]
elif input.converter_safe( param_values, trans ):
#run converter here
- assoc = trans.app.model.DatasetAssociatedFile( parent_id = data.id, file_type = "CONVERTED_%s" % target_ext, metadata_safe = False )
+ assoc = trans.app.model.ImplicitlyConvertedDatasetAssociation( parent = data, file_type = target_ext, metadata_safe = False )
new_data = data.datatype.convert_dataset( trans, data, target_ext, return_output = True, visible = False ).values()[0]
new_data.hid = data.hid
new_data.name = data.name
- assoc.dataset_id = new_data.id
+ new_data.flush()
+ assoc.dataset = new_data
+ assoc.flush()
data = new_data
- data.flush()
break
return data
if isinstance( input, DataToolParameter ):
@@ -122,19 +123,20 @@ class DefaultToolAction( object ):
## What is the following hack for? Need to document under what
## conditions can the following occur? (james@bx.psu.edu)
# HACK: the output data has already been created
+ # this happens i.e. as a result of the async controller
if name in incoming:
dataid = incoming[name]
- data = trans.app.model.Dataset.get( dataid )
+ data = trans.app.model.HistoryDatasetAssociation.get( dataid )
assert data != None
out_data[name] = data
- continue
- # the type should match the input
- ext = output.format
- if ext == "input":
- ext = input_ext
- data = trans.app.model.Dataset(extension=ext)
- # Commit the dataset immediately so it gets database assigned unique id
- data.flush()
+ else:
+ # the type should match the input
+ ext = output.format
+ if ext == "input":
+ ext = input_ext
+ data = trans.app.model.HistoryDatasetAssociation( extension=ext, create_dataset=True )
+ # Commit the dataset immediately so it gets database assigned unique id
+ data.flush()
# Create an empty file immediately
open( data.file_name, "w" ).close()
# This may not be neccesary with the new parent/child associations
@@ -167,7 +169,7 @@ class DefaultToolAction( object ):
# Add all the top-level (non-child) datasets to the history
for name in out_data.keys():
- if name not in child_dataset_names:
+ if name not in child_dataset_names and name not in incoming: #don't add children; or already existing datasets, i.e. async created
data = out_data[ name ]
trans.history.add_dataset( data, set_hid = set_output_hid )
data.flush()
@@ -176,11 +178,7 @@ class DefaultToolAction( object ):
for parent_name, child_name in parent_to_child_pairs:
parent_dataset = out_data[ parent_name ]
child_dataset = out_data[ child_name ]
- assoc = trans.app.model.DatasetChildAssociation()
- assoc.child = child_dataset
- assoc.designation = child_dataset.designation
- parent_dataset.children.append( assoc )
- # FIXME: Child dataset hid
+ parent_dataset.children.append( child_dataset )
# Store data after custom code runs
trans.app.model.flush()
diff --git a/lib/galaxy/tools/actions/upload.py b/lib/galaxy/tools/actions/upload.py
index 32628b13682..b3904436a63 100644
--- a/lib/galaxy/tools/actions/upload.py
+++ b/lib/galaxy/tools/actions/upload.py
@@ -65,7 +65,7 @@ class UploadToolAction( object ):
return dict( output=data_list[0] )
def upload_empty(self, trans, err_code, err_msg):
- data = trans.app.model.Dataset()
+ data = trans.app.model.HistoryDatasetAssociation( create_dataset = True )
data.name = err_code
data.extension = "txt"
data.dbkey = "?"
@@ -158,13 +158,12 @@ class UploadToolAction( object ):
if info is None:
info = 'uploaded %s file' %data_type
- data = trans.app.model.Dataset()
+ data = trans.app.model.HistoryDatasetAssociation( history = trans.history, extension = ext, create_dataset = True )
data.name = file_name
- data.extension = ext
data.dbkey = dbkey
data.info = info
data.flush()
- shutil.move(temp_name, data.file_name)
+ shutil.move( temp_name, data.file_name )
data.state = data.states.OK
data.init_meta()
if self.line_count is not None:
diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py
index 79bffb4d46f..acba4fbafe0 100644
--- a/lib/galaxy/tools/parameters/basic.py
+++ b/lib/galaxy/tools/parameters/basic.py
@@ -636,11 +636,11 @@ class ColumnListParameter( SelectToolParameter ):
# from a twill perspective...
>>> # Mock up a history (not connected to database)
- >>> from galaxy.model import History, Dataset
+ >>> from galaxy.model import History, HistoryDatasetAssociation
>>> from galaxy.util.bunch import Bunch
>>> hist = History()
>>> hist.flush()
- >>> hist.add_dataset( Dataset( id=1, extension='interval' ) )
+ >>> hist.add_dataset( HistoryDatasetAssociation( id=1, extension='interval', create_dataset=True ) )
>>> dtp = DataToolParameter( None, XML( '' ) )
>>> print dtp.name
blah
@@ -979,15 +979,15 @@ class DataToolParameter( ToolParameter ):
displayed as radio buttons and multiple selects as a set of checkboxes
>>> # Mock up a history (not connected to database)
- >>> from galaxy.model import History, Dataset
+ >>> from galaxy.model import History, HistoryDatasetAssociation
>>> from galaxy.util.bunch import Bunch
>>> hist = History()
>>> hist.flush()
- >>> hist.add_dataset( Dataset( id=1, extension='txt' ) )
- >>> hist.add_dataset( Dataset( id=2, extension='bed' ) )
- >>> hist.add_dataset( Dataset( id=3, extension='fasta' ) )
- >>> hist.add_dataset( Dataset( id=4, extension='png' ) )
- >>> hist.add_dataset( Dataset( id=5, extension='interval' ) )
+ >>> hist.add_dataset( HistoryDatasetAssociation( id=1, extension='txt', create_dataset=True ) )
+ >>> hist.add_dataset( HistoryDatasetAssociation( id=2, extension='bed', create_dataset=True ) )
+ >>> hist.add_dataset( HistoryDatasetAssociation( id=3, extension='fasta', create_dataset=True ) )
+ >>> hist.add_dataset( HistoryDatasetAssociation( id=4, extension='png', create_dataset=True ) )
+ >>> hist.add_dataset( HistoryDatasetAssociation( id=5, extension='interval', create_dataset=True ) )
>>> p = DataToolParameter( None, XML( '' ) )
>>> print p.name
blah
@@ -1056,16 +1056,16 @@ class DataToolParameter( ToolParameter ):
else:
for target_ext in self.extensions:
if target_ext in data.get_converter_types():
- assoc = data.get_associated_files_by_type( "CONVERTED_%s" % target_ext )
- if assoc:
- data = assoc[0].dataset
+ datasets = data.get_converted_files_by_type( target_ext )
+ if datasets:
+ data = datasets[0]
elif not self.converter_safe( other_values, trans ):
continue
selected = ( value and ( data in value ) )
field.add_option( "%s: (as %s) %s" % ( hid, target_ext, data.name[:30] ), data.id, selected )
break #we only report the first valid converter, assume self.extensions is a priority list
# Also collect children via association object
- dataset_collector( [ assoc.child for assoc in data.children ], hid )
+ dataset_collector( data.children, hid )
dataset_collector( history.datasets, None )
some_data = bool( field.options )
if some_data:
@@ -1116,7 +1116,7 @@ class DataToolParameter( ToolParameter ):
continue
most_recent_dataset[0] = data
# Also collect children via association object
- dataset_collector( [ assoc.child for assoc in data.children ] )
+ dataset_collector( data.children )
dataset_collector( history.datasets )
most_recent_dataset = most_recent_dataset.pop()
if most_recent_dataset is not None:
@@ -1133,11 +1133,11 @@ class DataToolParameter( ToolParameter ):
if value in [None, "None"]:
return None
if isinstance( value, list ):
- return [ trans.app.model.Dataset.get( v ) for v in value ]
- elif isinstance( value, trans.app.model.Dataset ):
+ return [ trans.app.model.HistoryDatasetAssociation.get( v ) for v in value ]
+ elif isinstance( value, trans.app.model.HistoryDatasetAssociation ):
return value
else:
- return trans.app.model.Dataset.get( value )
+ return trans.app.model.HistoryDatasetAssociation.get( value )
def value_to_basic( self, value, app ):
if value is None or isinstance( value, str ):
@@ -1152,7 +1152,7 @@ class DataToolParameter( ToolParameter ):
if value is None or value == '' or value == 'None':
return value
try:
- return app.model.Dataset.get( int( value ) )
+ return app.model.HistoryDatasetAssociation.get( int( value ) )
except:
if ignore_errors:
return value
@@ -1221,7 +1221,7 @@ class DataToolParameter( ToolParameter ):
# have the history accessable at the job level, it is necessary
# I also probably wrote this docstring test thing wrong.
#
-# >>> from galaxy.model import History, Dataset
+# >>> from galaxy.model import History
# >>> from galaxy.util.bunch import Bunch
# >>> hist = History( id=1 )
# >>> p = HistoryIDParameter( None, XML( '' ) )
diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py
index 801cb89e5d6..40fa0fb135f 100644
--- a/lib/galaxy/tools/parameters/dynamic_options.py
+++ b/lib/galaxy/tools/parameters/dynamic_options.py
@@ -104,7 +104,7 @@ class DataMetaFilter( Filter ):
return file_value == dataset_value
assert self.ref_name in other_values or trans.workflow_building_mode, "Required dependency '%s' not found in incoming values" % self.ref_name
ref = other_values.get( self.ref_name, None )
- if not isinstance( ref, self.dynamic_option.tool_param.tool.app.model.Dataset ):
+ if not isinstance( ref, self.dynamic_option.tool_param.tool.app.model.HistoryDatasetAssociation ):
return [] #not a valid dataset
meta_value = ref.metadata.get( self.key, None )
assert meta_value is not None, "Required metadata value '%s' not found in referenced dataset" % self.key
diff --git a/lib/galaxy/web/controllers/async.py b/lib/galaxy/web/controllers/async.py
index cf5933b205d..529a705385e 100644
--- a/lib/galaxy/web/controllers/async.py
+++ b/lib/galaxy/web/controllers/async.py
@@ -52,7 +52,7 @@ class ASync( BaseController ):
if data_id:
if not URL:
return "No URL parameter was submitted for data %s" % data_id
- data = trans.model.Dataset.get( data_id )
+ data = trans.model.HistoryDatasetAssociation.get( data_id )
if not data:
return "Data %s does not exist or has already been deleted" % data_id
@@ -67,7 +67,8 @@ class ASync( BaseController ):
trans.log_event( 'Async executing tool %s' % tool.id, tool_id=tool.id )
galaxy_url = trans.request.base + '/async/%s/%s/%s' % ( tool_id, data.id, key )
galaxy_url = params.get("GALAXY_URL",galaxy_url)
- params = dict(url=URL, dataid=data.id, output=data.file_name, GALAXY_URL=galaxy_url)
+ params = dict( url=URL, GALAXY_URL=galaxy_url )
+ params[tool.outputs.keys()[0]] = data.id #assume there is exactly one output file possible
#tool.execute( app=self.app, history=history, incoming=params )
tool.execute( trans, incoming=params )
else:
@@ -101,14 +102,14 @@ class ASync( BaseController ):
#data.dbkey = GALAXY_BUILD
#data.state = jobs.JOB_OK
#history.datasets.add_dataset( data )
-
- data = trans.app.model.Dataset()
+ data = trans.app.model.HistoryDatasetAssociation( create_dataset = True, extension = GALAXY_TYPE )
data.name = GALAXY_NAME
- data.extension = GALAXY_TYPE
data.dbkey = GALAXY_BUILD
data.info = GALAXY_INFO
data.state = data.states.NEW
+ data.flush()
+ open( data.file_name, 'wb' ).close() #create the file
trans.history.add_dataset( data, genome_build=GALAXY_BUILD )
trans.model.flush()
trans.log_event( "Added dataset %d to history %d" %(data.id, trans.history.id ), tool_id=tool_id )
diff --git a/lib/galaxy/web/controllers/dataset.py b/lib/galaxy/web/controllers/dataset.py
index 344538fccc7..f86b648b022 100644
--- a/lib/galaxy/web/controllers/dataset.py
+++ b/lib/galaxy/web/controllers/dataset.py
@@ -47,12 +47,12 @@ class DatasetInterface( BaseController ):
@web.expose
def errors( self, trans, id ):
- dataset = model.Dataset.get( id )
+ dataset = model.HistoryDatasetAssociation.get( id )
return trans.fill_template( "dataset/errors.tmpl", dataset=dataset )
@web.expose
def stderr( self, trans, id ):
- dataset = model.Dataset.get( id )
+ dataset = model.HistoryDatasetAssociation.get( id )
job = dataset.creating_job_associations[0].job
trans.response.set_content_type( 'text/plain' )
return job.stderr
@@ -66,7 +66,7 @@ class DatasetInterface( BaseController ):
if to_address is None:
return trans.show_error_message( "Sorry, error reporting has been disabled for this galaxy instance" )
# Get the dataset and associated job
- dataset = model.Dataset.get( id )
+ dataset = model.HistoryDatasetAssociation.get( id )
job = dataset.creating_job_associations[0].job
# Build the email message
msg = MIMEText( string.Template( error_report_template )
@@ -105,7 +105,7 @@ class DatasetInterface( BaseController ):
"""Catches the dataset id and displays file contents as directed"""
if filename is None or filename.lower() == "index":
try:
- data = trans.app.model.Dataset.get( dataset_id )
+ data = trans.app.model.HistoryDatasetAssociation.get( dataset_id )
if data:
mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() )
trans.response.set_content_type(mime)
@@ -120,7 +120,7 @@ class DatasetInterface( BaseController ):
else:
#display files from directory here
try:
- file_path = os.path.join(trans.app.model.Dataset.get( dataset_id ).extra_files_path, filename)
+ file_path = os.path.join(trans.app.model.HistoryDatasetAssociation.get( dataset_id ).extra_files_path, filename)
mime, encoding = mimetypes.guess_type(file_path)
if mime is None:
mime = trans.app.datatypes_registry.get_mimetype_by_extension(".".split(file_path)[-1])
diff --git a/lib/galaxy/web/controllers/root.py b/lib/galaxy/web/controllers/root.py
index 7011e69ae26..8c1b48b0642 100644
--- a/lib/galaxy/web/controllers/root.py
+++ b/lib/galaxy/web/controllers/root.py
@@ -67,7 +67,7 @@ class RootController( BaseController ):
def dataset_state ( self, trans, id=None, stamp=None ):
if id is not None:
try:
- data = self.app.model.Dataset.get( id )
+ data = self.app.model.HistoryDatasetAssociation.get( id )
except:
return trans.show_error_message( "Unable to check dataset %s." %str( id ) )
trans.response.headers['X-Dataset-State'] = data.state
@@ -81,7 +81,7 @@ class RootController( BaseController ):
def dataset_code( self, trans, id=None, hid=None, stamp=None ):
if id is not None:
try:
- data = self.app.model.Dataset.get( id )
+ data = self.app.model.HistoryDatasetAssociation.get( id )
except:
return trans.show_error_message( "Unable to check dataset %s." %str( id ) )
trans.response.headers['Pragma'] = 'no-cache'
@@ -101,7 +101,7 @@ class RootController( BaseController ):
ids = map( int, ids.split( "," ) )
states = states.split( "," )
for id, state in zip( ids, states ):
- data = self.app.model.Dataset.get( id )
+ data = self.app.model.HistoryDatasetAssociation.get( id )
if data.state != state:
rval[id] = {
"state": data.state,
@@ -131,7 +131,7 @@ class RootController( BaseController ):
raise Exception( "No dataset with hid '%d'" % hid )
else:
try:
- data = self.app.model.Dataset.get( id )
+ data = self.app.model.HistoryDatasetAssociation.get( id )
except:
return "Dataset id '%s' is invalid" %str( id )
if data:
@@ -160,7 +160,7 @@ class RootController( BaseController ):
Returns child data directly into the browser, based upon parent_id and designation.
"""
try:
- data = self.app.model.Dataset.get( parent_id )
+ data = self.app.model.HistoryDatasetAssociation.get( parent_id )
if data:
child = data.get_child_by_designation(designation)
if child:
@@ -172,7 +172,7 @@ class RootController( BaseController ):
@web.expose
def display_as( self, trans, id=None, display_app=None, **kwd ):
"""Returns a file in a format that can successfully be displayed in display_app"""
- data = self.app.model.Dataset.get( id )
+ data = self.app.model.HistoryDatasetAssociation.get( id )
if data:
trans.response.set_content_type(data.get_mime())
trans.log_event( "Formatted dataset id %s for display at %s" % ( str(id), display_app ) )
@@ -183,7 +183,7 @@ class RootController( BaseController ):
@web.expose
def peek(self, trans, id=None):
"""Returns a 'peek' at the data"""
- data = self.app.model.Dataset.get( id )
+ data = self.app.model.HistoryDatasetAssociation.get( id )
if data:
yield ""
yield data.peek
@@ -201,7 +201,7 @@ class RootController( BaseController ):
elif id is None:
return trans.show_error_message( "Problem loading dataset id %s with history id %s." % ( str( id ), str( hid ) ) )
else:
- data = self.app.model.Dataset.get( id )
+ data = self.app.model.HistoryDatasetAssociation.get( id )
if data is None:
return trans.show_error_message( "Problem retrieving dataset id %s with history id %s." % ( str( id ), str( hid ) ) )
@@ -281,15 +281,12 @@ class RootController( BaseController ):
int( id )
except:
continue
- data = self.app.model.Dataset.get( id )
+ data = self.app.model.HistoryDatasetAssociation.get( id )
if data:
# Walk up parent datasets to find the containing history
topmost_parent = data
while topmost_parent.parent:
- # data.parent is a list of associations, data.parent.parent
- # is the actual dataset
- assert len( data.parent ) == 1, "Dataset should only have one parent"
- topmost_parent = data.parent[0].parent
+ topmost_parent = topmost_parent.parent
assert topmost_parent in history.datasets, "Data does not belong to current history"
# Mark deleted and cleanup
data.mark_deleted()
@@ -311,15 +308,12 @@ class RootController( BaseController ):
except:
return "Dataset id '%s' is invalid" %str( id )
history = trans.get_history()
- data = self.app.model.Dataset.get( id )
+ data = self.app.model.HistoryDatasetAssociation.get( id )
if data:
# Walk up parent datasets to find the containing history
topmost_parent = data
while topmost_parent.parent:
- # data.parent is a list of associations, data.parent.parent
- # is the actual dataset
- assert len( data.parent ) == 1, "Dataset should only have one parent"
- topmost_parent = data.parent[0].parent
+ topmost_parent = topmost_parent.parent
assert topmost_parent in history.datasets, "Data does not belong to current history"
# Mark deleted and cleanup
data.mark_deleted()
@@ -556,7 +550,7 @@ class RootController( BaseController ):
"""Adds a POSTed file to a History"""
try:
history = trans.app.model.History.get( history_id )
- data = trans.app.model.Dataset( name = name, info = info, extension = ext, dbkey = dbkey )
+ data = trans.app.model.HistoryDatasetAssociation( name = name, info = info, extension = ext, dbkey = dbkey, create_file = True )
data.flush()
data_file = open( data.file_name, "wb" )
file_data.file.seek( 0 )
@@ -580,7 +574,7 @@ class RootController( BaseController ):
def dataset_make_primary( self, trans, id=None):
"""Copies a dataset and makes primary"""
try:
- old_data = self.app.model.Dataset.get( id )
+ old_data = self.app.model.HistoryDatasetAssociation.get( id )
new_data = old_data.copy()
## new_data.parent = None
## history = trans.app.model.History.get( old_data.history_id )
@@ -606,7 +600,7 @@ class RootController( BaseController ):
@web.expose
def dataset_errors( self, trans, id=None, **kwd ):
"""View/fix errors associated with dataset"""
- data = trans.app.model.Dataset.get( id )
+ data = trans.app.model.HistoryDatasetAssociation.get( id )
p = kwd
if p.get("fix_errors", None):
# launch tool to create new, (hopefully) error free dataset
diff --git a/scripts/cleanup_datasets/cleanup_datasets.py b/scripts/cleanup_datasets/cleanup_datasets.py
index 202a7ed6b53..ea87a5b2814 100644
--- a/scripts/cleanup_datasets/cleanup_datasets.py
+++ b/scripts/cleanup_datasets/cleanup_datasets.py
@@ -263,16 +263,24 @@ def purge_dataset( dataset ):
return "# Dataset for deletion ( id %s ) points to a file on disk being shared by another user's history ( dataset id %s )\n" %( str( dataset.id ), str( data.id ) )
elif dataset.deleted:
# Remove files from disk and update the database
+ purgable = False
try:
- os.unlink( dataset.file_name )
dataset.purged = True
dataset.file_size = 0
dataset.clear_associated_files( purge = True )
dataset.flush()
+ if dataset.dataset.purgable:
+ for shared_data in dataset.dataset.history_associations:
+ if not shared_data.purged:
+ break #only purge when not shared
+ else:
+ os.unlink( dataset.file_name )
+ purgable = True
except Exception, exc:
return "# Error, exception: %s caught attempting to purge %s\n" %( str( exc ), dataset.file_name )
try:
- os.unlink( dataset.extra_files_path )
+ if purgable:
+ os.unlink( dataset.extra_files_path )
except:
pass
else:
diff --git a/templates/root/history_common.mako b/templates/root/history_common.mako
index 7035941b51d..877705614ed 100644
--- a/templates/root/history_common.mako
+++ b/templates/root/history_common.mako
@@ -83,9 +83,9 @@
## be a 'visible_children' method on dataset.
<%
children = []
- for child_assoc in data.children:
- if child_assoc.child.visible:
- children.append( child_assoc.child )
+ for child in data.children:
+ if child.visible:
+ children.append( child )
%>
%if len( children ) > 0:
diff --git a/tools/data_source/encode_db.xml b/tools/data_source/encode_db.xml
index eccc16fdb1f..00f133121af 100644
--- a/tools/data_source/encode_db.xml
+++ b/tools/data_source/encode_db.xml
@@ -17,8 +17,10 @@
-
-