From 28b89d686fcfef98a5e99db20e7c71e2b6550ec8 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Wed, 9 Jul 2008 17:41:08 +0000 Subject: [PATCH] Change database schema to separate Dataset and HistoryDatasetAssociation. This is a significant change, be sure to follow the migration notes exactly. Make sure to backup your database before updating. Also fix a long standing bug in the async controller dealing with updating output datasets upon job completion. Credit for database migration directions go to Greg. *** This is a significant change, be sure to follow the migration steps below exactly and in order: *** --------------- 0. Stop Galaxy server --------------- 1. Backup database and galaxy_root/database directory --------------- 2. The current validation_error table is not being used, it is left over from Ian's work that was not made functional. However, if we want to keep it around, we should drop the old version of the table so it will get re-created correctly when the Galaxy server is restarted: any database - sql command(s): ------------------------------ DROP TABLE validation_error; --------------- 3. Stop server, perform svn update to get latest code, start server to create new history_dataset_association and implicitly_converted_dataset_association tables --------------- 4. Add new columns to dataset table: postgres sql command(s): ------------------------ ALTER TABLE dataset ADD COLUMN purgable boolean DEFAULT 't'; ALTER TABLE dataset ADD COLUMN external_filename text; ALTER TABLE dataset ADD COLUMN _extra_files_path text; mysql sql command(s): --------------------- ALTER TABLE dataset ADD COLUMN purgable boolean DEFAULT TRUE; ALTER TABLE dataset ADD COLUMN external_filename text; ALTER TABLE dataset ADD COLUMN _extra_files_path text; sqlite sql command(s): ---------------------- ALTER TABLE dataset ADD COLUMN purgable boolean DEFAULT 0; ALTER TABLE dataset ADD COLUMN external_filename text; ALTER TABLE dataset ADD COLUMN _extra_files_path text; --------------- 5. Populate new columns: postgres sql command(s): ------------------------ UPDATE dataset SET external_filename = dataset_filename.filename, _extra_files_path = dataset_filename.extra_files_path FROM dataset_filename WHERE dataset.filename_id = dataset_filename.id; mysql or sqlite sql command(s): ------------------------------- UPDATE dataset, dataset_filename SET dataset.external_filename = dataset_filename.filename, dataset._extra_files_path = dataset_filename.extra_files_path WHERE dataset.filename_id = dataset_filename.id; --------------- 6. Populate dataset table with info from dataset_child_association table: postgres sql command(s): ------------------------ UPDATE dataset SET parent_id = dataset_child_association.parent_dataset_id FROM dataset_child_association WHERE dataset.id = dataset_child_association.child_dataset_id; mysql or sqlite sql command(s): ------------------------------- UPDATE dataset, dataset_child_association SET dataset.parent_id = dataset_child_association.parent_dataset_id WHERE dataset.id = dataset_child_association.child_dataset_id; --------------- 7. Drop dataset_child_association table: any database - sql command(s): ------------------------------ DROP TABLE dataset_child_association; --------------- 8. Copy parts of the current dataset table to the new history_dataset_association table: postgres sql command(s): ------------------------ INSERT INTO history_dataset_association SELECT id, history_id, id AS dataset_id, create_time, update_time, hid, name, info, blurb, peek, extension, metadata, parent_id, designation, deleted, visible FROM dataset ORDER BY id; mysql or sqlite sql command(s): ------------------------------- INSERT INTO history_dataset_association ( history_id, dataset_id, create_time, update_time, hid, name, info, blurb, peek, extension, metadata, parent_id, designation, deleted, visible ) SELECT history_id, id, create_time, update_time, hid, name, info, blurb, peek, extension, metadata, parent_id, designation, deleted, visible FROM dataset ORDER BY id; --------------- 9. Update the nextval value of the primary key sequence in the history_dataset_association table NOTE: THIS IS NOT NECESSARY FOR MYSQL OR SQLITE! postgres sql command(s): ------------------------ SELECT setval('history_dataset_association_id_seq', max(id)) FROM history_dataset_association; --------------- 10. Update dataset table to mark dataset_associated_file dataset as deleted: postgres sql command(s): ------------------------ UPDATE dataset SET deleted = 't' WHERE id IN (SELECT dataset_id FROM dataset_associated_file ORDER BY dataset_id DESC); mysql or sqlite sql command(s): ------------------------------- UPDATE dataset SET deleted = TRUE WHERE id IN (SELECT dataset_id FROM dataset_associated_file ORDER BY dataset_id DESC); --------------- 11. Drop the dataset_associated_file table: any database - sql command(s): ------------------------------ DROP TABLE dataset_associated_file; --------------- 12. Alter foreign keys on job_to_input_dataset table: postgres or sqlite sql command(s): ------------------------ ALTER TABLE job_to_input_dataset DROP CONSTRAINT job_to_input_dataset_dataset_id_fkey; ALTER TABLE job_to_input_dataset ADD CONSTRAINT job_to_input_dataset_dataset_id_fkey FOREIGN KEY (dataset_id) REFERENCES history_dataset_association(id); mysql sql command(s): ------------------------------- ALTER TABLE job_to_input_dataset DROP FOREIGN KEY ix_job_to_input_dataset_dataset_id; ALTER TABLE job_to_input_dataset ADD FOREIGN KEY ix_job_to_input_dataset_dataset_id (dataset_id) REFERENCES history_dataset_association(id); --------------- 13. Alter foreign keys on job_to_output_dataset table: postgres or sqlite sql command(s): ------------------------ ALTER TABLE job_to_output_dataset DROP CONSTRAINT job_to_output_dataset_dataset_id_fkey; ALTER TABLE job_to_output_dataset ADD CONSTRAINT job_to_output_dataset_dataset_id_fkey FOREIGN KEY (dataset_id) REFERENCES history_dataset_association(id); mysql sql command(s): ------------------------------- ALTER TABLE job_to_output_dataset DROP FOREIGN KEY ix_job_to_output_dataset_dataset_id; ALTER TABLE job_to_output_dataset ADD FOREIGN KEY ix_job_to_output_dataset_dataset_id FOREIGN KEY (dataset_id) REFERENCES history_dataset_association(id); --------------- 14. Eliminate columns from the dataset table previously copied to history_dataset_association: any database - sql command(s): ------------------------------ ALTER TABLE dataset DROP COLUMN hid; ALTER TABLE dataset DROP COLUMN history_id; ALTER TABLE dataset DROP COLUMN name; ALTER TABLE dataset DROP COLUMN info; ALTER TABLE dataset DROP COLUMN blurb; ALTER TABLE dataset DROP COLUMN peek; ALTER TABLE dataset DROP COLUMN extension; ALTER TABLE dataset DROP COLUMN dbkey; ALTER TABLE dataset DROP COLUMN metadata; ALTER TABLE dataset DROP COLUMN parent_id; ALTER TABLE dataset DROP COLUMN designation; ALTER TABLE dataset DROP COLUMN visible; ALTER TABLE dataset DROP COLUMN filename_id; --- lib/galaxy/jobs/__init__.py | 68 +-- lib/galaxy/model/__init__.py | 433 +++++++++--------- lib/galaxy/model/mapping.py | 128 +++--- lib/galaxy/model/mapping_tests.py | 2 +- lib/galaxy/tools/__init__.py | 63 ++- lib/galaxy/tools/actions/__init__.py | 38 +- lib/galaxy/tools/actions/upload.py | 7 +- lib/galaxy/tools/parameters/basic.py | 36 +- .../tools/parameters/dynamic_options.py | 2 +- lib/galaxy/web/controllers/async.py | 11 +- lib/galaxy/web/controllers/dataset.py | 10 +- lib/galaxy/web/controllers/root.py | 36 +- scripts/cleanup_datasets/cleanup_datasets.py | 12 +- templates/root/history_common.mako | 6 +- tools/data_source/encode_db.xml | 6 +- tools/data_source/encode_import_code.py | 2 +- tools/data_source/encodedb_filter.py | 24 - tools/data_source/microbial_import_code.py | 2 +- tools/maf/maf_to_bed_code.py | 2 +- 19 files changed, 430 insertions(+), 458 deletions(-) delete mode 100644 tools/data_source/encodedb_filter.py diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 01127a988a1..ba1033751a9 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -369,6 +369,7 @@ class JobWrapper( object ): idata = dataset_assoc.dataset if not idata: continue idata.refresh() + idata.dataset.refresh() #we need to refresh the base Dataset, since that is where 'state' is stored # don't run jobs for which the input dataset was deleted if idata.deleted == True: self.fail( "input data %d was deleted before this job ran" % idata.hid ) @@ -398,30 +399,36 @@ class JobWrapper( object ): if job.state == job.states.DELETED: self.cleanup() return - job.state = 'ok' + if stderr: + job.state = "error" + else: + job.state = 'ok' for dataset_assoc in job.output_datasets: - dataset = dataset_assoc.dataset - dataset.refresh() - dataset.state = model.Dataset.states.OK - dataset.blurb = 'done' - dataset.peek = 'no peek' - dataset.info = stdout + stderr - dataset.set_size() - if dataset.has_data(): - # Only set metadata values if they are missing... - if dataset.missing_meta(): - dataset.set_meta() - else: - # ...however, some tools add / remove columns, - # so we have to reset the readonly metadata values - dataset.set_readonly_meta() - dataset.set_peek() - else: - dataset.blurb = "empty" if stderr: - dataset.state = model.Dataset.states.ERROR - dataset.blurb = "error" - job.state = "error" + dataset_assoc.dataset.dataset.state = model.Dataset.states.ERROR + else: + dataset_assoc.dataset.dataset.state = model.Dataset.states.OK + dataset_assoc.dataset.dataset.flush() + for dataset in dataset_assoc.dataset.dataset.history_associations: #need to update all associated output hdas, i.e. history was shared with job running + dataset.blurb = 'done' + dataset.peek = 'no peek' + dataset.info = stdout + stderr + dataset.set_size() + if stderr: + dataset.blurb = "error" + elif dataset.has_data(): + # Only set metadata values if they are missing... + if dataset.missing_meta(): + dataset.set_meta() + else: + # ...however, some tools add / remove columns, + # so we have to reset the readonly metadata values + dataset.set_readonly_meta() + dataset.set_peek() + else: + dataset.blurb = "empty" + dataset.flush() + # Save stdout and stderr if len( stdout ) > 32768: log.error( "stdout for job %d is greater than 32K, only first part will be logged to database" % job.id ) @@ -588,6 +595,8 @@ class JobStopQueue( object ): for dataset_assoc in job.output_datasets: dataset = dataset_assoc.dataset dataset.refresh() + #only the originator of the job can delete a dataset to cause + #cancellation of the job, no need to loop through history_associations if not dataset.deleted: return False return True @@ -601,11 +610,16 @@ class JobStopQueue( object ): for dataset_assoc in job.output_datasets: dataset = dataset_assoc.dataset dataset.refresh() - dataset.state = model.Dataset.states.DELETED - dataset.blurb = 'deleted' - dataset.peek = 'Job deleted' - dataset.info = 'Job deleted by user before it completed' - dataset.flush() + dataset.deleted = True + dataset.state = dataset.states.DISCARDED + dataset.dataset.flush() + for dataset in dataset.dataset.history_associations: + #propagate info across shared datasets + dataset.deleted = True + dataset.blurb = 'deleted' + dataset.peek = 'Job deleted' + dataset.info = 'Job deleted by user before it completed' + dataset.flush() def put( self, job ): self.queue.put( job ) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 74370a8a6fa..983948aaa72 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -99,92 +99,11 @@ class JobToOutputDatasetAssociation( object ): def __init__( self, name, dataset ): self.name = name self.dataset = dataset - -class History( object ): - def __init__( self, id=None, name=None, user=None ): - self.id = id - self.name = name or "Unnamed history" - self.deleted = False - self.purged = False - self.genome_build = None - # Relationships - self.user = user - self.datasets = [] - self.galaxy_sessions = [] - - def _next_hid( self ): - # TODO: override this with something in the database that ensures - # better integrity - if len( self.datasets ) == 0: - return 1 - else: - last_hid = 0 - for dataset in self.datasets: - if dataset.hid > last_hid: - last_hid = dataset.hid - return last_hid + 1 - def add_galaxy_session( self, galaxy_session, association=None ): - if association is None: - self.galaxy_sessions.append( GalaxySessionToHistoryAssociation( galaxy_session, self ) ) - else: - self.galaxy_sessions.append( association ) - - def add_dataset( self, dataset, parent_id=None, genome_build=None, set_hid = True ): - if parent_id: - for data in self.datasets: - if data.id == parent_id: - dataset.hid = data.hid - break - else: - if set_hid: dataset.hid = self._next_hid() - else: - if set_hid: dataset.hid = self._next_hid() - self.genome_build = genome_build - self.datasets.append( dataset ) - - def copy(self): - des = History() - des.flush() - des.name = self.name - des.user_id = self.user_id - for data in self.datasets: - new_data = data.copy() - des.add_dataset(new_data) - new_data.hid = data.hid - new_data.flush() - for child_assoc in data.children: - new_child = child_assoc.child.copy() - new_assoc = DatasetChildAssociation( child_assoc.designation ) - new_assoc.child = new_child - new_assoc.parent = new_data - new_child.flush() - des.hid_counter = self.hid_counter - des.flush() - return des - -# class Query( object ): -# def __init__( self, name=None, state=None, tool_parameters=None, history=None ): -# self.name = name or "Unnamed query" -# self.state = state -# self.tool_parameters = tool_parameters -# # Relationships -# self.history = history -# self.datasets = [] - -class Dataset( object ): - states = Bunch( NEW = 'new', - QUEUED = 'queued', - RUNNING = 'running', - OK = 'ok', - EMPTY = 'empty', - ERROR = 'error', - DELETED = 'deleted') - file_path = "/tmp/" - engine = None +class HistoryDatasetAssociation( object ): def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None, - dbkey=None, state=None, metadata=None, history=None, parent_id=None, designation=None, - validation_errors=None, visible=True, filename_id = None, file_size=None ): + dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None, + parent_id=None, validation_errors=None, visible=True, create_dataset = False ): self.name = name or "Unnamed dataset" self.id = id self.hid = hid @@ -193,71 +112,44 @@ class Dataset( object ): self.peek = peek self.extension = extension self.dbkey = dbkey - self.state = state - self._metadata = metadata or dict() - self.parent_id = parent_id self.designation = designation - self.deleted = False - self.purged = False + self._metadata = metadata or dict() + self.deleted = deleted self.visible = visible - self.filename_id = filename_id - self.file_size = file_size # Relationships self.history = history + if not dataset and create_dataset: + dataset = Dataset() + dataset.flush() + self.dataset = dataset + self.parent_id = parent_id self.validation_errors = validation_errors - + @property def ext( self ): return self.extension + @property + def states( self ): + return self.dataset.states + + def get_dataset_state( self ): + return self.dataset.state + def set_dataset_state ( self, state ): + self.dataset.state = state + state = property( get_dataset_state, set_dataset_state ) + def get_file_name( self ): - if self.filename_id is None: - assert self.id is not None, "ID must be set before filename used (commit the object)" - # First try filename directly under file_path - filename = os.path.join( self.file_path, "dataset_%d.dat" % self.id ) - # Only use that filename if it already exists (backward compatibility), - # otherwise construct hashed path - if not os.path.exists( filename ): - dir = os.path.join( self.file_path, *directory_hash_id( self.id ) ) - # Create directory if it does not exist - try: - os.makedirs( dir ) - except OSError, e: - # File Exists is okay, otherwise reraise - if e.errno != errno.EEXIST: - raise - # Return filename inside hashed directory - return os.path.abspath( os.path.join( dir, "dataset_%d.dat" % self.id ) ) - else: - filename = self.dataset_file.filename - # Make filename absolute - return os.path.abspath( filename ) + return self.dataset.get_file_name() def set_file_name (self, filename): - if filename is None: - self.filename_id = None - else: - filename_obj = DatasetFileName.get_by(filename=filename) - if filename_obj is None: - filename_obj = DatasetFileName(filename=filename, extra_files_path=self.extra_files_path) - filename_obj.flush() - self.filename_id = filename_obj.id - self.flush() - self.refresh() + return self.dataset.set_file_name( filename ) file_name = property( get_file_name, set_file_name ) @property def extra_files_path( self ): - if self.dataset_file and self.dataset_file.extra_files_path: - path = self.dataset_file.extra_files_path - else: - path = os.path.join( self.file_path, "dataset_%d_files" % self.id ) - #only use path directly under self.file_path if it exists - if not os.path.exists( path ): - path = os.path.join( os.path.join( self.file_path, *directory_hash_id( self.id ) ), "dataset_%d_files" % self.id ) - # Make path absolute - return os.path.abspath( path ) + return self.dataset.extra_files_path @property def datatype( self ): @@ -279,7 +171,7 @@ class Dataset( object ): def get_dbkey( self ): dbkey = self.metadata.dbkey if not isinstance(dbkey, list): dbkey = [dbkey] - if dbkey in [["?"], [None], []]: dbkey = [self.old_dbkey] + #if dbkey in [["?"], [None], []]: dbkey = [self.old_dbkey] if dbkey in [[None], []]: return "?" return dbkey[0] def set_dbkey( self, value ): @@ -288,10 +180,10 @@ class Dataset( object ): self.metadata.dbkey = [value] else: self.metadata.dbkey = value - if isinstance(value, list): - self.old_dbkey = value[0] - else: - self.old_dbkey = value + #if isinstance(value, list): + # self.old_dbkey = value[0] + #else: + # self.old_dbkey = value dbkey = property( get_dbkey, set_dbkey ) def change_datatype( self, new_ext ): @@ -299,22 +191,13 @@ class Dataset( object ): datatypes_registry.change_datatype( self, new_ext ) def get_size( self ): """Returns the size of the data on disk""" - if self.file_size: - return self.file_size - else: - try: - return os.path.getsize( self.file_name ) - except OSError: - return 0 + return self.dataset.get_size() def set_size( self ): """Returns the size of the data on disk""" - try: - self.file_size = os.path.getsize( self.file_name ) - except OSError: - self.file_size = 0 + return self.dataset.set_size() def has_data( self ): """Detects whether there is any data""" - return self.get_size() > 0 + return self.dataset.has_data() def get_raw_data( self ): """Returns the full data. To stream it open the file_name and read/write as needed""" return self.datatype.get_raw_data( self ) @@ -346,55 +229,33 @@ class Dataset( object ): return self.datatype.display_name( self ) def display_info( self ): return self.datatype.display_info( self ) - def get_associated_files_by_type( self, file_type ): + def get_converted_files_by_type( self, file_type ): valid = [] - for assoc in self.associated_files: + for assoc in self.implicitly_converted_datasets: if not assoc.deleted and assoc.type == file_type: - valid.append( assoc ) + valid.append( assoc.dataset ) return valid def clear_associated_files( self, metadata_safe = False, purge = False ): #metadata_safe = True means to only clear when assoc.metadata_safe == False - for assoc in self.associated_files: + for assoc in self.implicitly_converted_datasets: if not metadata_safe or not assoc.metadata_safe: assoc.clear( purge = purge ) def get_child_by_designation(self, designation): - # if self.history: - # for data in self.history.datasets: - # if data.parent_id and data.parent_id == self.id: - # if designation == data.designation: - # return data - for child_association in self.children: - if child_association.designation == designation: - return child_association.child + for child in self.children: + if child.designation == designation: + return child return None + def get_converter_types(self): return self.datatype.get_converter_types( self, datatypes_registry) - def copy(self, parent_id=None): - des = Dataset(extension=self.ext) + def copy( self, copy_children = False, parent_id = None ): + des = HistoryDatasetAssociation( hid=self.hid, name=self.name, info=self.info, blurb=self.blurb, peek=self.peek, extension=self.extension, dbkey=self.dbkey, metadata=self._metadata, dataset = self.dataset, visible=self.visible, deleted=self.deleted, parent_id=parent_id ) des.flush() - des.name = self.name - des.info = self.info - des.blurb = self.blurb - des.peek = self.peek - des.extension = self.extension - des.dbkey = str( self.dbkey ) - des.state = self.state - des.metadata = self.metadata - des.hid = self.hid - des.deleted = self.deleted - des.purged = self.purged - # Make sure source is using filename table, so purge works properly - if not self.dataset_file: - self.set_file_name(self.file_name) - self.flush() - self.refresh() - self.dataset_file.extra_files_path = self.extra_files_path - self.flush() - # Don't copy file contents, share original file - des.file_name = self.file_name - des.hid = self.hid - des.designation = self.designation + if copy_children: + for child in self.children: + child_copy = child.copy( copy_children = copy_children, parent_id = des.id ) + des.set_peek() #in some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs des.flush() return des @@ -407,9 +268,166 @@ class Dataset( object ): def mark_deleted( self, include_children=True ): self.deleted = True if include_children: - for child_assoc in self.children: - child_assoc.child.mark_deleted() + for child in self.children: + child.mark_deleted() + + + +class History( object ): + def __init__( self, id=None, name=None, user=None ): + self.id = id + self.name = name or "Unnamed history" + self.deleted = False + self.purged = False + self.genome_build = None + # Relationships + self.user = user + self.datasets = [] + self.galaxy_sessions = [] + + def _next_hid( self ): + # TODO: override this with something in the database that ensures + # better integrity + if len( self.datasets ) == 0: + return 1 + else: + last_hid = 0 + for dataset in self.datasets: + if dataset.hid > last_hid: + last_hid = dataset.hid + return last_hid + 1 + + def add_galaxy_session( self, galaxy_session, association=None ): + if association is None: + self.galaxy_sessions.append( GalaxySessionToHistoryAssociation( galaxy_session, self ) ) + else: + self.galaxy_sessions.append( association ) + + def add_dataset( self, dataset, parent_id=None, genome_build=None, set_hid = True ): + if isinstance( dataset, Dataset ): + dataset = HistoryDatasetAssociation( dataset = dataset ) + dataset.flush() + elif not isinstance( dataset, HistoryDatasetAssociation ): + raise TypeError, "You can only add Dataset and HistoryDatasetAssociation instances to a history." + if parent_id: + for data in self.datasets: + if data.id == parent_id: + dataset.hid = data.hid + break + else: + if set_hid: dataset.hid = self._next_hid() + else: + if set_hid: dataset.hid = self._next_hid() + dataset.history = self + if genome_build not in [None, '?']: + self.genome_build = genome_build + self.datasets.append( dataset ) + + def copy(self): + des = History() + des.flush() + des.name = self.name + des.user_id = self.user_id + for data in self.datasets: + new_data = data.copy( copy_children = True ) + des.add_dataset( new_data ) + new_data.flush() + des.hid_counter = self.hid_counter + des.flush() + return des + +# class Query( object ): +# def __init__( self, name=None, state=None, tool_parameters=None, history=None ): +# self.name = name or "Unnamed query" +# self.state = state +# self.tool_parameters = tool_parameters +# # Relationships +# self.history = history +# self.datasets = [] + +class Dataset( object ): + states = Bunch( NEW = 'new', + QUEUED = 'queued', + RUNNING = 'running', + OK = 'ok', + EMPTY = 'empty', + ERROR = 'error', + DISCARDED = 'discarded' ) + file_path = "/tmp/" + engine = None + def __init__( self, id=None, state=None, external_filename=None, extra_files_path=None, file_size=None, purgable=True ): + self.id = id + self.state = state + self.deleted = False + self.purged = False + self.purgable = purgable + self.external_filename = external_filename + self._extra_files_path = extra_files_path + self.file_size = file_size + + def get_file_name( self ): + if not self.external_filename: + assert self.id is not None, "ID must be set before filename used (commit the object)" + # First try filename directly under file_path + filename = os.path.join( self.file_path, "dataset_%d.dat" % self.id ) + # Only use that filename if it already exists (backward compatibility), + # otherwise construct hashed path + if not os.path.exists( filename ): + dir = os.path.join( self.file_path, *directory_hash_id( self.id ) ) + # Create directory if it does not exist + try: + os.makedirs( dir ) + except OSError, e: + # File Exists is okay, otherwise reraise + if e.errno != errno.EEXIST: + raise + # Return filename inside hashed directory + return os.path.abspath( os.path.join( dir, "dataset_%d.dat" % self.id ) ) + else: + filename = self.external_filename + # Make filename absolute + return os.path.abspath( filename ) + def set_file_name ( self, filename ): + if not filename: + self.external_filename = None + else: + self.external_filename = filename + + file_name = property( get_file_name, set_file_name ) + + @property + def extra_files_path( self ): + if self._extra_files_path: + path = self._extra_files_path + else: + path = os.path.join( self.file_path, "dataset_%d_files" % self.id ) + #only use path directly under self.file_path if it exists + if not os.path.exists( path ): + path = os.path.join( os.path.join( self.file_path, *directory_hash_id( self.id ) ), "dataset_%d_files" % self.id ) + # Make path absolute + return os.path.abspath( path ) + + def get_size( self ): + """Returns the size of the data on disk""" + if self.file_size: + return self.file_size + else: + try: + return os.path.getsize( self.file_name ) + except OSError: + return 0 + def set_size( self ): + """Returns the size of the data on disk""" + try: + self.file_size = os.path.getsize( self.file_name ) + except OSError: + self.file_size = 0 + def has_data( self ): + """Detects whether there is any data""" + return self.get_size() > 0 + def mark_deleted( self, include_children=True ): + self.deleted = True # FIXME: sqlalchemy will replace this def _delete(self): @@ -419,12 +437,6 @@ class Dataset( object ): except OSError, e: log.critical('%s delete error %s' % (self.__class__.__name__, e)) -class DatasetFileName( object ): - def __init__( self, filename=None, readonly=False, extra_files_path=None ): - self.filename = filename - self.readonly = readonly - self.extra_files_path = extra_files_path - class Old_Dataset( Dataset ): pass @@ -439,47 +451,22 @@ class DatasetToValidationErrorAssociation( object ): self.dataset = dataset self.validation_error = validation_error -class DatasetChildAssociation( object ): - def __init__( self, designation=None ): - self.designation = designation - self.parent = None - self.child = None - -class DatasetAssociatedFile( object ): - def __init__( self, id = None, dataset_id = None, file_type = None, parent_id = None, filename = None, deleted = False, purged = False, metadata_safe = True ): +class ImplicitlyConvertedDatasetAssociation( object ): + def __init__( self, id = None, parent = None, dataset = None, file_type = None, deleted = False, purged = False, metadata_safe = True ): self.id = id - self.dataset_id = dataset_id + self.dataset = dataset + self.parent = parent self.type = file_type - self.parent_id = parent_id - self.filename = filename self.deleted = deleted self.purged = purged self.metadata_safe = metadata_safe - def get_file_name( self ): - #return absolute path of the filename - if self.filename: - return os.path.abspath( self.filename ) - if self.dataset_id is not None: - return self.dataset.file_name - else: - assert self.id is not None, "ID must be set before filename used (commit the object)" - assert self.parent_id is not None, "Parent ID must be set before filename used" - return os.path.abspath( "%s_accociated_%s" % ( self.parent.file_name, self.id ) ) - def set_file_name ( self, filename ): - self.filename = filename - if self.dataset: - self.dataset.deleted = True - self.dataset = None - self.dataset_id = None - file_name = property( get_file_name, set_file_name ) - def clear( self, purge = False ): self.deleted = True if self.dataset: self.dataset.deleted = True self.dataset.purged = purge - if purge: + if purge: #do something with purging self.purged = True try: os.unlink( self.file_name ) except Exception, e: print "Failed to purge associated file (%s) from disk: %s" % ( self.file_name, e ) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 4b2011aec51..27d8d4342f6 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -63,62 +63,54 @@ History.table = Table( "history", metadata, # Column( "state", String( 64 ) ), # Column( "tool_parameters", Pickle() ) ) -Dataset.table = Table( "dataset", metadata, + +HistoryDatasetAssociation.table = Table( "history_dataset_association", metadata, Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, index=True, default=now, onupdate=now ), - Column( "hid", Integer ), Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ), + Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "hid", Integer ), Column( "name", TrimmedString( 255 ) ), Column( "info", TrimmedString( 255 ) ), Column( "blurb", TrimmedString( 255 ) ), Column( "peek" , TEXT ), Column( "extension", TrimmedString( 64 ) ), - Column( "dbkey", TrimmedString( 64 ), key="old_dbkey" ), # maps to old_dbkey, see __init__.py - Column( "state", TrimmedString( 64 ) ), Column( "metadata", MetadataType(), key="_metadata" ), - Column( "parent_id", Integer, nullable=True ), + Column( "parent_id", Integer, ForeignKey( "history_dataset_association.id" ), nullable=True ), Column( "designation", TrimmedString( 255 ) ), Column( "deleted", Boolean, index=True, default=False ), - Column( "purged", Boolean, index=True, default=False ), - Column( "visible", Boolean ), - Column( "filename_id", Integer, ForeignKey( "dataset_filename.id" ), index=True, nullable=True ), - Column( 'file_size', Numeric( 15, 0 ) ), - ForeignKeyConstraint(['parent_id'],['dataset.id'], ondelete="CASCADE") ) + Column( "visible", Boolean ) ) -DatasetAssociatedFile.table = Table( "dataset_associated_file", metadata, +Dataset.table = Table( "dataset", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, index=True, default=now, onupdate=now ), + Column( "state", TrimmedString( 64 ) ), + Column( "deleted", Boolean, index=True, default=False ), + Column( "purged", Boolean, index=True, default=False ), + Column( "purgable", Boolean, default=True ), + Column( "external_filename" , TEXT ), + Column( "_extra_files_path", TEXT ), + Column( 'file_size', Numeric( 15, 0 ) ) ) + +ImplicitlyConvertedDatasetAssociation.table = Table( "implicitly_converted_dataset_association", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True, nullable=True ), - Column( "parent_id", Integer, ForeignKey( "dataset.id" ), index=True ), - Column( "filename", TEXT ), + Column( "hda_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True, nullable=True ), + Column( "hda_parent_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ), Column( "deleted", Boolean, index=True, default=False ), - Column( "purged", Boolean, index=True, default=False ), Column( "metadata_safe", Boolean, index=True, default=True ), Column( "type", TrimmedString( 255 ) ) ) -DatasetFileName.table = Table( "dataset_filename", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "filename", TEXT ), - Column( "extra_files_path", TEXT, nullable=True, default=None ), - Column( "readonly", Boolean, default=False ) ) - ValidationError.table = Table( "validation_error", metadata, Column( "id", Integer, primary_key=True ), - Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ), + Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ), Column( "message", TrimmedString( 255 ) ), Column( "err_type", TrimmedString( 64 ) ), Column( "attributes", TEXT ) ) -DatasetChildAssociation.table = Table( "dataset_child_association", metadata, - Column( "id", Integer, primary_key=True ), - Column( "parent_dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ), - Column( "child_dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ), - Column( "designation", TrimmedString( 255 ) ) ) - Job.table = Table( "job", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -147,13 +139,13 @@ JobParameter.table = Table( "job_parameter", metadata, JobToInputDatasetAssociation.table = Table( "job_to_input_dataset", metadata, Column( "id", Integer, primary_key=True ), Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), - Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ), + Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ), Column( "name", String(255) ) ) JobToOutputDatasetAssociation.table = Table( "job_to_output_dataset", metadata, Column( "id", Integer, primary_key=True ), Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), - Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ), + Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ), Column( "name", String(255) ) ) Event.table = Table( "event", metadata, @@ -236,57 +228,59 @@ WorkflowStepConnection.table = Table( "workflow_step_connection", metadata, assign_mapper( context, ValidationError, ValidationError.table ) -# assign_mapper( context, Dataset, Dataset.table, -# properties=dict( children=relation( DatasetChildAssociation, primaryjoin=( DatasetChildAssociation.table.c.parent_dataset_id == Dataset.table.c.id ), -# lazy=False ), -# validation_errors=relation( ValidationError, lazy=False ) ) ) - -assign_mapper( context, Dataset, Dataset.table, +assign_mapper( context, HistoryDatasetAssociation, HistoryDatasetAssociation.table, properties=dict( + dataset=relation( + Dataset, + primaryjoin=( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) ), + history=relation( + History, + primaryjoin=( History.table.c.id == HistoryDatasetAssociation.table.c.history_id ) ), + implicitly_converted_datasets=relation( + ImplicitlyConvertedDatasetAssociation, + primaryjoin=( ImplicitlyConvertedDatasetAssociation.table.c.hda_parent_id == HistoryDatasetAssociation.table.c.id ) ), children=relation( - DatasetChildAssociation, - primaryjoin=( DatasetChildAssociation.table.c.parent_dataset_id == Dataset.table.c.id ), - lazy=False, - backref="parent" ), - dataset_file=relation( - DatasetFileName, - primaryjoin=( DatasetFileName.table.c.id == Dataset.table.c.filename_id ) ), - associated_files=relation( - DatasetAssociatedFile, - primaryjoin=( DatasetAssociatedFile.table.c.parent_id == Dataset.table.c.id ) ) + HistoryDatasetAssociation, + primaryjoin=( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ), + backref=backref( "parent", remote_side=[HistoryDatasetAssociation.table.c.id] ) ) ) ) -assign_mapper( context, DatasetFileName, DatasetFileName.table ) +assign_mapper( context, Dataset, Dataset.table, + properties=dict( + history_associations=relation( + HistoryDatasetAssociation, + primaryjoin=( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) ) + ) ) -assign_mapper( context, DatasetChildAssociation, DatasetChildAssociation.table, - properties=dict( child=relation( Dataset, backref="parent", primaryjoin=( DatasetChildAssociation.table.c.child_dataset_id == Dataset.table.c.id ) ) ) ) - -assign_mapper( context, DatasetAssociatedFile, DatasetAssociatedFile.table, - properties=dict( parent=relation( - Dataset, - primaryjoin=( DatasetAssociatedFile.table.c.parent_id == Dataset.table.c.id ) ), - - dataset=relation( - Dataset, - primaryjoin=( DatasetAssociatedFile.table.c.dataset_id == Dataset.table.c.id ) ) ) ) # assign_mapper( model.Query, model.Query.table, # properties=dict( datasets=relation( model.Dataset.mapper, backref="query") ) ) + +assign_mapper( context, ImplicitlyConvertedDatasetAssociation, ImplicitlyConvertedDatasetAssociation.table, + properties=dict( parent=relation( + HistoryDatasetAssociation, + primaryjoin=( ImplicitlyConvertedDatasetAssociation.table.c.hda_parent_id == HistoryDatasetAssociation.table.c.id ) ), + + dataset=relation( + HistoryDatasetAssociation, + primaryjoin=( ImplicitlyConvertedDatasetAssociation.table.c.hda_id == HistoryDatasetAssociation.table.c.id ) ) ) ) + assign_mapper( context, History, History.table, properties=dict( galaxy_sessions=relation( GalaxySessionToHistoryAssociation ), - datasets=relation( Dataset, backref="history", order_by=asc(Dataset.table.c.hid) ), - active_datasets=relation( Dataset, primaryjoin=( ( Dataset.c.history_id == History.table.c.id ) & ( not_( Dataset.c.deleted ) ) ), order_by=asc( Dataset.table.c.hid ), lazy=False, viewonly=True ) ) ) + datasets=relation( HistoryDatasetAssociation, backref="history", order_by=asc(HistoryDatasetAssociation.table.c.hid) ), + active_datasets=relation( HistoryDatasetAssociation, primaryjoin=( ( HistoryDatasetAssociation.table.c.history_id == History.table.c.id ) & ( not_( HistoryDatasetAssociation.table.c.deleted ) ) ), order_by=asc( HistoryDatasetAssociation.table.c.hid ), lazy=False, viewonly=True ) ) ) + assign_mapper( context, User, User.table, properties=dict( histories=relation( History, backref="user", order_by=desc(History.table.c.update_time) ) ) ) assign_mapper( context, JobToInputDatasetAssociation, JobToInputDatasetAssociation.table, - properties=dict( job=relation( Job ), dataset=relation( Dataset ) ) ) + properties=dict( job=relation( Job ), dataset=relation( HistoryDatasetAssociation ) ) ) assign_mapper( context, JobToOutputDatasetAssociation, JobToOutputDatasetAssociation.table, - properties=dict( job=relation( Job ), dataset=relation( Dataset ) ) ) + properties=dict( job=relation( Job ), dataset=relation( HistoryDatasetAssociation ) ) ) assign_mapper( context, JobParameter, JobParameter.table ) @@ -310,7 +304,7 @@ assign_mapper( context, GalaxySessionToHistoryAssociation, GalaxySessionToHistor properties=dict( galaxy_session=relation( GalaxySession ), history=relation( History ) ) ) -Dataset.mapper.add_property( "creating_job_associations", relation( JobToOutputDatasetAssociation ) ) +HistoryDatasetAssociation.mapper.add_property( "creating_job_associations", relation( JobToOutputDatasetAssociation ) ) assign_mapper( context, Workflow, Workflow.table, properties=dict( steps=relation( WorkflowStep, backref='workflow', order_by=asc(WorkflowStep.table.c.order_index), cascade="all, delete-orphan" ) ) ) diff --git a/lib/galaxy/model/mapping_tests.py b/lib/galaxy/model/mapping_tests.py index b267bcf1195..e040c59e393 100644 --- a/lib/galaxy/model/mapping_tests.py +++ b/lib/galaxy/model/mapping_tests.py @@ -16,7 +16,7 @@ class MappingTests( unittest.TestCase ): #h1.queries.append( model.Query( "h1->q2" ) ) h2 = model.History( name=( "H" * 1024 ) ) #q1 = model.Query( "h2->q1" ) - d1 = model.Dataset( metadata=dict(chromCol=1,startCol=2,endCol=3 ), history=h2 ) + d1 = model.HistoryDatasetAssociation( metadata=dict(chromCol=1,startCol=2,endCol=3 ), history=h2, create_dataset=True ) #h2.queries.append( q1 ) #h2.queries.append( model.Query( "h2->q2" ) ) model.context.current.flush() diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index d7ad1e9fd6b..2e3d3dda6b0 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -970,20 +970,15 @@ class Tool: for name, data in input_datasets.items(): param_dict[name] = DatasetFilenameWrapper( data, datatypes_registry = self.app.datatypes_registry, tool = self, name = name ) if data: - for child_association in data.children: - child = child_association.child - key = "_CHILD___%s___%s" % ( name, child.designation ) - param_dict[ key ] = DatasetFilenameWrapper( child ) + for child in data.children: + param_dict[ "_CHILD___%s___%s" % ( name, child.designation ) ] = DatasetFilenameWrapper( child ) for name, data in output_datasets.items(): param_dict[name] = DatasetFilenameWrapper( data ) # Provide access to a path to store additional files # TODO: path munging for cluster/dataset server relocatability param_dict[name].files_path = os.path.abspath(os.path.join(self.app.config.new_file_path, "dataset_%s_files" % (data.id) )) - - for child_association in data.children: - child = child_association.child - key = "_CHILD___%s___%s" % ( name, child.designation ) - param_dict[ key ] = DatasetFilenameWrapper( child ) + for child in data.children: + param_dict[ "_CHILD___%s___%s" % ( name, child.designation ) ] = DatasetFilenameWrapper( child ) # We add access to app here, this allows access to app.config, etc param_dict['__app__'] = RawObjectWrapper( self.app ) # More convienent access to app.config.new_file_path; we don't need to wrap a string @@ -1092,25 +1087,24 @@ class Tool: if visible == "visible": visible = True else: visible = False ext = fields.pop(0).lower() - # Create new child dataset - child_data = self.app.model.Dataset(extension=ext, parent_id=parent_id, designation=designation, visible=visible, dbkey=outdata.dbkey) - child_data.flush() + child_dataset = self.app.model.HistoryDatasetAssociation( extension=ext, parent_id=outdata.id, designation=designation, visible=visible, dbkey=outdata.dbkey, create_dataset=True ) # Move data from temp location to dataset location - shutil.move(filename, child_data.file_name) - child_data.name = "Secondary Dataset (%s)" % (designation) - child_data.state = child_data.states.OK - child_data.init_meta() - child_data.set_meta() - child_data.set_peek() - child_data.set_size() - child_data.flush() - # Add to child accociation table - assoc = self.app.model.DatasetChildAssociation() - assoc.child = child_data - assoc.designation = child_data.designation - outdata.children.append( assoc ) + shutil.move( filename, child_dataset.file_name ) + child_dataset.flush() + child_dataset.name = "Secondary Dataset (%s)" % ( designation ) + child_dataset.state = child_dataset.states.OK + child_dataset.init_meta() + child_dataset.set_meta() + child_dataset.set_peek() + child_dataset.set_size() + child_dataset.flush() # Add child to return dict - children[name][designation] = child_data + children[name][designation] = child_dataset + for dataset in outdata.dataset.history_associations: #need to update all associated output hdas, i.e. history was shared with job running + if outdata == dataset: continue + # Create new child dataset + child_data = child_dataset.copy( parent_id = dataset.id ) + child_data.flush() return children def collect_primary_datasets( self, output): @@ -1129,20 +1123,25 @@ class Tool: else: visible = False ext = fields.pop(0).lower() # Create new primary dataset - primary_data = self.app.model.Dataset(extension=ext, designation=designation, visible=visible, dbkey=outdata.dbkey) + primary_data = self.app.model.HistoryDatasetAssociation( extension=ext, designation=designation, visible=visible, dbkey=outdata.dbkey, create_dataset=True ) primary_data.flush() - self.app.model.History.get(outdata.history_id).add_dataset(primary_data) # Move data from temp location to dataset location - shutil.move(filename, primary_data.file_name) - primary_data.name = outdata.name - primary_data.info = outdata.info + shutil.move( filename, primary_data.file_name ) + primary_data.name = dataset.name + primary_data.info = dataset.info primary_data.state = primary_data.states.OK - primary_data.init_meta(copy_from=outdata) + primary_data.init_meta( copy_from=dataset ) primary_data.set_peek() primary_data.set_size() primary_data.flush() + outdata.history.add_dataset( primary_data ) # Add dataset to return dict primary_datasets[name][designation] = primary_data + for dataset in outdata.dataset.history_associations: #need to update all associated output hdas, i.e. history was shared with job running + if outdata == dataset: continue + new_data = primary_data.copy() + dataset.history.add( new_data ) + new_data.flush() return primary_datasets diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index ca0ba568354..595997b9719 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -30,17 +30,18 @@ class DefaultToolAction( object ): for target_ext in input.extensions: if target_ext in data.get_converter_types(): data.refresh() #need to refresh incase this conversion just took place, i.e. input above in tool performed the same conversion - assoc = data.get_associated_files_by_type( "CONVERTED_%s" % target_ext ) - if assoc: data = assoc[0].dataset + datasets = data.get_converted_files_by_type( target_ext ) + if datasets: data = datasets[0] elif input.converter_safe( param_values, trans ): #run converter here - assoc = trans.app.model.DatasetAssociatedFile( parent_id = data.id, file_type = "CONVERTED_%s" % target_ext, metadata_safe = False ) + assoc = trans.app.model.ImplicitlyConvertedDatasetAssociation( parent = data, file_type = target_ext, metadata_safe = False ) new_data = data.datatype.convert_dataset( trans, data, target_ext, return_output = True, visible = False ).values()[0] new_data.hid = data.hid new_data.name = data.name - assoc.dataset_id = new_data.id + new_data.flush() + assoc.dataset = new_data + assoc.flush() data = new_data - data.flush() break return data if isinstance( input, DataToolParameter ): @@ -122,19 +123,20 @@ class DefaultToolAction( object ): ## What is the following hack for? Need to document under what ## conditions can the following occur? (james@bx.psu.edu) # HACK: the output data has already been created + # this happens i.e. as a result of the async controller if name in incoming: dataid = incoming[name] - data = trans.app.model.Dataset.get( dataid ) + data = trans.app.model.HistoryDatasetAssociation.get( dataid ) assert data != None out_data[name] = data - continue - # the type should match the input - ext = output.format - if ext == "input": - ext = input_ext - data = trans.app.model.Dataset(extension=ext) - # Commit the dataset immediately so it gets database assigned unique id - data.flush() + else: + # the type should match the input + ext = output.format + if ext == "input": + ext = input_ext + data = trans.app.model.HistoryDatasetAssociation( extension=ext, create_dataset=True ) + # Commit the dataset immediately so it gets database assigned unique id + data.flush() # Create an empty file immediately open( data.file_name, "w" ).close() # This may not be neccesary with the new parent/child associations @@ -167,7 +169,7 @@ class DefaultToolAction( object ): # Add all the top-level (non-child) datasets to the history for name in out_data.keys(): - if name not in child_dataset_names: + if name not in child_dataset_names and name not in incoming: #don't add children; or already existing datasets, i.e. async created data = out_data[ name ] trans.history.add_dataset( data, set_hid = set_output_hid ) data.flush() @@ -176,11 +178,7 @@ class DefaultToolAction( object ): for parent_name, child_name in parent_to_child_pairs: parent_dataset = out_data[ parent_name ] child_dataset = out_data[ child_name ] - assoc = trans.app.model.DatasetChildAssociation() - assoc.child = child_dataset - assoc.designation = child_dataset.designation - parent_dataset.children.append( assoc ) - # FIXME: Child dataset hid + parent_dataset.children.append( child_dataset ) # Store data after custom code runs trans.app.model.flush() diff --git a/lib/galaxy/tools/actions/upload.py b/lib/galaxy/tools/actions/upload.py index 32628b13682..b3904436a63 100644 --- a/lib/galaxy/tools/actions/upload.py +++ b/lib/galaxy/tools/actions/upload.py @@ -65,7 +65,7 @@ class UploadToolAction( object ): return dict( output=data_list[0] ) def upload_empty(self, trans, err_code, err_msg): - data = trans.app.model.Dataset() + data = trans.app.model.HistoryDatasetAssociation( create_dataset = True ) data.name = err_code data.extension = "txt" data.dbkey = "?" @@ -158,13 +158,12 @@ class UploadToolAction( object ): if info is None: info = 'uploaded %s file' %data_type - data = trans.app.model.Dataset() + data = trans.app.model.HistoryDatasetAssociation( history = trans.history, extension = ext, create_dataset = True ) data.name = file_name - data.extension = ext data.dbkey = dbkey data.info = info data.flush() - shutil.move(temp_name, data.file_name) + shutil.move( temp_name, data.file_name ) data.state = data.states.OK data.init_meta() if self.line_count is not None: diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 79bffb4d46f..acba4fbafe0 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -636,11 +636,11 @@ class ColumnListParameter( SelectToolParameter ): # from a twill perspective... >>> # Mock up a history (not connected to database) - >>> from galaxy.model import History, Dataset + >>> from galaxy.model import History, HistoryDatasetAssociation >>> from galaxy.util.bunch import Bunch >>> hist = History() >>> hist.flush() - >>> hist.add_dataset( Dataset( id=1, extension='interval' ) ) + >>> hist.add_dataset( HistoryDatasetAssociation( id=1, extension='interval', create_dataset=True ) ) >>> dtp = DataToolParameter( None, XML( '' ) ) >>> print dtp.name blah @@ -979,15 +979,15 @@ class DataToolParameter( ToolParameter ): displayed as radio buttons and multiple selects as a set of checkboxes >>> # Mock up a history (not connected to database) - >>> from galaxy.model import History, Dataset + >>> from galaxy.model import History, HistoryDatasetAssociation >>> from galaxy.util.bunch import Bunch >>> hist = History() >>> hist.flush() - >>> hist.add_dataset( Dataset( id=1, extension='txt' ) ) - >>> hist.add_dataset( Dataset( id=2, extension='bed' ) ) - >>> hist.add_dataset( Dataset( id=3, extension='fasta' ) ) - >>> hist.add_dataset( Dataset( id=4, extension='png' ) ) - >>> hist.add_dataset( Dataset( id=5, extension='interval' ) ) + >>> hist.add_dataset( HistoryDatasetAssociation( id=1, extension='txt', create_dataset=True ) ) + >>> hist.add_dataset( HistoryDatasetAssociation( id=2, extension='bed', create_dataset=True ) ) + >>> hist.add_dataset( HistoryDatasetAssociation( id=3, extension='fasta', create_dataset=True ) ) + >>> hist.add_dataset( HistoryDatasetAssociation( id=4, extension='png', create_dataset=True ) ) + >>> hist.add_dataset( HistoryDatasetAssociation( id=5, extension='interval', create_dataset=True ) ) >>> p = DataToolParameter( None, XML( '' ) ) >>> print p.name blah @@ -1056,16 +1056,16 @@ class DataToolParameter( ToolParameter ): else: for target_ext in self.extensions: if target_ext in data.get_converter_types(): - assoc = data.get_associated_files_by_type( "CONVERTED_%s" % target_ext ) - if assoc: - data = assoc[0].dataset + datasets = data.get_converted_files_by_type( target_ext ) + if datasets: + data = datasets[0] elif not self.converter_safe( other_values, trans ): continue selected = ( value and ( data in value ) ) field.add_option( "%s: (as %s) %s" % ( hid, target_ext, data.name[:30] ), data.id, selected ) break #we only report the first valid converter, assume self.extensions is a priority list # Also collect children via association object - dataset_collector( [ assoc.child for assoc in data.children ], hid ) + dataset_collector( data.children, hid ) dataset_collector( history.datasets, None ) some_data = bool( field.options ) if some_data: @@ -1116,7 +1116,7 @@ class DataToolParameter( ToolParameter ): continue most_recent_dataset[0] = data # Also collect children via association object - dataset_collector( [ assoc.child for assoc in data.children ] ) + dataset_collector( data.children ) dataset_collector( history.datasets ) most_recent_dataset = most_recent_dataset.pop() if most_recent_dataset is not None: @@ -1133,11 +1133,11 @@ class DataToolParameter( ToolParameter ): if value in [None, "None"]: return None if isinstance( value, list ): - return [ trans.app.model.Dataset.get( v ) for v in value ] - elif isinstance( value, trans.app.model.Dataset ): + return [ trans.app.model.HistoryDatasetAssociation.get( v ) for v in value ] + elif isinstance( value, trans.app.model.HistoryDatasetAssociation ): return value else: - return trans.app.model.Dataset.get( value ) + return trans.app.model.HistoryDatasetAssociation.get( value ) def value_to_basic( self, value, app ): if value is None or isinstance( value, str ): @@ -1152,7 +1152,7 @@ class DataToolParameter( ToolParameter ): if value is None or value == '' or value == 'None': return value try: - return app.model.Dataset.get( int( value ) ) + return app.model.HistoryDatasetAssociation.get( int( value ) ) except: if ignore_errors: return value @@ -1221,7 +1221,7 @@ class DataToolParameter( ToolParameter ): # have the history accessable at the job level, it is necessary # I also probably wrote this docstring test thing wrong. # -# >>> from galaxy.model import History, Dataset +# >>> from galaxy.model import History # >>> from galaxy.util.bunch import Bunch # >>> hist = History( id=1 ) # >>> p = HistoryIDParameter( None, XML( '' ) ) diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py index 801cb89e5d6..40fa0fb135f 100644 --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -104,7 +104,7 @@ class DataMetaFilter( Filter ): return file_value == dataset_value assert self.ref_name in other_values or trans.workflow_building_mode, "Required dependency '%s' not found in incoming values" % self.ref_name ref = other_values.get( self.ref_name, None ) - if not isinstance( ref, self.dynamic_option.tool_param.tool.app.model.Dataset ): + if not isinstance( ref, self.dynamic_option.tool_param.tool.app.model.HistoryDatasetAssociation ): return [] #not a valid dataset meta_value = ref.metadata.get( self.key, None ) assert meta_value is not None, "Required metadata value '%s' not found in referenced dataset" % self.key diff --git a/lib/galaxy/web/controllers/async.py b/lib/galaxy/web/controllers/async.py index cf5933b205d..529a705385e 100644 --- a/lib/galaxy/web/controllers/async.py +++ b/lib/galaxy/web/controllers/async.py @@ -52,7 +52,7 @@ class ASync( BaseController ): if data_id: if not URL: return "No URL parameter was submitted for data %s" % data_id - data = trans.model.Dataset.get( data_id ) + data = trans.model.HistoryDatasetAssociation.get( data_id ) if not data: return "Data %s does not exist or has already been deleted" % data_id @@ -67,7 +67,8 @@ class ASync( BaseController ): trans.log_event( 'Async executing tool %s' % tool.id, tool_id=tool.id ) galaxy_url = trans.request.base + '/async/%s/%s/%s' % ( tool_id, data.id, key ) galaxy_url = params.get("GALAXY_URL",galaxy_url) - params = dict(url=URL, dataid=data.id, output=data.file_name, GALAXY_URL=galaxy_url) + params = dict( url=URL, GALAXY_URL=galaxy_url ) + params[tool.outputs.keys()[0]] = data.id #assume there is exactly one output file possible #tool.execute( app=self.app, history=history, incoming=params ) tool.execute( trans, incoming=params ) else: @@ -101,14 +102,14 @@ class ASync( BaseController ): #data.dbkey = GALAXY_BUILD #data.state = jobs.JOB_OK #history.datasets.add_dataset( data ) - - data = trans.app.model.Dataset() + data = trans.app.model.HistoryDatasetAssociation( create_dataset = True, extension = GALAXY_TYPE ) data.name = GALAXY_NAME - data.extension = GALAXY_TYPE data.dbkey = GALAXY_BUILD data.info = GALAXY_INFO data.state = data.states.NEW + data.flush() + open( data.file_name, 'wb' ).close() #create the file trans.history.add_dataset( data, genome_build=GALAXY_BUILD ) trans.model.flush() trans.log_event( "Added dataset %d to history %d" %(data.id, trans.history.id ), tool_id=tool_id ) diff --git a/lib/galaxy/web/controllers/dataset.py b/lib/galaxy/web/controllers/dataset.py index 344538fccc7..f86b648b022 100644 --- a/lib/galaxy/web/controllers/dataset.py +++ b/lib/galaxy/web/controllers/dataset.py @@ -47,12 +47,12 @@ class DatasetInterface( BaseController ): @web.expose def errors( self, trans, id ): - dataset = model.Dataset.get( id ) + dataset = model.HistoryDatasetAssociation.get( id ) return trans.fill_template( "dataset/errors.tmpl", dataset=dataset ) @web.expose def stderr( self, trans, id ): - dataset = model.Dataset.get( id ) + dataset = model.HistoryDatasetAssociation.get( id ) job = dataset.creating_job_associations[0].job trans.response.set_content_type( 'text/plain' ) return job.stderr @@ -66,7 +66,7 @@ class DatasetInterface( BaseController ): if to_address is None: return trans.show_error_message( "Sorry, error reporting has been disabled for this galaxy instance" ) # Get the dataset and associated job - dataset = model.Dataset.get( id ) + dataset = model.HistoryDatasetAssociation.get( id ) job = dataset.creating_job_associations[0].job # Build the email message msg = MIMEText( string.Template( error_report_template ) @@ -105,7 +105,7 @@ class DatasetInterface( BaseController ): """Catches the dataset id and displays file contents as directed""" if filename is None or filename.lower() == "index": try: - data = trans.app.model.Dataset.get( dataset_id ) + data = trans.app.model.HistoryDatasetAssociation.get( dataset_id ) if data: mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() ) trans.response.set_content_type(mime) @@ -120,7 +120,7 @@ class DatasetInterface( BaseController ): else: #display files from directory here try: - file_path = os.path.join(trans.app.model.Dataset.get( dataset_id ).extra_files_path, filename) + file_path = os.path.join(trans.app.model.HistoryDatasetAssociation.get( dataset_id ).extra_files_path, filename) mime, encoding = mimetypes.guess_type(file_path) if mime is None: mime = trans.app.datatypes_registry.get_mimetype_by_extension(".".split(file_path)[-1]) diff --git a/lib/galaxy/web/controllers/root.py b/lib/galaxy/web/controllers/root.py index 7011e69ae26..8c1b48b0642 100644 --- a/lib/galaxy/web/controllers/root.py +++ b/lib/galaxy/web/controllers/root.py @@ -67,7 +67,7 @@ class RootController( BaseController ): def dataset_state ( self, trans, id=None, stamp=None ): if id is not None: try: - data = self.app.model.Dataset.get( id ) + data = self.app.model.HistoryDatasetAssociation.get( id ) except: return trans.show_error_message( "Unable to check dataset %s." %str( id ) ) trans.response.headers['X-Dataset-State'] = data.state @@ -81,7 +81,7 @@ class RootController( BaseController ): def dataset_code( self, trans, id=None, hid=None, stamp=None ): if id is not None: try: - data = self.app.model.Dataset.get( id ) + data = self.app.model.HistoryDatasetAssociation.get( id ) except: return trans.show_error_message( "Unable to check dataset %s." %str( id ) ) trans.response.headers['Pragma'] = 'no-cache' @@ -101,7 +101,7 @@ class RootController( BaseController ): ids = map( int, ids.split( "," ) ) states = states.split( "," ) for id, state in zip( ids, states ): - data = self.app.model.Dataset.get( id ) + data = self.app.model.HistoryDatasetAssociation.get( id ) if data.state != state: rval[id] = { "state": data.state, @@ -131,7 +131,7 @@ class RootController( BaseController ): raise Exception( "No dataset with hid '%d'" % hid ) else: try: - data = self.app.model.Dataset.get( id ) + data = self.app.model.HistoryDatasetAssociation.get( id ) except: return "Dataset id '%s' is invalid" %str( id ) if data: @@ -160,7 +160,7 @@ class RootController( BaseController ): Returns child data directly into the browser, based upon parent_id and designation. """ try: - data = self.app.model.Dataset.get( parent_id ) + data = self.app.model.HistoryDatasetAssociation.get( parent_id ) if data: child = data.get_child_by_designation(designation) if child: @@ -172,7 +172,7 @@ class RootController( BaseController ): @web.expose def display_as( self, trans, id=None, display_app=None, **kwd ): """Returns a file in a format that can successfully be displayed in display_app""" - data = self.app.model.Dataset.get( id ) + data = self.app.model.HistoryDatasetAssociation.get( id ) if data: trans.response.set_content_type(data.get_mime()) trans.log_event( "Formatted dataset id %s for display at %s" % ( str(id), display_app ) ) @@ -183,7 +183,7 @@ class RootController( BaseController ): @web.expose def peek(self, trans, id=None): """Returns a 'peek' at the data""" - data = self.app.model.Dataset.get( id ) + data = self.app.model.HistoryDatasetAssociation.get( id ) if data: yield "
"
             yield data.peek
@@ -201,7 +201,7 @@ class RootController( BaseController ):
         elif id is None: 
             return trans.show_error_message( "Problem loading dataset id %s with history id %s." % ( str( id ), str( hid ) ) )
         else:
-            data = self.app.model.Dataset.get( id )
+            data = self.app.model.HistoryDatasetAssociation.get( id )
         if data is None:
             return trans.show_error_message( "Problem retrieving dataset id %s with history id %s." % ( str( id ), str( hid ) ) )
 
@@ -281,15 +281,12 @@ class RootController( BaseController ):
                     int( id )
                 except:
                     continue
-                data = self.app.model.Dataset.get( id )
+                data = self.app.model.HistoryDatasetAssociation.get( id )
                 if data:
                     # Walk up parent datasets to find the containing history
                     topmost_parent = data
                     while topmost_parent.parent:
-                        # data.parent is a list of associations, data.parent.parent 
-                        # is the actual dataset
-                        assert len( data.parent ) == 1, "Dataset should only have one parent"
-                        topmost_parent = data.parent[0].parent
+                        topmost_parent = topmost_parent.parent
                     assert topmost_parent in history.datasets, "Data does not belong to current history"
                     # Mark deleted and cleanup
                     data.mark_deleted()
@@ -311,15 +308,12 @@ class RootController( BaseController ):
             except:
                 return "Dataset id '%s' is invalid" %str( id )
             history = trans.get_history()
-            data = self.app.model.Dataset.get( id )
+            data = self.app.model.HistoryDatasetAssociation.get( id )
             if data:
                 # Walk up parent datasets to find the containing history
                 topmost_parent = data
                 while topmost_parent.parent:
-                    # data.parent is a list of associations, data.parent.parent 
-                    # is the actual dataset
-                    assert len( data.parent ) == 1, "Dataset should only have one parent"
-                    topmost_parent = data.parent[0].parent
+                    topmost_parent = topmost_parent.parent
                 assert topmost_parent in history.datasets, "Data does not belong to current history"
                 # Mark deleted and cleanup
                 data.mark_deleted()
@@ -556,7 +550,7 @@ class RootController( BaseController ):
         """Adds a POSTed file to a History"""
         try:
             history = trans.app.model.History.get( history_id )
-            data = trans.app.model.Dataset( name = name, info = info, extension = ext, dbkey = dbkey )
+            data = trans.app.model.HistoryDatasetAssociation( name = name, info = info, extension = ext, dbkey = dbkey, create_file = True )
             data.flush()
             data_file = open( data.file_name, "wb" )
             file_data.file.seek( 0 )
@@ -580,7 +574,7 @@ class RootController( BaseController ):
     def dataset_make_primary( self, trans, id=None):
         """Copies a dataset and makes primary"""
         try:
-            old_data = self.app.model.Dataset.get( id )
+            old_data = self.app.model.HistoryDatasetAssociation.get( id )
             new_data = old_data.copy()
             ## new_data.parent = None
             ## history = trans.app.model.History.get( old_data.history_id )
@@ -606,7 +600,7 @@ class RootController( BaseController ):
     @web.expose
     def dataset_errors( self, trans, id=None, **kwd ):
         """View/fix errors associated with dataset"""
-        data = trans.app.model.Dataset.get( id )
+        data = trans.app.model.HistoryDatasetAssociation.get( id )
         p = kwd
         if p.get("fix_errors", None):
             # launch tool to create new, (hopefully) error free dataset
diff --git a/scripts/cleanup_datasets/cleanup_datasets.py b/scripts/cleanup_datasets/cleanup_datasets.py
index 202a7ed6b53..ea87a5b2814 100644
--- a/scripts/cleanup_datasets/cleanup_datasets.py
+++ b/scripts/cleanup_datasets/cleanup_datasets.py
@@ -263,16 +263,24 @@ def purge_dataset( dataset ):
                     return "# Dataset for deletion ( id %s ) points to a file on disk being shared by another user's history ( dataset id %s )\n" %( str( dataset.id ), str( data.id ) )
         elif dataset.deleted:
             # Remove files from disk and update the database
+            purgable = False
             try:
-                os.unlink( dataset.file_name )
                 dataset.purged = True
                 dataset.file_size = 0
                 dataset.clear_associated_files( purge = True )
                 dataset.flush()
+                if dataset.dataset.purgable:
+                    for shared_data in dataset.dataset.history_associations:
+                        if not shared_data.purged:
+                            break #only purge when not shared
+                    else:
+                        os.unlink( dataset.file_name )
+                        purgable = True
             except Exception, exc:
                 return "# Error, exception: %s caught attempting to purge %s\n" %( str( exc ), dataset.file_name )
             try:
-                os.unlink( dataset.extra_files_path )
+                if purgable:
+                    os.unlink( dataset.extra_files_path )
             except:
                 pass
         else:
diff --git a/templates/root/history_common.mako b/templates/root/history_common.mako
index 7035941b51d..877705614ed 100644
--- a/templates/root/history_common.mako
+++ b/templates/root/history_common.mako
@@ -83,9 +83,9 @@
 		##        be a 'visible_children' method on dataset.
                 <%
 		children = []
-                for child_assoc in data.children:
-                    if child_assoc.child.visible:
-                        children.append( child_assoc.child )
+                for child in data.children:
+                    if child.visible:
+                        children.append( child )
                 %>
                 %if len( children ) > 0:
                     

diff --git a/tools/data_source/encode_db.xml b/tools/data_source/encode_db.xml
index eccc16fdb1f..00f133121af 100644
--- a/tools/data_source/encode_db.xml
+++ b/tools/data_source/encode_db.xml
@@ -17,8 +17,10 @@
 	
 	
   
-	
-	
+  
+  
+    
+  
 	
 	
 
diff --git a/tools/data_source/encode_import_code.py b/tools/data_source/encode_import_code.py
index 2db88b83bdf..09a6e8a9823 100644
--- a/tools/data_source/encode_import_code.py
+++ b/tools/data_source/encode_import_code.py
@@ -33,7 +33,7 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
             dbkey = fields[2]
             filepath = fields[3]
             file_type = fields[4]
-            newdata = app.model.Dataset()
+            newdata = app.model.HistoryDatasetAssociation( create_dataset = True ) #This import should become a library
             newdata.extension = file_type
             newdata.name = basic_name + " (" + description + ")"
             history.add_dataset( newdata )
diff --git a/tools/data_source/encodedb_filter.py b/tools/data_source/encodedb_filter.py
deleted file mode 100644
index 6fcc26c89d8..00000000000
--- a/tools/data_source/encodedb_filter.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# runs after the job (and after the default post-filter)
-
-def validate(incoming):
-    """Validator"""
-    #raise Exception, 'not quite right'
-    pass
-
-def exec_before_job( app, inp_data, out_data, param_dict, tool=None):
-    """Sets the name of the data"""
-    dataid = param_dict.get( 'dataid', None )
-    data = app.model.Dataset.get( dataid )
-    if data:
-        data.info = data.states.RUNNING
-        data.flush()
-
-def exec_after_process( app, inp_data, out_data, param_dict, **kwd):
-    """Sets the name of the data"""
-    dataid = param_dict.get( 'dataid', None )
-    data = app.model.Dataset.get(dataid)
-    if data:
-        data.state = data.states.OK
-        data.set_peek()
-        data.set_size()
-        data.flush()
diff --git a/tools/data_source/microbial_import_code.py b/tools/data_source/microbial_import_code.py
index ab784a4487d..b6bc2f6bd85 100644
--- a/tools/data_source/microbial_import_code.py
+++ b/tools/data_source/microbial_import_code.py
@@ -124,7 +124,7 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
             dbkey = fields[3]
             filepath = fields[4]
             file_type = fields[5]
-            newdata = app.model.Dataset()
+            newdata = app.model.HistoryDatasetAssociation( create_dataset = True ) #This import should become a library
             newdata.extension = file_type
             newdata.name = basic_name + " (" + microbe_info[kingdom][org]['chrs'][chr]['data'][description]['feature'] +" for "+microbe_info[kingdom][org]['name']+":"+chr + ")"
             newdata.flush()
diff --git a/tools/maf/maf_to_bed_code.py b/tools/maf/maf_to_bed_code.py
index 2abc31bf1b4..c8f1e905e8e 100644
--- a/tools/maf/maf_to_bed_code.py
+++ b/tools/maf/maf_to_bed_code.py
@@ -28,7 +28,7 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
             dbkey = fields[1]
             filepath = fields[2]
 
-            newdata = app.model.Dataset()
+            newdata = app.model.HistoryDatasetAssociation( create_dataset = True )
             newdata.extension = "bed"
             newdata.name = basic_name + " (" + dbkey + ")"
             newdata.flush()