From ed964dd043edd66ddeab7a2301c19cd608a011b6 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Mon, 24 Sep 2007 19:42:09 +0000 Subject: [PATCH] Add dataset filename table. Filenames can now be assigned, setting the readonly flag on a dataset filename object will prevent Galaxy from deleting the file when the dataset is purged. Copies of datasets (such as when sharing a history), no longer copy file contents. Database changes required: CREATE TABLE dataset_filename ( id INTEGER NOT NULL, create_time TIMESTAMP DEFAULT current_timestamp, update_time TIMESTAMP DEFAULT current_timestamp, filename TEXT, extra_files_path TEXT, readonly BOOLEAN, PRIMARY KEY (id) ); ALTER TABLE dataset ADD filename_id INTEGER; --- lib/galaxy/model/__init__.py | 98 ++++++++++++++++++++++++--- lib/galaxy/model/mapping.py | 19 +++++- lib/galaxy/web/controllers/dataset.py | 3 +- lib/galaxy/web/controllers/root.py | 51 ++------------ scripts/cleanup_datasets.py | 3 +- 5 files changed, 114 insertions(+), 60 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 96604fb5e40..fc302981ccc 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -122,6 +122,26 @@ class History( object ): self.genome_build = genome_build self.datasets.append( dataset ) + def copy(self): + des = History() + des.flush() + des.name = self.name + des.user_id = self.user_id + for data in self.datasets: + new_data = data.copy() + des.add_dataset(new_data) + new_data.hid = data.hid + new_data.flush() + for child_assoc in data.children: + new_child = child_assoc.child.copy() + new_assoc = DatasetChildAssociation( child_assoc.designation ) + new_assoc.child = new_child + new_assoc.parent = new_data + new_child.flush() + des.hid_counter = self.hid_counter + des.flush() + return des + # class Query( object ): # def __init__( self, name=None, state=None, tool_parameters=None, history=None ): # self.name = name or "Unnamed query" @@ -143,7 +163,7 @@ class Dataset( object ): engine = None def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None, dbkey=None, state=None, metadata=None, history=None, parent_id=None, designation=None, - validation_errors=None, visible=True ): + validation_errors=None, visible=True, filename_id = None ): self.name = name or "Unnamed dataset" self.id = id self.hid = hid @@ -159,6 +179,7 @@ class Dataset( object ): self.deleted = False self.purged = False self.visible = visible + self.filename_id = filename_id # Relationships self.history = history self.validation_errors = validation_errors @@ -166,10 +187,31 @@ class Dataset( object ): @property def ext( self ): return self.extension + + def get_file_name( self ): + if self.filename_id is None: + assert self.id is not None, "ID must be set before filename used (commit the object)" + return os.path.join( self.file_path, "dataset_%d.dat" % self.id ) + else: + return self.dataset_file.filename + def set_file_name (self, filename): + if filename is None: + self.filename_id = None + else: + filename_obj = DatasetFileName.get_by(filename=filename) + if filename_obj is None: + filename_obj = DatasetFileName(filename=filename, extra_files_path=self.extra_files_path) + filename_obj.flush() + self.filename_id = filename_obj.id + self.flush() + self.refresh() + file_name = property( get_file_name, set_file_name ) + @property - def file_name( self ): - assert self.id is not None, "ID must be set before filename used (commit the object)" - return os.path.join( self.file_path, "dataset_%d.dat" % self.id ) + def extra_files_path( self ): + if self.dataset_file and self.dataset_file.extra_files_path: return self.dataset_file.extra_files_path + return os.path.join( self.file_path, "dataset_%d_files" % self.id ) + @property def datatype( self ): return datatypes_registry.get_datatype_by_extension( self.extension ) @@ -255,12 +297,46 @@ class Dataset( object ): """Removes the file contents from disk """ self.deleted = True self.purged = True - try: os.unlink(self.file_name) - except: pass - try: os.unlink(os.path.join(self.file_path, "dataset_%d_files" % (self.id))) - except: pass + if self.dataset_file is None or not self.dataset_file.readonly: + #Check to see if another dataset is using this file + if self.dataset_file: + for data in self.select_by(purged=False, filename_id=self.dataset_file.id): + if data.id != self.id: return + #Delete files + try: os.unlink(self.file_name) + except: pass + try: os.unlink(self.extra_files_path) + except: pass + def get_converter_types(self): return self.datatype.get_converter_types( self, datatypes_registry) + + def copy(self, parent_id=None): + des = Dataset(extension=self.ext) + des.flush() + des.name = self.name + des.info = self.info + des.blurb = self.blurb + des.peek = self.peek + des.extension = self.extension + des.dbkey = str( self.dbkey ) + des.state = self.state + des.metadata = self.metadata + des.hid = self.hid + # Make sure source is using filename table, so purge works properly + if not self.dataset_file: + self.set_file_name(self.file_name) + self.flush() + self.refresh() + self.dataset_file.extra_files_path = self.extra_files_path + self.flush() + # Don't copy file contents, share original file + des.file_name = self.file_name + des.hid = self.hid + des.designation = self.designation + des.flush() + return des + def add_validation_error( self, validation_error ): self.validation_errors.append( validation_error ) @@ -275,6 +351,12 @@ class Dataset( object ): except OSError, e: log.critical('%s delete error %s' % (self.__class__.__name__, e)) +class DatasetFileName( object ): + def __init__( self, filename=None, readonly=False, extra_files_path=None ): + self.filename = filename + self.readonly = readonly + self.extra_files_path = extra_files_path + class Old_Dataset( Dataset ): pass diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f0d6abaec97..1a8255daa6f 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -74,8 +74,17 @@ Dataset.table = Table( "dataset", metadata, Column( "deleted", Boolean ), Column( "purged", Boolean ), Column( "visible", Boolean ), + Column( "filename_id", Integer, ForeignKey( "dataset_filename.id" ), nullable=True ), ForeignKeyConstraint(['parent_id'],['dataset.id'], ondelete="CASCADE") ) +DatasetFileName.table = Table( "dataset_filename", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, PassiveDefault( now ) ), + Column( "update_time", DateTime, PassiveDefault( now ), onupdate=now ), + Column( "filename", TEXT ), + Column( "extra_files_path", TEXT, nullable=True, default=None ), + Column( "readonly", Boolean, default=False ) ) + ValidationError.table = Table( "validation_error", metadata, Column( "id", Integer, primary_key=True ), Column( "dataset_id", Integer, ForeignKey( "dataset.id" ) ), @@ -163,8 +172,14 @@ assign_mapper( context, Dataset, Dataset.table, DatasetChildAssociation, primaryjoin=( DatasetChildAssociation.table.c.parent_dataset_id == Dataset.table.c.id ), lazy=False, - backref="parent" ) ) ) - + backref="parent" ), + dataset_file=relation( + DatasetFileName, + primaryjoin=( DatasetFileName.table.c.id == Dataset.table.c.filename_id ) ) + ) ) + +assign_mapper( context, DatasetFileName, DatasetFileName.table ) + assign_mapper( context, DatasetChildAssociation, DatasetChildAssociation.table, properties=dict( child=relation( Dataset, primaryjoin=( DatasetChildAssociation.table.c.child_dataset_id == Dataset.table.c.id ) ) ) ) diff --git a/lib/galaxy/web/controllers/dataset.py b/lib/galaxy/web/controllers/dataset.py index f103f7a9b68..d6071492dc9 100644 --- a/lib/galaxy/web/controllers/dataset.py +++ b/lib/galaxy/web/controllers/dataset.py @@ -112,8 +112,7 @@ class DatasetInterface( BaseController ): else: #display files from directory here try: - file_path = os.path.join(trans.app.config.file_path, "dataset_%s_files" % (dataset_id)) - file_path = os.path.join(file_path, filename) + file_path = os.path.join(trans.app.model.Dataset.get( dataset_id ).extra_files_path, filename) return open(file_path) except: raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % (filename) ) diff --git a/lib/galaxy/web/controllers/root.py b/lib/galaxy/web/controllers/root.py index 2bdbca36711..99dfadf1fe0 100644 --- a/lib/galaxy/web/controllers/root.py +++ b/lib/galaxy/web/controllers/root.py @@ -363,7 +363,7 @@ class Universe( BaseController ): send_to_err = "You can't send histories to yourself" else: for history in histories: - new_history = self.copy_history(history, trans) + new_history = history.copy() new_history.name = history.name+" from "+user.email new_history.user_id = send_to_user.id new_history.add_galaxy_session(trans.get_galaxy_session( create=True )) @@ -392,7 +392,7 @@ class Universe( BaseController ): if user: if import_history.user_id == user.id: return trans.show_error_message( "You cannot import your own history.") - new_history = self.copy_history(import_history, trans) + new_history = import_history.copy() new_history.name = "imported: "+new_history.name new_history.user_id = user.id new_history.add_galaxy_session(trans.get_galaxy_session( create=True )) @@ -402,7 +402,7 @@ class Universe( BaseController ): trans.log_event( "History imported, id: %s, name: '%s': " % (str(new_history.id) , new_history.name ) ) return trans.fill_template("history_imported.tmpl", history=new_history) elif not user_history.datasets or confirm: - new_history = self.copy_history(import_history, trans) + new_history = import_history.copy() new_history.name = "imported: "+new_history.name new_history.user_id = None new_history.add_galaxy_session(trans.get_galaxy_session( create=True )) @@ -554,7 +554,7 @@ class Universe( BaseController ): """Copies a dataset and makes primary""" try: old_data = self.app.model.Dataset.get( id ) - new_data = self.copy_dataset(old_data, trans) + new_data = old_data.copy() ## new_data.parent = None ## history = trans.app.model.History.get( old_data.history_id ) history = trans.get_history() @@ -605,46 +605,3 @@ class Universe( BaseController ): if isinstance( kwd[k], FieldStorage ): rval += "-> %s" % kwd[k].file.read() return rval - - # ---- Work methods ----------------------------------------------------- - - def copy_dataset(self, src, trans, parent_id=None): - des = self.app.model.Dataset() - des.flush() - des.change_datatype( src.ext ) - des.name = src.name - des.info = src.info - des.blurb = src.blurb - des.peek = src.peek - des.extension = src.extension - des.dbkey = str( src.dbkey ) - des.state = src.state - des.metadata = src.metadata - des.hid = src.hid - ## des.parent_id = parent_id - shutil.copyfile(src.file_name,des.file_name) - des.hid = src.hid - des.designation = src.designation - des.flush() - return des - - def copy_history(self, src, trans): - des = self.app.model.History() - des.flush() - des.name = src.name - des.user_id = src.user_id - for data in src.datasets: - new_data = self.copy_dataset(data, trans) - des.add_dataset(new_data) - new_data.hid = data.hid - new_data.flush() - for child_assoc in data.children: - new_child = self.copy_dataset(child_assoc.child, trans) - new_assoc = self.app.model.DatasetChildAssociation( child_assoc.designation ) - new_assoc.child = new_child - new_assoc.parent = new_data - #des.add_dataset(new_child, parent_id = new_data.id) - new_child.flush() - des.hid_counter = src.hid_counter - self.app.model.flush() - return des diff --git a/scripts/cleanup_datasets.py b/scripts/cleanup_datasets.py index f885ba3915e..e9ef05c38fa 100644 --- a/scripts/cleanup_datasets.py +++ b/scripts/cleanup_datasets.py @@ -74,13 +74,14 @@ def purge( app, days ): """ Purges deleted datasets older than specified number of days """ count = 0 now = time.time() - for row in app.model.Dataset.table.select().execute(): + for row in list(app.model.Dataset.table.select().execute()): data = app.model.Dataset.get(row.id) if data.deleted and not data.purged: last = time.mktime( time.strptime( data.update_time.strftime('%a %b %d %H:%M:%S %Y') )) diff = (now - last) /3600/24 # days if diff>days: data.purge() + data.flush() count += 1 try: app.model.flush()