Add dataset filename table. Filenames can now be assigned, setting the readonly flag on a dataset filename object

will prevent Galaxy from deleting the file when the dataset is purged.  Copies of datasets (such as when sharing a
history), no longer copy file contents.

Database changes required:

CREATE TABLE dataset_filename (
        id INTEGER NOT NULL,
        create_time TIMESTAMP DEFAULT current_timestamp,
        update_time TIMESTAMP DEFAULT current_timestamp,
        filename TEXT,
        extra_files_path TEXT,
        readonly BOOLEAN,
        PRIMARY KEY (id)
);

ALTER TABLE dataset ADD filename_id INTEGER;

lib/galaxy/model/__init__.py

@@ -122,6 +122,26 @@ class History( object ):
         self.genome_build = genome_build
         self.datasets.append( dataset )
 
+    def copy(self):
+        des = History()
+        des.flush()
+        des.name = self.name
+        des.user_id = self.user_id
+        for data in self.datasets:
+            new_data = data.copy()
+            des.add_dataset(new_data)
+            new_data.hid = data.hid
+            new_data.flush()
+            for child_assoc in data.children:
+                new_child = child_assoc.child.copy()
+                new_assoc = DatasetChildAssociation( child_assoc.designation )
+                new_assoc.child = new_child
+                new_assoc.parent = new_data
+                new_child.flush()
+        des.hid_counter = self.hid_counter
+        des.flush()
+        return des
+
 # class Query( object ):
 #     def __init__( self, name=None, state=None, tool_parameters=None, history=None ):
 #         self.name = name or "Unnamed query"
@@ -143,7 +163,7 @@ class Dataset( object ):
     engine = None
     def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None, 
                   dbkey=None, state=None, metadata=None, history=None, parent_id=None, designation=None,
-                  validation_errors=None, visible=True ):
+                  validation_errors=None, visible=True, filename_id = None ):
         self.name = name or "Unnamed dataset"
         self.id = id
         self.hid = hid
@@ -159,6 +179,7 @@ class Dataset( object ):
         self.deleted = False
         self.purged = False
         self.visible = visible
+        self.filename_id = filename_id
         # Relationships
         self.history = history
         self.validation_errors = validation_errors
@@ -166,10 +187,31 @@ class Dataset( object ):
     @property
     def ext( self ):
         return self.extension
+    
+    def get_file_name( self ):
+        if self.filename_id is None:
+            assert self.id is not None, "ID must be set before filename used (commit the object)"
+            return os.path.join( self.file_path, "dataset_%d.dat" % self.id )
+        else:
+            return self.dataset_file.filename
+    def set_file_name (self, filename):
+        if filename is None:
+            self.filename_id = None
+        else:
+            filename_obj = DatasetFileName.get_by(filename=filename)
+            if filename_obj is None:
+                filename_obj = DatasetFileName(filename=filename, extra_files_path=self.extra_files_path)
+                filename_obj.flush()
+            self.filename_id = filename_obj.id
+        self.flush()
+        self.refresh()
+    file_name = property( get_file_name, set_file_name )
+    
     @property
-    def file_name( self ):
-        assert self.id is not None, "ID must be set before filename used (commit the object)"
-        return os.path.join( self.file_path, "dataset_%d.dat" % self.id )
+    def extra_files_path( self ):
+        if self.dataset_file and self.dataset_file.extra_files_path: return self.dataset_file.extra_files_path
+        return os.path.join( self.file_path, "dataset_%d_files" % self.id )
+    
     @property
     def datatype( self ):
         return datatypes_registry.get_datatype_by_extension( self.extension )
@@ -255,12 +297,46 @@ class Dataset( object ):
         """Removes the file contents from disk """
         self.deleted = True
         self.purged = True
-        try: os.unlink(self.file_name)
-        except: pass
-        try: os.unlink(os.path.join(self.file_path, "dataset_%d_files" % (self.id)))
-        except: pass
+        if self.dataset_file is None or not self.dataset_file.readonly:
+            #Check to see if another dataset is using this file
+            if self.dataset_file:
+                for data in self.select_by(purged=False, filename_id=self.dataset_file.id):
+                    if data.id != self.id: return
+            #Delete files
+            try: os.unlink(self.file_name)
+            except: pass
+            try: os.unlink(self.extra_files_path)
+            except: pass
+            
     def get_converter_types(self):
         return self.datatype.get_converter_types( self, datatypes_registry)
+    
+    def copy(self, parent_id=None):
+        des = Dataset(extension=self.ext)
+        des.flush()
+        des.name = self.name
+        des.info = self.info
+        des.blurb = self.blurb
+        des.peek = self.peek
+        des.extension = self.extension
+        des.dbkey = str( self.dbkey )
+        des.state = self.state
+        des.metadata = self.metadata
+        des.hid = self.hid
+        # Make sure source is using filename table, so purge works properly
+        if not self.dataset_file:
+            self.set_file_name(self.file_name)
+            self.flush()
+            self.refresh()
+            self.dataset_file.extra_files_path = self.extra_files_path
+            self.flush()
+        # Don't copy file contents, share original file
+        des.file_name = self.file_name
+        des.hid = self.hid
+        des.designation = self.designation
+        des.flush()
+        return des
+
     def add_validation_error( self, validation_error ):
         self.validation_errors.append( validation_error )
 
@@ -275,6 +351,12 @@ class Dataset( object ):
         except OSError, e:
             log.critical('%s delete error %s' % (self.__class__.__name__, e))
 
+class DatasetFileName( object ):
+    def __init__( self, filename=None, readonly=False, extra_files_path=None ):
+        self.filename = filename
+        self.readonly = readonly
+        self.extra_files_path = extra_files_path
+
 class Old_Dataset( Dataset ):
     pass
             

lib/galaxy/model/mapping.py

@@ -74,8 +74,17 @@ Dataset.table = Table( "dataset", metadata,
     Column( "deleted", Boolean ),
     Column( "purged", Boolean ),
     Column( "visible", Boolean ),
+    Column( "filename_id", Integer, ForeignKey( "dataset_filename.id" ), nullable=True ),
     ForeignKeyConstraint(['parent_id'],['dataset.id'], ondelete="CASCADE") )
 
+DatasetFileName.table = Table( "dataset_filename", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, PassiveDefault( now ) ),
+    Column( "update_time", DateTime, PassiveDefault( now ), onupdate=now ),
+    Column( "filename", TEXT ),
+    Column( "extra_files_path", TEXT, nullable=True, default=None ),
+    Column( "readonly", Boolean, default=False ) )
+
 ValidationError.table = Table( "validation_error", metadata,
     Column( "id", Integer, primary_key=True ),
     Column( "dataset_id", Integer, ForeignKey( "dataset.id" ) ),
@@ -163,8 +172,14 @@ assign_mapper( context, Dataset, Dataset.table,
             DatasetChildAssociation, 
             primaryjoin=( DatasetChildAssociation.table.c.parent_dataset_id == Dataset.table.c.id ),
             lazy=False,
-            backref="parent" ) ) )
-                                        
+            backref="parent" ),
+        dataset_file=relation( 
+            DatasetFileName, 
+            primaryjoin=( DatasetFileName.table.c.id == Dataset.table.c.filename_id ) )
+            ) )
+
+assign_mapper( context, DatasetFileName, DatasetFileName.table )
+
 assign_mapper( context, DatasetChildAssociation, DatasetChildAssociation.table,
     properties=dict( child=relation( Dataset, primaryjoin=( DatasetChildAssociation.table.c.child_dataset_id == Dataset.table.c.id ) ) ) )
 

lib/galaxy/web/controllers/dataset.py

@@ -112,8 +112,7 @@ class DatasetInterface( BaseController ):
         else:
             #display files from directory here
             try:
-                file_path = os.path.join(trans.app.config.file_path, "dataset_%s_files" % (dataset_id))
-                file_path = os.path.join(file_path, filename)
+                file_path = os.path.join(trans.app.model.Dataset.get( dataset_id ).extra_files_path, filename)
                 return open(file_path)
             except:
                 raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % (filename) )

lib/galaxy/web/controllers/root.py

@@ -363,7 +363,7 @@ class Universe( BaseController ):
             send_to_err = "You can't send histories to yourself"
         else:
             for history in histories:
-                new_history = self.copy_history(history, trans)
+                new_history = history.copy()
                 new_history.name = history.name+" from "+user.email
                 new_history.user_id = send_to_user.id
                 new_history.add_galaxy_session(trans.get_galaxy_session( create=True ))
@@ -392,7 +392,7 @@ class Universe( BaseController ):
         if user:
             if import_history.user_id == user.id:
                 return trans.show_error_message( "You cannot import your own history.")
-            new_history = self.copy_history(import_history, trans)
+            new_history = import_history.copy()
             new_history.name = "imported: "+new_history.name
             new_history.user_id = user.id
             new_history.add_galaxy_session(trans.get_galaxy_session( create=True ))
@@ -402,7 +402,7 @@ class Universe( BaseController ):
             trans.log_event( "History imported, id: %s, name: '%s': " % (str(new_history.id) , new_history.name ) )
             return trans.fill_template("history_imported.tmpl", history=new_history)
         elif not user_history.datasets or confirm:
-            new_history = self.copy_history(import_history, trans)
+            new_history = import_history.copy()
             new_history.name = "imported: "+new_history.name
             new_history.user_id = None
             new_history.add_galaxy_session(trans.get_galaxy_session( create=True ))
@@ -554,7 +554,7 @@ class Universe( BaseController ):
         """Copies a dataset and makes primary"""
         try:
             old_data = self.app.model.Dataset.get( id )
-            new_data = self.copy_dataset(old_data, trans)
+            new_data = old_data.copy()
             ## new_data.parent = None
             ## history = trans.app.model.History.get( old_data.history_id )
             history = trans.get_history()
@@ -605,46 +605,3 @@ class Universe( BaseController ):
             if isinstance( kwd[k], FieldStorage ):
                 rval += "-> %s" % kwd[k].file.read()
         return rval
-
-    # ---- Work methods -----------------------------------------------------
-    
-    def copy_dataset(self, src, trans, parent_id=None):
-        des = self.app.model.Dataset()
-        des.flush()
-        des.change_datatype( src.ext )
-        des.name = src.name
-        des.info = src.info
-        des.blurb = src.blurb
-        des.peek = src.peek
-        des.extension = src.extension
-        des.dbkey = str( src.dbkey )
-        des.state = src.state
-        des.metadata = src.metadata
-        des.hid = src.hid
-        ## des.parent_id = parent_id
-        shutil.copyfile(src.file_name,des.file_name)
-        des.hid = src.hid
-        des.designation = src.designation
-        des.flush()
-        return des
-        
-    def copy_history(self, src, trans):
-        des = self.app.model.History()
-        des.flush()
-        des.name = src.name
-        des.user_id = src.user_id
-        for data in src.datasets:
-            new_data = self.copy_dataset(data, trans)
-            des.add_dataset(new_data)
-            new_data.hid = data.hid
-            new_data.flush()
-            for child_assoc in data.children:
-                new_child = self.copy_dataset(child_assoc.child, trans)
-                new_assoc = self.app.model.DatasetChildAssociation( child_assoc.designation )
-                new_assoc.child = new_child
-                new_assoc.parent = new_data
-                #des.add_dataset(new_child, parent_id = new_data.id)
-                new_child.flush()
-        des.hid_counter = src.hid_counter
-        self.app.model.flush()
-        return des

scripts/cleanup_datasets.py

@@ -74,13 +74,14 @@ def purge( app, days ):
     """ Purges deleted datasets older than specified number of days """
     count = 0
     now  = time.time()
-    for row in app.model.Dataset.table.select().execute():
+    for row in list(app.model.Dataset.table.select().execute()):
         data = app.model.Dataset.get(row.id)
         if data.deleted and not data.purged:
             last = time.mktime( time.strptime( data.update_time.strftime('%a %b %d %H:%M:%S %Y') ))
             diff = (now - last) /3600/24 # days
             if diff>days:
                 data.purge()
+                data.flush()
                 count += 1
     try:
         app.model.flush()