push changesets to security from central

.hgignore

@@ -0,0 +1,23 @@
+syntax: glob
+
+# Downloaded and locally built eggs
+eggs
+scripts/scramble/build
+scripts/scramble/lib
+scripts/scramble/archives
+
+# Database stuff
+database/beaker_sessions
+database/compiled_templates
+database/files
+database/*.sqlite
+
+# Python bytecode
+*.pyc
+
+# Config files
+universe_wsgi.ini
+reports_wsgi.ini
+datatypes_conf.xml
+tool_conf.xml
+

LICENSE.txt

@@ -17,4 +17,9 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 
 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 
 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 
+
+Some icons found in Galaxy are from the Silk Icons set, available under
+the Creative Commons Attribution 2.5 License, from:
+
+http://www.famfamfam.com/lab/icons/silk/

datatypes_conf.xml.sample

@@ -5,8 +5,10 @@
     <datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true"/>
     <datatype extension="bed" type="galaxy.datatypes.interval:Bed" display_in_upload="true">
       <converter file="bed_to_gff_converter.xml" target_datatype="gff"/>
+      <converter file="interval_to_coverage.xml" target_datatype="coverage"/>
     </datatype>
     <datatype extension="binseq.zip" type="galaxy.datatypes.images:Binseq" mimetype="application/zip" display_in_upload="true"/>
+    <datatype extension="coverage" type="galaxy.datatypes.coverage:LastzCoverage" display_in_upload="true"/>
     <datatype extension="customtrack" type="galaxy.datatypes.interval:CustomTrack"/>
     <datatype extension="csfasta" type="galaxy.datatypes.sequence:csFasta" display_in_upload="true"/>
     <datatype extension="data" type="galaxy.datatypes.data:Data" mimetype="application/octet-stream"/>
@@ -17,6 +19,7 @@
       <converter file="fastqsolexa_to_fasta_converter.xml" target_datatype="fasta"/>
       <converter file="fastqsolexa_to_qual_converter.xml" target_datatype="qual"/>
     </datatype>
+    <datatype extension="genetrack" type="galaxy.datatypes.tracks:GeneTrack"/>
     <datatype extension="gff" type="galaxy.datatypes.interval:Gff" display_in_upload="true">
       <converter file="gff_to_bed_converter.xml" target_datatype="bed"/>
     </datatype>
@@ -141,6 +144,30 @@
     <datatype extension="wobble" type="galaxy.datatypes.data:Text"/>
     <datatype extension="wordcount" type="galaxy.datatypes.data:Text"/>
     <datatype extension="tagseq" type="galaxy.datatypes.data:Text"/>
+  <!-- Start RGenetics Datatypes -->
+    <!-- genome graphs ucsc file - first col is always marker then numeric values to plot -->
+    <datatype extension="gg" type="galaxy.datatypes.genetics:GenomeGraphs"/>
+    <datatype extension="rgenetics" type="galaxy.datatypes.genetics:Rgenetics"/>
+    <!-- linkage format pedigree (separate .map file) -->
+    <datatype extension="lped" type="galaxy.datatypes.genetics:Lped"/>
+    <!-- plink compressed file - has bed extension unfortunately -->
+    <datatype extension="pbed" type="galaxy.datatypes.genetics:Pbed"/>
+    <!-- eigenstrat pedigree input file -->
+    <datatype extension="eigenstratgeno" type="galaxy.datatypes.genetics:Eigenstratgeno"/>
+    <!-- eigenstrat pca output file for adjusted eigenQTL eg -->
+    <datatype extension="eigenstratpca" type="galaxy.datatypes.genetics:Eigenstratpca"/>
+    <!-- fbat/pbat format pedigree (header row of marker names) -->
+    <datatype extension="fped" type="galaxy.datatypes.genetics:Fped"/>
+    <!-- part of linkage format pedigree -->
+    <datatype extension="lmap" type="galaxy.datatypes.genetics:Lmap"/>
+    <!-- phenotype file - fbat format -->
+    <datatype extension="fphe" type="galaxy.datatypes.genetics:Fphe"/>
+    <!-- phenotype file - plink format -->
+    <datatype extension="pphe" type="galaxy.datatypes.genetics:Pphe"/>
+    <datatype extension="snptest" type="galaxy.datatypes.genetics:Snptest"/>
+    <datatype extension="snpmatrix" type="galaxy.datatypes.genetics:SNPMatrix"/>
+    <datatype extension="xls" type="galaxy.datatypes.tabular:Tabular"/>
+  <!-- End RGenetics Datatypes -->
   </registration>
   <sniffers>
     <!--
@@ -153,6 +180,8 @@
     <sniffer type="galaxy.datatypes.xml:BlastXml"/>
     <sniffer type="galaxy.datatypes.sequence:Maf"/>
     <sniffer type="galaxy.datatypes.sequence:Lav"/>
+    <sniffer type="galaxy.datatypes.sequence:csFasta"/>
+    <sniffer type="galaxy.datatypes.qualityscore:QualityScore"/>
     <sniffer type="galaxy.datatypes.sequence:Fasta"/>
     <sniffer type="galaxy.datatypes.sequence:FastqSolexa"/>
     <sniffer type="galaxy.datatypes.interval:Wiggle"/>

lib/galaxy/app.py

@@ -5,6 +5,7 @@ from galaxy.web import security
 import galaxy.model
 import galaxy.model.mapping
 import galaxy.datatypes.registry
+import galaxy.security
 
 class UniverseApplication( object ):
     """Encapsulates the state of a Universe application"""
@@ -26,13 +27,15 @@ class UniverseApplication( object ):
         self.model = galaxy.model.mapping.init( self.config.file_path,
                                                 db_url,
                                                 self.config.database_engine_options,
-                                                create_tables = True )
+                                                create_tables = self.config.database_create_tables )
         # Security helper
         self.security = security.SecurityHelper( id_secret=self.config.id_secret )
         # Initialize the tools
         self.toolbox = tools.ToolBox( self.config.tool_config, self.config.tool_path, self )
         #Load datatype converters
         self.datatypes_registry.load_datatype_converters( self.toolbox )
+        #Load security policy
+        self.security_agent = self.model.security_agent
         # Heartbeat and memdump for thread / heap profiling
         self.heartbeat = None
         self.memdump = None

lib/galaxy/config.py

@@ -27,6 +27,7 @@ class Configuration( object ):
         self.database = resolve_path( kwargs.get( "database_file", "database/universe.d" ), self.root )
         self.database_connection =  kwargs.get( "database_connection", False )
         self.database_engine_options = get_database_engine_options( kwargs )                        
+        self.database_create_tables = string_as_bool( kwargs.get( "database_create_tables", "True" ) )
         # Where dataset files are stored
         self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root )
         self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root )
@@ -36,19 +37,23 @@ class Configuration( object ):
         self.tool_config = resolve_path( kwargs.get( 'tool_config_file', 'tool_conf.xml' ), self.root )
         self.tool_secret = kwargs.get( "tool_secret", "" )
         self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" )
+        self.set_metadata_externally = string_as_bool( kwargs.get( "set_metadata_externally", "False" ) )
         self.use_remote_user = string_as_bool( kwargs.get( "use_remote_user", "False" ) )
         self.remote_user_maildomain = kwargs.get( "remote_user_maildomain", None )
+        self.require_login = string_as_bool( kwargs.get( "require_login", "False" ) )
+        self.allow_user_creation = string_as_bool( kwargs.get( "allow_user_creation", "True" ) )
+        self.allow_user_deletion = string_as_bool( kwargs.get( "allow_user_deletion", "False" ) )
         self.template_path = resolve_path( kwargs.get( "template_path", "templates" ), self.root )
         self.template_cache = resolve_path( kwargs.get( "template_cache_path", "database/compiled_templates" ), self.root )
         self.local_job_queue_workers = int( kwargs.get( "local_job_queue_workers", "5" ) )
-        self.cluster_job_queue_workers = int( kwargs.get( "cluster_job_queue_workers", "5" ) )
+        self.cluster_job_queue_workers = int( kwargs.get( "cluster_job_queue_workers", "3" ) )
         self.job_scheduler_policy = kwargs.get("job_scheduler_policy", "FIFO")
         self.job_queue_cleanup_interval = int( kwargs.get("job_queue_cleanup_interval", "5") )
         self.cluster_files_directory = os.path.abspath( kwargs.get( "cluster_files_directory", "database/pbs" ) )
         self.job_working_directory = resolve_path( kwargs.get( "job_working_directory", "database/job_working_directory" ), self.root )
         self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) )
         self.output_size_limit = int( kwargs.get( 'output_size_limit', 0 ) )
-        self.admin_pass = kwargs.get('admin_pass',"galaxy")
+        self.admin_users = kwargs.get( "admin_users", "" )
         self.sendmail_path = kwargs.get('sendmail_path',"/usr/sbin/sendmail")
         self.mailing_join_addr = kwargs.get('mailing_join_addr',"galaxy-user-join@bx.psu.edu")
         self.error_email_to = kwargs.get( 'error_email_to', None )
@@ -59,10 +64,10 @@ class Configuration( object ):
         self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "" )
         self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "" )
         self.pbs_stage_path = kwargs.get('pbs_stage_path', "" )
-        self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', False ) )
-        self.use_memdump = string_as_bool( kwargs.get( 'use_memdump', False ) )
-        self.log_memory_usage = string_as_bool( kwargs.get( 'log_memory_usage', False ) )
-        self.log_events = string_as_bool( kwargs.get( 'log_events', False ) )
+        self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', 'False' ) )
+        self.use_memdump = string_as_bool( kwargs.get( 'use_memdump', 'False' ) )
+        self.log_memory_usage = string_as_bool( kwargs.get( 'log_memory_usage', 'False' ) )
+        self.log_events = string_as_bool( kwargs.get( 'log_events', 'False' ) )
         self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea" ).lower().split(",")
         self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "wormbase,flybase,elegans" ).lower().split(",")
         self.brand = kwargs.get( 'brand', None )
@@ -70,6 +75,9 @@ class Configuration( object ):
         self.bugs_email = kwargs.get( 'bugs_email', None )
         self.blog_url = kwargs.get( 'blog_url', None )
         self.screencasts_url = kwargs.get( 'screencasts_url', None )
+        self.library_import_dir = kwargs.get( 'library_import_dir', None )
+        if self.library_import_dir is not None and not os.path.exists( self.library_import_dir ):
+            raise ConfigurationError( "library_import_dir specified in config (%s) does not exist" % self.library_import_dir )
         # Configuration options for taking advantage of nginx features
         self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False )
         self.nginx_upload_location = kwargs.get( 'nginx_upload_store', False )
@@ -103,7 +111,17 @@ class Configuration( object ):
         for path in self.tool_config, self.datatypes_config:
             if not os.path.isfile(path):
                 raise ConfigurationError("File not found: %s" % path )
-
+                
+    def is_admin_user( self,user ):
+        """
+        Determine if the provided user is listed in `admin_users`.
+        
+        NOTE: This is temporary, admin users will likely be specified in the
+              database in the future.
+        """
+        admin_users = self.get( "admin_users", "" ).split( "," )
+        return ( user is not None and user.email in admin_users )
+            
 def get_database_engine_options( kwargs ):
     """
     Allow options for the SQLAlchemy database engine to be passed by using

lib/galaxy/datatypes/converters/interval_to_coverage.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+"""
+Converter to generate 3 (or 4) column base-pair coverage from an interval file.
+
+usage: %prog bed_file out_file
+    -1, --cols1=N,N,N,N: Columns for chrom, start, end, strand in interval file
+    -2, --cols2=N,N,N,N: Columns for chrom, start, end, strand in coverage file
+"""
+import sys
+from galaxy import eggs
+import pkg_resources; pkg_resources.require( "bx-python" )
+from bx.intervals import io
+from bx.cookbook import doc_optparse
+import psyco_full
+import commands
+import os
+from os import environ
+import tempfile
+from bisect import bisect
+
+INTERVAL_METADATA = ('chromCol',
+                     'startCol',
+                     'endCol',
+                     'strandCol',)
+
+COVERAGE_METADATA = ('chromCol',
+                     'positionCol',
+                     'forwardCol',
+                     'reverseCol',)
+
+def main( interval, coverage ):
+    """
+    Uses a sliding window of partitions to count coverages.
+    Every interval record adds its start and end to the partitions.  The result
+    is a list of partitions, or every position that has a (maybe) different
+    number of basepairs covered.  We don't worry about merging because we pop
+    as the sorted intervals are read in.  As the input start positions exceed
+    the partition positions in partitions, coverages are kicked out in bulk.
+    """
+    partitions = []
+    forward_covs = []
+    reverse_covs = []
+    offset = 0
+    chrom = None
+    lastchrom = None
+    for record in interval:
+        chrom = record.chrom
+        if lastchrom and not lastchrom == chrom and partitions:
+            for partition in xrange(0, len(partitions)-1):
+                forward = forward_covs[partition]
+                reverse = reverse_covs[partition]
+                if forward+reverse > 0:
+                    coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), 
+                                   forward=forward, reverse=reverse)
+            partitions = []
+            forward_covs = []
+            reverse_covs = []
+            
+        start_index = bisect(partitions, record.start)
+        forward = int(record.strand == "+")
+        reverse = int(record.strand == "-")
+        forward_base = 0
+        reverse_base = 0
+        if start_index > 0:
+            forward_base = forward_covs[start_index-1]
+            reverse_base = reverse_covs[start_index-1]
+        partitions.insert(start_index, record.start)
+        forward_covs.insert(start_index, forward_base)
+        reverse_covs.insert(start_index, reverse_base)
+        end_index = bisect(partitions, record.end)
+        for index in xrange(start_index, end_index):
+            forward_covs[index] += forward
+            reverse_covs[index] += reverse
+        partitions.insert(end_index, record.end)
+        forward_covs.insert(end_index, forward_covs[end_index-1] - forward )
+        reverse_covs.insert(end_index, reverse_covs[end_index-1] - reverse )
+        
+        if partitions:
+            for partition in xrange(0, start_index):
+                forward = forward_covs[partition]
+                reverse = reverse_covs[partition]
+                if forward+reverse > 0:
+                    coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), 
+                                   forward=forward, reverse=reverse)
+            partitions = partitions[start_index:]
+            forward_covs = forward_covs[start_index:]
+            reverse_covs = reverse_covs[start_index:]
+        
+        lastchrom = chrom
+        
+    # Finish the last chromosome
+    if partitions:
+        for partition in xrange(0, len(partitions)-1):
+            forward = forward_covs[partition]
+            reverse = reverse_covs[partition]
+            if forward+reverse > 0:
+                coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), 
+                               forward=forward, reverse=reverse)
+    
+class CoverageWriter( object ):
+    def __init__( self, out_stream=None, chromCol=0, positionCol=1, forwardCol=2, reverseCol=3 ):
+        self.out_stream = out_stream
+        self.reverseCol = reverseCol
+        self.nlines = 0
+        positions = {str(chromCol):'%(chrom)s', 
+                     str(positionCol):'%(position)d', 
+                     str(forwardCol):'%(forward)d', 
+                     str(reverseCol):'%(reverse)d'}
+        if reverseCol < 0: 
+            self.template = "%(0)s\t%(1)s\t%(2)s\n" % positions
+        else:
+            self.template = "%(0)s\t%(1)s\t%(2)s\t%(3)s\n" % positions
+            
+    def write(self, **kwargs ):
+        if self.reverseCol < 0: kwargs['forward'] += kwargs['reverse']
+        posgen = kwargs['position']
+        for position in posgen:
+            kwargs['position'] = position
+            self.out_stream.write(self.template % kwargs)
+
+    def close(self):
+        self.out_stream.flush()
+        self.out_stream.close()
+        
+if __name__ == "__main__":
+    options, args = doc_optparse.parse( __doc__ )
+    try:
+        chr_col_1, start_col_1, end_col_1, strand_col_1 = [int(x)-1 for x in options.cols1.split(',')]
+        chr_col_2, position_col_2, forward_col_2, reverse_col_2 = [int(x)-1 for x in options.cols2.split(',')]      
+        in_fname, out_fname = args
+    except:
+        doc_optparse.exception()
+
+    # Sort through a tempfile first
+    temp_file = tempfile.NamedTemporaryFile(mode="r")
+    environ['LC_ALL'] = 'POSIX'
+    commandline = "sort -f -n -k %d -k %d -k %d -o %s %s" % (chr_col_1+1,start_col_1+1,end_col_1+1, temp_file.name, in_fname)
+    errorcode, stdout = commands.getstatusoutput(commandline)
+
+    coverage = CoverageWriter( out_stream = open(out_fname, "a"),
+                               chromCol = chr_col_2, positionCol = position_col_2,
+                               forwardCol = forward_col_2, reverseCol = reverse_col_2, )
+    temp_file.seek(0)
+    interval = io.NiceReaderWrapper( temp_file, 
+                                     chrom_col=chr_col_1,
+                                     start_col=start_col_1,
+                                     end_col=end_col_1,
+                                     strand_col=strand_col_1,
+                                     fix_strand=True )
+    main( interval, coverage )
+    temp_file.close()
+    coverage.close()

lib/galaxy/datatypes/converters/interval_to_coverage.xml

@@ -0,0 +1,18 @@
+<tool id="CONVERTER_interval_to_coverage_0" name="Convert Genomic Intervals To COVERAGE">
+  <!--  <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
+  <!-- Used on the metadata edit page. -->
+  <command interpreter="python">interval_to_coverage.py $input1 $output1 
+  -1 ${input1.metadata.chromCol},${input1.metadata.startCol},${input1.metadata.endCol},${input1.metadata.strandCol}
+  -2 ${output1.metadata.chromCol},${output1.metadata.positionCol},${output1.metadata.forwardCol},${output1.metadata.reverseCol}
+  </command>
+  <inputs>
+    <page>
+      <param format="interval" name="input1" type="data" label="Choose intervals"/>
+    </page>
+   </inputs>
+  <outputs>
+    <data format="coverage" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/coverage.py

@@ -0,0 +1,30 @@
+"""
+Coverage datatypes
+
+"""
+import pkg_resources
+pkg_resources.require( "bx-python" )
+
+import logging, os, sys, time, sets, tempfile, shutil
+import data
+from galaxy import util
+from galaxy.datatypes.sniff import *
+from galaxy.web import url_for
+from cgi import escape
+import urllib
+from bx.intervals.io import *
+from galaxy.datatypes import metadata
+from galaxy.datatypes.metadata import MetadataElement
+from galaxy.datatypes.tabular import Tabular
+
+log = logging.getLogger(__name__)
+
+class LastzCoverage( Tabular ):
+    file_ext = "coverage"
+    
+    MetadataElement( name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter )
+    MetadataElement( name="positionCol", default=2, desc="Position column", param=metadata.ColumnParameter )
+    MetadataElement( name="forwardCol", default=3, desc="Forward or aggregate read column", param=metadata.ColumnParameter )
+    MetadataElement( name="reverseCol", desc="Optional reverse read column", param=metadata.ColumnParameter, optional=True, no_value=0 )
+    MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True, visible=False )
+    

lib/galaxy/datatypes/data.py

@@ -45,6 +45,9 @@ class Data( object ):
     """Stores the set of display applications, and viewing methods, supported by this datatype """
     supported_display_apps = {}
     
+    """If False, the peek is regenerated whenever a dataset of this type is copied"""
+    copy_safe_peek = True
+    
     def __init__(self, **kwd):
         """Initialize the datatype"""
         object.__init__(self, **kwd)
@@ -192,12 +195,17 @@ class Data( object ):
         return "This display type (%s) is not implemented for this datatype (%s)." % ( type, dataset.ext)
         
     def get_display_links(self, dataset, type, app, base_url, **kwd):
-        """Returns a list of tuples of (name, link) for a particular display type """
-        try:
-            if type in self.get_display_types():
-                return getattr (self, self.supported_display_apps[type]['links_function']) (dataset, type, app, base_url, **kwd)
-        except:
-            log.exception('Function %s is referred to in datatype %s for generating links for type %s, but is not accessible' % (self.supported_display_apps[type]['links_function'], self.__class__.__name__, type) )
+        """
+        Returns a list of tuples of (name, link) for a particular display type
+        as long as the dataset is not associated with a role restricting its access.
+        We determine this by sending None as the user to the allow_action method.
+        """
+        if app.security_agent.allow_action( None, dataset.permitted_actions.DATASET_ACCESS, dataset=dataset ):
+            try:
+                if type in self.get_display_types():
+                    return getattr (self, self.supported_display_apps[type]['links_function']) (dataset, type, app, base_url, **kwd)
+            except:
+                log.exception('Function %s is referred to in datatype %s for generating links for type %s, but is not accessible' % (self.supported_display_apps[type]['links_function'], self.__class__.__name__, type) )
         return []
 
     def get_converter_types(self, original_dataset, datatypes_registry):

lib/galaxy/datatypes/genetics.py

@@ -139,10 +139,10 @@ class SNPMatrix(Rgenetics):
         else:
             dataset.peek = 'file does not exist'
             dataset.blurb = 'file purged from disk'
-    def sniff( self, filename ):
-        """
-        """
-        return True
+    #def sniff( self, filename ):
+    #    """
+    #    """
+    #    return True
 
 class Lped(Rgenetics):
     """fake class to distinguish different species of Rgenetics data collections

lib/galaxy/datatypes/images.py

@@ -5,7 +5,7 @@ Image classes
 import data
 import logging
 from galaxy.datatypes.sniff import *
-from urllib import urlencode
+from urllib import urlencode, quote_plus
 import zipfile
 
 log = logging.getLogger(__name__)
@@ -123,20 +123,25 @@ def create_applet_tag_peek( class_name, archive, params ):
 class Gmaj( data.Data ):
     """Class describing a GMAJ Applet"""
     file_ext = "gmaj.zip"
+    copy_safe_peek = False
     def set_peek( self, dataset ):
         if not dataset.dataset.purged:
-            params = {
-            "bundle":"display?id=%s&tofile=yes&toext=.zip" % dataset.id,
-            "buttonlabel": "Launch GMAJ",
-            "nobutton": "false",
-            "urlpause" :"100",
-            "debug": "false",
-            "posturl": "history_add_to?%s" % urlencode( { 'history_id': dataset.history_id, 'ext': 'maf', 'name': 'GMAJ Output on data %s' % dataset.hid, 'info': 'Added by GMAJ', 'dbkey': dataset.dbkey } )
-            }
-            class_name = "edu.psu.bx.gmaj.MajApplet.class"
-            archive = "/static/gmaj/gmaj.jar"
-            dataset.peek = create_applet_tag_peek( class_name, archive, params )
-            dataset.blurb = 'GMAJ Multiple Alignment Viewer'
+            if hasattr( dataset, 'history_id' ):
+                params = {
+                "bundle":"display?id=%s&tofile=yes&toext=.zip" % dataset.id,
+                "buttonlabel": "Launch GMAJ",
+                "nobutton": "false",
+                "urlpause" :"100",
+                "debug": "false",
+                "posturl": quote_plus( "history_add_to?%s" % "&".join( [ "%s=%s" % ( key, value ) for key, value in { 'history_id': dataset.history_id, 'ext': 'maf', 'name': 'GMAJ Output on data %s' % dataset.hid, 'info': 'Added by GMAJ', 'dbkey': dataset.dbkey, 'copy_access_from': dataset.id }.items() ] ) )
+                }
+                class_name = "edu.psu.bx.gmaj.MajApplet.class"
+                archive = "/static/gmaj/gmaj.jar"
+                dataset.peek = create_applet_tag_peek( class_name, archive, params )
+                dataset.blurb = 'GMAJ Multiple Alignment Viewer'
+            else:
+                dataset.peek = "After you add this item to your history, you will be able to launch the GMAJ applet."
+                dataset.blurb = 'GMAJ Multiple Alignment Viewer'
         else:
             dataset.peek = 'file does not exist'
             dataset.blurb = 'file purged from disk'
@@ -203,19 +208,23 @@ class Html( data.Text ):
 class Laj( data.Text ):
     """Class describing a LAJ Applet"""
     file_ext = "laj"
+    copy_safe_peek = False
     def set_peek( self, dataset ):
         if not dataset.dataset.purged:
-            params = {
-            "alignfile1": "display?id=%s" % dataset.id,
-            "buttonlabel": "Launch LAJ",
-            "title": "LAJ in Galaxy",
-            "posturl": "history_add_to?%s" % urlencode( { 'history_id': dataset.history_id, 'ext': 'lav', 'name': 'LAJ Output', 'info': 'Added by LAJ', 'dbkey': dataset.dbkey } ),
-            "noseq": "true"
-            }
-            class_name = "edu.psu.cse.bio.laj.LajApplet.class"
-            archive = "/static/laj/laj.jar"
-            dataset.peek = create_applet_tag_peek( class_name, archive, params )
-            dataset.blurb = 'LAJ Multiple Alignment Viewer'
+            if hasattr( dataset, 'history_id' ):
+                params = {
+                "alignfile1": "display?id=%s" % dataset.id,
+                "buttonlabel": "Launch LAJ",
+                "title": "LAJ in Galaxy",
+                "posturl": quote_plus( "history_add_to?%s" % "&".join( [ "%s=%s" % ( key, value ) for key, value in { 'history_id': dataset.history_id, 'ext': 'lav', 'name': 'LAJ Output', 'info': 'Added by LAJ', 'dbkey': dataset.dbkey, 'copy_access_from': dataset.id }.items() ] ) ),
+                "noseq": "true"
+                }
+                class_name = "edu.psu.cse.bio.laj.LajApplet.class"
+                archive = "/static/laj/laj.jar"
+                dataset.peek = create_applet_tag_peek( class_name, archive, params )
+            else:
+                dataset.peek = "After you add this item to your history, you will be able to launch the LAJ applet."
+                dataset.blurb = 'LAJ Multiple Alignment Viewer'
         else:
             dataset.peek = 'file does not exist'
             dataset.blurb = 'file purged from disk'

lib/galaxy/datatypes/interval.py

@@ -457,7 +457,7 @@ class Gff( Tabular ):
     def __init__(self, **kwd):
         """Initialize datatype, by adding GBrowse display app"""
         Tabular.__init__(self, **kwd)
-        self.add_display_app ( 'elegans', 'display in GBrowse', 'as_gbrowse_display_file', 'gbrowse_links' )
+        self.add_display_app ( 'c_elegans', 'display in Wormbase', 'as_gbrowse_display_file', 'gbrowse_links' )
 
     def set_meta( self, dataset, overwrite = True, **kwd ):
         i = 0
@@ -879,7 +879,7 @@ class GBrowseTrack ( Tabular ):
     def __init__(self, **kwd):
         """Initialize datatype, by adding GBrowse display app"""
         Tabular.__init__(self, **kwd)
-        self.add_display_app ('elegans', 'display in GBrowse', 'as_gbrowse_display_file', 'gbrowse_links' )
+        self.add_display_app ('c_elegans', 'display in Wormbase', 'as_gbrowse_display_file', 'gbrowse_links' )
     
     def set_readonly_meta( self, dataset, skip=1, **kwd ):
         """Resets the values of readonly metadata elements."""

lib/galaxy/datatypes/metadata.py

@@ -1,14 +1,20 @@
-import sys, logging, copy, shutil, weakref
+import sys, logging, copy, shutil, weakref, cPickle, tempfile, os
 
-from galaxy.util import string_as_bool
+from galaxy.util import string_as_bool, relpath, stringify_dictionary_keys
 from galaxy.util.odict import odict
 from galaxy.web import form_builder
 import galaxy.model
 
+import pkg_resources
+pkg_resources.require("simplejson")
+import simplejson
+
 log = logging.getLogger( __name__ )
 
 STATEMENTS = "__galaxy_statements__" #this is the name of the property in a Datatype class where new metadata spec element Statements are stored
 
+DATABASE_CONNECTION_AVAILABLE = True #When False, certain metadata parameter types (see FileParameter) will behave differently
+
 class Statement( object ):
     """
     This class inserts its target into a list in the surrounding
@@ -28,7 +34,7 @@ class Statement( object ):
             statement.target( element, *args, **kwargs ) #statement.target is MetadataElementSpec, element is a Datatype class
 
 
-class MetadataCollection:
+class MetadataCollection( object ):
     """
     MetadataCollection is not a collection at all, but rather a proxy
     to the real metadata which is stored as a Dictionary. This class
@@ -90,6 +96,27 @@ class MetadataCollection:
             if key in self.spec:
                 rval[key] = self.spec[key].param.make_copy( value, target_context=self, source_context=to_copy )
         return rval
+    def from_JSON_dict( self, filename ):
+        dataset = self.parent
+        log.debug( 'loading metadata from file for: %s %s' % ( dataset.__class__.__name__, dataset.id ) )
+        JSONified_dict = simplejson.load( open( filename ) )
+        for name, spec in self.spec.items():
+            if name in JSONified_dict:
+                dataset._metadata[ name ] = spec.param.from_external_value( JSONified_dict[ name ], dataset )
+            elif name in dataset._metadata:
+                #if the metadata value is not found in our externally set metadata but it has a value in the 'old'
+                #metadata associated with our dataset, we'll delete it from our dataset's metadata dict
+                del dataset._metadata[ name ]
+    def to_JSON_dict( self, filename ):
+        #galaxy.model.customtypes.json_encoder.encode()
+        meta_dict = {}
+        dataset_meta_dict = self.parent._metadata
+        for name, spec in self.spec.items():
+            if name in dataset_meta_dict:
+                meta_dict[ name ] = spec.param.to_external_value( dataset_meta_dict[ name ] )
+        simplejson.dump( meta_dict, open( filename, 'wb+' ) )
+    def __getstate__( self ):
+        return None #cannot pickle a weakref item (self._parent), when data._metadata_collection is None, it will be recreated on demand
 
 class MetadataSpecCollection( odict ):
     """
@@ -168,6 +195,17 @@ class MetadataParameter( object ):
         """
         return value
 
+    def from_external_value( self, value, parent ):
+        """
+        Turns a value read from an external dict into its value to be pushed directly into the metadata dict.
+        """
+        return value
+    def to_external_value( self, value ):
+        """
+        Turns a value read from a metadata into its value to be pushed directly into the external dict.
+        """
+        return value
+
 class MetadataElementSpec( object ):
     """
     Defines a metadata element and adds it to the metadata_spec (which
@@ -326,14 +364,19 @@ class FileParameter( MetadataParameter ):
         return "<div>No display available for Metadata Files</div>"
 
     def wrap( self, value ):
-        if isinstance( value, galaxy.model.MetadataFile ):
+        if isinstance( value, galaxy.model.MetadataFile ) or isinstance( value, MetadataTempFile ):
             return value
-        try:
-            return galaxy.model.MetadataFile.get( value )
-        except:
-            #value was not a valid id
-            return None
-
+        if DATABASE_CONNECTION_AVAILABLE:
+            try:
+                return galaxy.model.MetadataFile.get( value )
+            except:
+                #value was not a valid id
+                return None
+        elif value is not None:
+            mf = galaxy.model.MetadataFile()
+            mf.id = value #we assume this is a valid id, since we cannot check it
+            return mf
+        return None
     def make_copy( self, value, target_context = None, source_context = None ):
         value = self.wrap( value )
         if value:
@@ -342,9 +385,164 @@ class FileParameter( MetadataParameter ):
             shutil.copy( value.file_name, new_value.file_name )
             return self.unwrap( new_value )
         return None
-
+    
     @classmethod
     def marshal( cls, value ):
         if isinstance( value, galaxy.model.MetadataFile ):
             value = value.id
         return value
+    
+    def from_external_value( self, value, parent ):
+        """
+        Turns a value read from a external dict into its value to be pushed directly into the metadata dict.
+        """
+        if MetadataTempFile.is_JSONified_value( value ):
+            value = MetadataTempFile.from_JSON( value )
+        if isinstance( value, MetadataTempFile ):
+            mf = self.new_file( dataset = parent, **value.kwds )
+            shutil.move( value.file_name, mf.file_name )
+            value = mf.id
+        return value
+    def to_external_value( self, value ):
+        """
+        Turns a value read from a metadata into its value to be pushed directly into the external dict.
+        """
+        if isinstance( value, galaxy.model.MetadataFile ):
+            value = value.id
+        elif isinstance( value, MetadataTempFile ):
+            value = MetadataTempFile.to_JSON( value )
+        return value
+    
+    def new_file( self, dataset = None, **kwds ):
+        if DATABASE_CONNECTION_AVAILABLE:
+            mf = galaxy.model.MetadataFile( name = self.spec.name, dataset = dataset, **kwds )
+            mf.flush() #flush to assign id
+            return mf
+        else:
+            #we need to make a tmp file that is accessable to the head node, 
+            #we will be copying its contents into the MetadataFile objects filename after restoring from JSON
+            #we do not include 'dataset' in the kwds passed, as from_JSON_value() will handle this for us
+            return MetadataTempFile( **kwds )
+
+#This class is used when a database file connection is not available
+class MetadataTempFile( object ):
+    tmp_dir = 'database/tmp' #this should be overwritten as necessary in calling scripts
+    def __init__( self, **kwds ):
+        self.kwds = kwds
+        self._filename = None
+    @property
+    def file_name( self ):
+        if self._filename is None:
+            #we need to create a tmp file, accessable across all nodes/heads, save the name, and return it
+            self._filename = relpath( tempfile.NamedTemporaryFile( dir = self.tmp_dir, prefix = "metadata_temp_file_" ).name )
+            open( self._filename, 'wb+' ) #create an empty file, so it can't be reused using tempfile
+        return self._filename
+    def to_JSON( self ):
+        return { 'object_type':self.__class__.__name__, 'filename':self.file_name, 'kwds':self.kwds }
+    @classmethod
+    def from_JSON( cls, json_dict ):
+        #need to ensure our keywords are not unicode
+        rval = cls( **stringify_dictionary_keys( json_dict['kwds'] ) )
+        rval._filename = json_dict['filename']
+        return rval
+    @classmethod
+    def is_JSONified_value( cls, value ):
+        return ( isinstance( value, dict ) and value.get( 'object_type', None ) == cls.__name__ )
+    @classmethod
+    def cleanup_from_JSON_dict_filename( cls, filename ):
+        try:
+            for key, value in simplejson.load( open( filename ) ).items():
+                if cls.is_JSONified_value( value ):
+                    value = cls.from_JSON( value )
+                if isinstance( value, cls ) and os.path.exists( value.file_name ):
+                    log.debug( 'Cleaning up abandoned MetadataTempFile file: %s' % value.file_name )
+                    os.unlink( value.file_name )
+        except Exception, e:
+            log.debug( 'Failed to cleanup MetadataTempFile temp files from %s: %s' % ( filename, e ) )
+
+#Class with methods allowing set_meta() to be called externally to the Galaxy head
+class JobExternalOutputMetadataWrapper( object ):
+    #this class allows access to external metadata filenames for all outputs associated with a job
+    #We will use JSON as the medium of exchange of information, except for the DatasetInstance object which will use pickle (in the future this could be JSONified as well)
+    def __init__( self, job ):
+        self.job_id = job.id
+    def get_output_filenames_by_dataset( self, dataset ):
+        if isinstance( dataset, galaxy.model.HistoryDatasetAssociation ):
+            return galaxy.model.JobExternalOutputMetadata.filter_by( job_id = self.job_id, history_dataset_association_id = dataset.id ).first() #there should only be one or None
+        elif isinstance( dataset, galaxy.model.LibraryDatasetDatasetAssociation ):
+            return galaxy.model.JobExternalOutputMetadata.filter_by( job_id = self.job_id, library_dataset_dataset_association_id = dataset.id ).first() #there should only be one or None
+        return None
+    def get_dataset_metadata_key( self, dataset ):
+        return "%s_%d" % ( dataset.__class__.__name__, dataset.id ) #set meta can be called on library items and history items, need to make different keys for them, since ids can overlap
+    def setup_external_metadata( self, datasets, exec_dir = None, tmp_dir = None, dataset_files_path = None, kwds = {} ):
+        #fill in metadata_files_dict and return the command with args required to set metadata
+        def __metadata_files_list_to_cmd_line( metadata_files ):
+            return "%s,%s,%s,%s" % ( metadata_files.filename_in, metadata_files.filename_kwds, metadata_files.filename_out, metadata_files.filename_results_code )
+        if not isinstance( datasets, list ):
+            datasets = [ datasets ]
+        if exec_dir is None:
+            exec_dir = os.path.abspath( os.getcwd() )
+        if tmp_dir is None:
+            tmp_dir = MetadataTempFile.tmp_dir
+        if dataset_files_path is None:
+            dataset_files_path = galaxy.model.Dataset.file_path
+        metadata_files_list = []
+        for dataset in datasets:
+            key = self.get_dataset_metadata_key( dataset )
+            #future note:
+            #wonkiness in job execution causes build command line to be called more than once
+            #when setting metadata externally, via 'auto-detect' button in edit attributes, etc., 
+            #we don't want to overwrite (losing the ability to cleanup) our existing dataset keys and files, 
+            #so we will only populate the dictionary once
+            metadata_files = self.get_output_filenames_by_dataset( dataset )
+            if not metadata_files:
+                metadata_files = galaxy.model.JobExternalOutputMetadata( dataset = dataset)
+                metadata_files.job_id = self.job_id
+                #we are using tempfile to create unique filenames, tempfile always returns an absolute path
+                #we will use pathnames relative to the galaxy root, to accommodate instances where the galaxy root
+                #is located differently, i.e. on a cluster node with a different filesystem structure
+                
+                #file to store existing dataset
+                metadata_files.filename_in = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_in_%s_" % key ).name )
+                cPickle.dump( dataset, open( metadata_files.filename_in, 'wb+' ) )
+                #file to store metadata results of set_meta()
+                metadata_files.filename_out = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_out_%s_" % key ).name )
+                open( metadata_files.filename_out, 'wb+' ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
+                #file to store a 'return code' indicating the results of the set_meta() call
+                #results code is like (True/False - if setting metadata was successful/failed , exception or string of reason of success/failure )
+                metadata_files.filename_results_code = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_out_%s_" % key ).name )
+                simplejson.dump( ( False, 'External set_meta() not called' ), open( metadata_files.filename_results_code, 'wb+' ) ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
+                #file to store kwds passed to set_meta()
+                metadata_files.filename_kwds = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_kwds_%s_" % key ).name )
+                simplejson.dump( kwds, open( metadata_files.filename_kwds, 'wb+' ), ensure_ascii=True )
+                metadata_files.flush()
+            metadata_files_list.append( metadata_files )
+        #return command required to build
+        return "%s %s %s %s" % ( os.path.join( exec_dir, 'set_metadata.sh' ), dataset_files_path, tmp_dir, " ".join( map( __metadata_files_list_to_cmd_line, metadata_files_list ) ) )
+    
+    def external_metadata_set_successfully( self, dataset ):
+        metadata_files = self.get_output_filenames_by_dataset( dataset )
+        if not metadata_files:
+            return False # this file doesn't exist
+        rval, rstring = simplejson.load( open( metadata_files.filename_results_code ) )
+        if not rval:
+            log.debug( 'setting metadata externally failed for %s %s: %s' % ( dataset.__class__.__name__, dataset.id, rstring ) )
+        return rval
+    
+    def cleanup_external_metadata( self ):
+        log.debug( 'Cleaning up external metadata files' )
+        for metadata_files in galaxy.model.Job.get( self.job_id ).external_output_metadata:
+            #we need to confirm that any MetadataTempFile files were removed, if not we need to remove them
+            #can occur if the job was stopped before completion, but a MetadataTempFile is used in the set_meta
+            MetadataTempFile.cleanup_from_JSON_dict_filename( metadata_files.filename_out )
+            dataset_key = self.get_dataset_metadata_key( metadata_files.dataset )
+            for key, fname in [ ( 'filename_in', metadata_files.filename_in ), ( 'filename_out', metadata_files.filename_out ), ( 'filename_results_code', metadata_files.filename_results_code ), ( 'filename_kwds', metadata_files.filename_kwds ) ]:
+                try:
+                    os.remove( fname )
+                except Exception, e:
+                    log.debug( 'Failed to cleanup external metadata file (%s) for %s: %s' % ( key, dataset_key, e ) )
+    def set_job_runner_external_pid( self, pid ):
+        for metadata_files in galaxy.model.Job.get( self.job_id ).external_output_metadata:
+            metadata_files.job_runner_external_pid = pid
+            metadata_files.flush()
+        

lib/galaxy/datatypes/qualityscore.py

@@ -32,5 +32,34 @@ class QualityScore ( data.Text ):
         except:
             return "Quality score file (%s)" % ( data.nice_size( dataset.get_size() ) )
 
-
-      
+    def sniff( self, filename ):
+        """
+        >>> fname = get_test_fname( 'sequence.fasta' )
+        >>> QualityScore().sniff( fname )
+        False
+        >>> fname = get_test_fname( 'sequence.qual' )
+        >>> QualityScore().sniff( fname )
+        True
+        """
+        try:
+            fh = open( filename )
+            while True:
+                line = fh.readline()
+                if not line:
+                    break #EOF
+                line = line.strip()
+                if line and not line.startswith( '#' ): #first non-empty non-comment line
+                    if line.startswith( '>' ):
+                        line = fh.readline().strip()
+                        if line == '' or line.startswith( '>' ):
+                            break
+                        try:
+                            [ int( x ) for x in line.split() ]
+                        except:
+                            break
+                        return True
+                    else:
+                        break #we found a non-empty line, but it's not a header
+        except:
+            pass
+        return False

lib/galaxy/datatypes/registry.py

@@ -3,7 +3,7 @@ Provides mapping between extensions and datatypes, mime-types, etc.
 """
 import os
 import logging
-import data, tabular, interval, images, sequence, qualityscore, genetics, xml
+import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks
 import galaxy.util
 from galaxy.util.odict import odict
 
@@ -94,12 +94,14 @@ class Registry( object ):
                 'bed'         : interval.Bed(), 
                 'binseq.zip'  : images.Binseq(),
                 'blastxml'    : xml.BlastXml(),
+                'coverage'    : coverage.LastzCoverage(),
                 'customtrack' : interval.CustomTrack(),
                 'csfasta'     : sequence.csFasta(),
                 'fasta'       : sequence.Fasta(),
                 'fastqsolexa' : sequence.FastqSolexa(),
                 'gff'         : interval.Gff(),
-                'gff3'        : interval.Gff3(),  
+                'gff3'        : interval.Gff3(),
+                'genetrack'   : tracks.GeneTrack(),
                 'interval'    : interval.Interval(), 
                 'laj'         : images.Laj(),
                 'lav'         : sequence.Lav(),

lib/galaxy/datatypes/sequence.py

@@ -5,6 +5,7 @@ Image classes
 import data
 import logging
 import re
+import string
 from cgi import escape
 from galaxy.datatypes.metadata import MetadataElement
 from galaxy.datatypes import metadata
@@ -102,15 +103,37 @@ class csFasta( Sequence ):
         Color-space sequence: 
             >2_15_85_F3
             T213021013012303002332212012112221222112212222
-        
-        TODO:
-            add sniff function
-        """
-        
-        return False
-        
 
-                
+        >>> fname = get_test_fname( 'sequence.fasta' )
+        >>> csFasta().sniff( fname )
+        False
+        >>> fname = get_test_fname( 'sequence.csfasta' )
+        >>> csFasta().sniff( fname )
+        True
+        """
+        try:
+            fh = open( filename )
+            while True:
+                line = fh.readline()
+                if not line:
+                    break #EOF
+                line = line.strip()
+                if line and not line.startswith( '#' ): #first non-empty non-comment line
+                    if line.startswith( '>' ):
+                        line = fh.readline().strip()
+                        if line == '' or line.startswith( '>' ):
+                            break
+                        elif line[0] not in string.ascii_uppercase:
+                            return False
+                        elif len( line ) > 1 and not re.search( '^\d+$', line[1:] ):
+                            return False
+                        return True
+                    else:
+                        break #we found a non-empty line, but it's not a header
+        except:
+            pass
+        return False
+
 class FastqSolexa( Sequence ):
     """Class representing a FASTQ sequence ( the Solexa variant )"""
     file_ext = "fastqsolexa"
@@ -231,8 +254,7 @@ class Maf( Alignment ):
                 tmp_file.write( "%s\t%s\n" % ( spec, "\t".join( chroms ) ) )
             
             if not chrom_file:
-                chrom_file = galaxy.model.MetadataFile( dataset = dataset, name = "species_chromosomes" )
-                chrom_file.flush()
+                chrom_file = dataset.metadata.spec['species_chromosomes'].param.new_file( dataset = dataset )
             tmp_file.seek( 0 )
             open( chrom_file.file_name, 'wb' ).write( tmp_file.read() )
             dataset.metadata.species_chromosomes = chrom_file
@@ -240,8 +262,7 @@ class Maf( Alignment ):
         
         index_file = dataset.metadata.maf_index
         if not index_file:
-            index_file = galaxy.model.MetadataFile( dataset = dataset, name="maf_index" )
-            index_file.flush()
+            index_file = dataset.metadata.spec['maf_index'].param.new_file( dataset = dataset )
         indexes.write( open( index_file.file_name, 'w' ) )
         dataset.metadata.maf_index = index_file
     

lib/galaxy/datatypes/sniff.py

@@ -25,6 +25,22 @@ def stream_to_file( stream, suffix='', prefix='', dir=None, text=False ):
     os.close(fd)
     return temp_name
 
+def check_newlines( fname, bytes_to_read=52428800 ):
+    """
+    Determines if there are any non-POSIX newlines in the first
+    number_of_bytes (by default, 50MB) of the file.
+    """
+    CHUNK_SIZE = 2 ** 20
+    f = open( fname, 'r' )
+    for chunk in f.read( CHUNK_SIZE ):
+        if f.tell() > bytes_to_read:
+            break
+        if chunk.count( '\r' ):
+            f.close()
+            return True
+    f.close()
+    return False
+
 def convert_newlines( fname ):
     """
     Converts in place a file from universal line endings 

lib/galaxy/datatypes/test/sequence.csfasta

@@ -0,0 +1,21 @@
+#comment
+>2_14_26_F3,-1282216.0
+T011213122200221123032111221021210131332222101
+>2_14_192_F3,-1383225.3
+T110021221100310030120022032222111321022112223
+>2_14_233_F3,-1082751.1
+T011001332311121212312022310203312201132111223
+>2_14_294_F3,-687179.1
+T213012132300000021323212232103300033102330332
+>2_14_463_F3
+T132032030200202202003211302222202230022110222
+>2_14_578_F3
+T131013032310120222321211010130110221312110222
+>2_14_956_F3,-1625621.2,-1625360.0
+T210213030022120032001012021321220011232201231
+>2_14_988_F3,1687674.3
+T221202031310031102033002302330301301010023133
+>2_14_1028_F3,754444.2
+T112230301101101120201331111302110031102111321
+>2_14_1035_F3,-1570954.1
+T003033103303232110201102100032203301023110332

lib/galaxy/datatypes/test/sequence.qual

@@ -0,0 +1,9 @@
+#comment
+>920_14_164_F3
+4 13 18 14 13 14 16 19 22 16 8 16 9 16 6 5 13 13 10 4 6 8 11 6 5 2 20 14 10 3 2 28 6 6 24 
+>920_14_977_F3
+8 10 2 2 2 2 4 3 4 7 3 2 5 2 7 4 5 2 3 3 6 3 3 6 9 2 2 10 3 4 2 2 2 5 5 
+>920_15_315_F3
+7 7 2 2 2 2 3 2 2 2 4 2 2 4 2 3 2 2 2 2 3 4 4 2 2 5 2 3 2 2 2 3 2 2 6 
+>920_16_347_F3
+6 7 3 2 4 3 2 2 2 2 2 2 2 3 3 3 3 2 3 2 3 2 5 3 4 4 2 4 4 3 3 2 4 2 2 

lib/galaxy/datatypes/tracks.py

@@ -0,0 +1,30 @@
+"""
+Datatype classes for tracks/track views within galaxy.
+"""
+
+import data
+import logging
+import re
+from cgi import escape
+from galaxy.datatypes.metadata import MetadataElement
+from galaxy.datatypes import metadata
+import galaxy.model
+from galaxy import util
+from galaxy.web import url_for
+from sniff import *
+
+log = logging.getLogger(__name__)
+
+class GeneTrack( data.Binary ):
+    file_ext = "genetrack"
+    
+    MetadataElement( name="hdf", default="data.hdf", desc="HDF DB", readonly=True, visible=True, no_value=0 )
+    MetadataElement( name="sqlite", default="features.sqlite", desc="SQLite Features DB", readonly=True, visible=True, no_value=0 )
+    MetadataElement( name="label", default="Custom", desc="Track Label", readonly=True, visible=True, no_value="Custom" )
+    
+    def __init__(self, **kwargs):
+        super(GeneTrack, self).__init__(**kwargs)
+        self.add_display_app( 'genetrack', 'View in ', '', 'genetrack_link' )
+    
+    def genetrack_link( self, dataset, type, app, base_url ):
+        return [('GeneTrack', url_for(controller='genetrack', action='index', dataset_id=dataset.id ))]

lib/galaxy/jobs/__init__.py

@@ -4,6 +4,7 @@ from galaxy import util, model
 from galaxy.model import mapping
 from galaxy.datatypes.tabular import *
 from galaxy.datatypes.interval import *
+from galaxy.datatypes import metadata
 
 import pkg_resources
 pkg_resources.require( "PasteDeploy" )
@@ -306,6 +307,7 @@ class JobWrapper( object ):
         self.working_directory = \
             os.path.join( self.app.config.job_working_directory, str( self.job_id ) )
         self.output_paths = None
+        self.external_output_metadata = metadata.JobExternalOutputMetadataWrapper( job ) #wrapper holding the info required to restore and clean up from files used for setting metadata externally
         
     def get_param_dict( self ):
         """
@@ -462,6 +464,7 @@ class JobWrapper( object ):
                     self.fail( "Job %s's output dataset(s) could not be read" % job.id )
                     return
         for dataset_assoc in job.output_datasets:
+            #should this also be checking library associations? - can a library item be added from a history before the job has ended? - lets not allow this to occur
             for dataset in dataset_assoc.dataset.dataset.history_associations: #need to update all associated output hdas, i.e. history was shared with job running
                 dataset.blurb = 'done'
                 dataset.peek  = 'no peek'
@@ -470,13 +473,25 @@ class JobWrapper( object ):
                 if stderr:
                     dataset.blurb = "error"
                 elif dataset.has_data():
-                    # Only set metadata values if they are missing...
-                    dataset.set_meta( overwrite = False )
+                    #if a dataset was copied, it won't appear in our dictionary:
+                    #either use the metadata from originating output dataset, or call set_meta on the copies
+                    #it would be quicker to just copy the metadata from the originating output dataset, 
+                    #but somewhat trickier (need to recurse up the copied_from tree), for now we'll call set_meta()
+                    if not self.external_output_metadata.external_metadata_set_successfully( dataset ):
+                        # Only set metadata values if they are missing...
+                        dataset.set_meta( overwrite = False )
+                    else:
+                        #load metadata from file
+                        #we need to no longer allow metadata to be edited while the job is still running,
+                        #since if it is edited, the metadata changed on the running output will no longer match
+                        #the metadata that was stored to disk for use via the external process, 
+                        #and the changes made by the user will be lost, without warning or notice
+                        dataset.metadata.from_JSON_dict( self.external_output_metadata.get_output_filenames_by_dataset( dataset ).filename_out )
                     dataset.set_peek()
                 else:
                     dataset.blurb = "empty"
                 dataset.flush()
-            if stderr: 
+            if stderr:
                 dataset_assoc.dataset.dataset.state = model.Dataset.states.ERROR
             else:
                 dataset_assoc.dataset.dataset.state = model.Dataset.states.OK
@@ -517,10 +532,12 @@ class JobWrapper( object ):
     def cleanup( self ):
         # remove temporary files
         try:
-            for fname in self.extra_filenames: 
+            for fname in self.extra_filenames:
                 os.remove( fname )
             if self.working_directory is not None:
                 shutil.rmtree( self.working_directory )
+            if self.app.config.set_metadata_externally:
+                self.external_output_metadata.cleanup_external_metadata()
         except:
             log.exception( "Unable to cleanup job %d" % self.job_id )
         
@@ -573,7 +590,15 @@ class JobWrapper( object ):
         for outfile in [ str( o ) for o in output_paths ]:
             sizes.append( ( outfile, os.stat( outfile ).st_size ) )
         return sizes
-        
+    def setup_external_metadata( self, exec_dir = None, tmp_dir = None, dataset_files_path = None, **kwds ):
+        if tmp_dir is None:
+            #this dir should should relative to the exec_dir
+            tmp_dir = self.app.config.new_file_path
+        if dataset_files_path is None:
+            dataset_files_path = self.app.model.Dataset.file_path
+        job = model.Job.get( self.job_id )
+        return self.external_output_metadata.setup_external_metadata( [ output_dataset_assoc.dataset for output_dataset_assoc in job.output_datasets ], exec_dir = exec_dir, tmp_dir = tmp_dir, dataset_files_path = dataset_files_path, **kwds )
+
 class DefaultJobDispatcher( object ):
     def __init__( self, app ):
         self.app = app

lib/galaxy/jobs/runners/local.py

@@ -99,6 +99,21 @@ class LocalJobRunner( object ):
                 job_wrapper.fail( "failure running job", exception=True )
                 log.exception("failure running job %d" % job_wrapper.job_id)
                 return
+        
+        #run the metadata setting script here
+        #this is terminatable when output dataset/job is deleted
+        #so that long running set_meta()s can be cancelled without having to reboot the server
+        if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and self.app.config.set_metadata_externally:
+            external_metadata_script = job_wrapper.setup_external_metadata( kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
+            log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) )
+            external_metadata_proc = subprocess.Popen( args = external_metadata_script, 
+                                         shell = True, 
+                                         env = env,
+                                         preexec_fn = os.setpgrp )
+            job_wrapper.external_output_metadata.set_job_runner_external_pid( external_metadata_proc.pid )
+            external_metadata_proc.wait()
+            log.debug( 'execution of external set_meta finished for job %d' % job_wrapper.job_id )
+        
         # Finish the job                
         try:
             job_wrapper.finish( stdout, stderr )
@@ -131,12 +146,17 @@ class LocalJobRunner( object ):
             return False
 
     def stop_job( self, job ):
-        if job.job_runner_external_id is None:
+        #if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished
+        if job.external_output_metadata:
+            pid = job.external_output_metadata[0].job_runner_external_pid #every JobExternalOutputMetadata has a pid set, we just need to take from one of them
+        else:
+            pid = job.job_runner_external_id
+        if pid in [ None, '' ]:
             log.warning( "stop_job(): %s: no PID in database for job, unable to stop" % job.id )
             return
-        pid = int( job.job_runner_external_id )
+        pid = int( pid )
         if not self.check_pid( pid ):
-            log.warning( "stop_job(): %s: PID %d was already dead or can't be signaled" %job.id )
+            log.warning( "stop_job(): %s: PID %d was already dead or can't be signaled" % ( job.id, pid ) )
             return
         for sig in [ 15, 9 ]:
             try:

lib/galaxy/jobs/runners/pbs.py

@@ -27,6 +27,7 @@ if [ "$GALAXY_LIB" != "None" ]; then
 fi
 cd %s
 %s
+%s
 """
 
 pbs_symlink_template = """#!/bin/sh
@@ -208,7 +209,11 @@ class PBSJobRunner( object ):
         if self.app.config.pbs_stage_path != '':
             script = pbs_symlink_template % (job_wrapper.galaxy_lib_dir, " ".join(job_wrapper.get_input_fnames() + job_wrapper.get_output_fnames()), self.app.config.pbs_stage_path, exec_dir, command_line)
         else:
-            script = pbs_template % (job_wrapper.galaxy_lib_dir, exec_dir, command_line)
+            if self.app.config.set_metadata_externally:
+                external_metadata_script = job_wrapper.setup_external_metadata( exec_dir = exec_dir, tmp_dir = self.app.config.new_file_path, dataset_files_path = self.app.model.Dataset.file_path, kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
+            else:
+                external_metadata_script = ""
+            script = pbs_template % ( job_wrapper.galaxy_lib_dir, exec_dir, command_line, external_metadata_script )
         job_file = "%s/%s.sh" % (self.app.config.cluster_files_directory, job_wrapper.job_id)
         fh = file(job_file, "w")
         fh.write(script)

lib/galaxy/model/__init__.py

@@ -13,6 +13,7 @@ from galaxy import util
 import tempfile
 import galaxy.datatypes.registry
 from galaxy.datatypes.metadata import MetadataCollection
+from galaxy.security import RBACAgent, get_permitted_actions
 
 import logging
 log = logging.getLogger( __name__ )
@@ -31,15 +32,25 @@ class User( object ):
         self.email = email
         self.password = password
         self.external = False
+        self.deleted = False
+        self.purged = False
         # Relationships
         self.histories = []
+        
     def set_password_cleartext( self, cleartext ):
         """Set 'self.password' to the digest of 'cleartext'."""
         self.password = sha.new( cleartext ).hexdigest()
     def check_password( self, cleartext ):
         """Check if 'cleartext' matches 'self.password' when hashed."""
         return self.password == sha.new( cleartext ).hexdigest()
-        
+    def all_roles( self ):
+        roles = [ ura.role for ura in self.roles ]
+        for group in [ uga.group for uga in self.groups ]:
+            for role in [ gra.role for gra in group.roles ]:
+                if role not in roles:
+                    roles.append( role )
+        return roles
+    
 class Job( object ):
     """
     A job represents a request to run a tool given input datasets, tool 
@@ -130,13 +141,290 @@ class JobToOutputDatasetAssociation( object ):
         self.name = name
         self.dataset = dataset
 
-class HistoryDatasetAssociation( object ):
+class JobExternalOutputMetadata( object ):
+    def __init__( self, job = None, dataset = None ):
+        self.job = job
+        if isinstance( dataset, galaxy.model.HistoryDatasetAssociation ):
+            self.history_dataset_association = dataset
+        elif isinstance( dataset, galaxy.model.LibraryDatasetDatasetAssociation ):
+            self.library_dataset_dataset_association = dataset
+    @property
+    def dataset( self ):
+        if self.history_dataset_association:
+            return self.history_dataset_association
+        elif self.library_dataset_dataset_association:
+            return self.library_dataset_dataset_association
+        return None
+
+class Group( object ):
+    def __init__( self, name = None ):
+        self.name = name
+        self.deleted = False
+
+class UserGroupAssociation( object ):
+    def __init__( self, user, group ):
+        self.user = user
+        self.group = group
+
+class History( object ):
+    def __init__( self, id=None, name=None, user=None ):
+        self.id = id
+        self.name = name or "Unnamed history"
+        self.deleted = False
+        self.purged = False
+        self.genome_build = None
+        # Relationships
+        self.user = user
+        self.datasets = []
+        self.galaxy_sessions = []
+    def _next_hid( self ):
+        # TODO: override this with something in the database that ensures 
+        # better integrity
+        if len( self.datasets ) == 0:
+            return 1
+        else:
+            last_hid = 0
+            for dataset in self.datasets:
+                if dataset.hid > last_hid:
+                    last_hid = dataset.hid
+            return last_hid + 1
+    def add_galaxy_session( self, galaxy_session, association=None ):
+        if association is None:
+            self.galaxy_sessions.append( GalaxySessionToHistoryAssociation( galaxy_session, self ) )
+        else:
+            self.galaxy_sessions.append( association )
+    def add_dataset( self, dataset, parent_id=None, genome_build=None, set_hid = True ):
+        if isinstance( dataset, Dataset ):
+            dataset = HistoryDatasetAssociation( dataset = dataset, copied_from = dataset )
+            dataset.flush()
+        elif not isinstance( dataset, HistoryDatasetAssociation ):
+            raise TypeError, "You can only add Dataset and HistoryDatasetAssociation instances to a history ( you tried to add %s )." % str( dataset )
+        if parent_id:
+            for data in self.datasets:
+                if data.id == parent_id:
+                    dataset.hid = data.hid
+                    break
+            else:
+                if set_hid:
+                    dataset.hid = self._next_hid()
+        else:
+            if set_hid:
+                dataset.hid = self._next_hid()
+        dataset.history = self
+        if genome_build not in [None, '?']:
+            self.genome_build = genome_build
+        self.datasets.append( dataset )
+    def copy( self, target_user = None ):
+        if not target_user:
+            target_user = self.user
+        des = History( user = target_user )
+        des.flush()
+        des.name = self.name
+        for data in self.datasets:
+            new_data = data.copy( copy_children = True, target_history = des )
+            des.add_dataset( new_data, set_hid = False )
+            new_data.flush()
+        des.hid_counter = self.hid_counter
+        des.flush()
+        return des
+    @property
+    def activatable_datasets( self ):
+        return [ hda for hda in self.datasets if not hda.dataset.purged ] #this needs to be a list
+
+class UserRoleAssociation( object ):
+    def __init__( self, user, role ):
+        self.user = user
+        self.role = role
+
+class GroupRoleAssociation( object ):
+    def __init__( self, group, role ):
+        self.group = group
+        self.role = role
+
+class Role( object ):
+    private_id = None
+    types = Bunch( 
+        PRIVATE = 'private',
+        SYSTEM = 'system',
+        USER = 'user',
+        ADMIN = 'admin',
+        SHARING = 'sharing'
+    )
+    def __init__( self, name="", description="", type="system", deleted=False ):
+        self.name = name
+        self.description = description
+        self.type = type
+        self.deleted = deleted
+
+class DatasetPermissions( object ):
+    def __init__( self, action, dataset, role ):
+        self.action = action
+        self.dataset = dataset
+        self.role = role
+
+class LibraryPermissions( object ):
+    def __init__( self, action, library_item, role ):
+        self.action = action
+        if isinstance( library_item, Library ):
+            self.library = library_item
+        else:
+            raise "Invalid Library specified: %s" % library_item.__class__.__name__
+        self.role = role
+
+class LibraryFolderPermissions( object ):
+    def __init__( self, action, library_item, role ):
+        self.action = action
+        if isinstance( library_item, LibraryFolder ):
+            self.folder = library_item
+        else:
+            raise "Invalid LibraryFolder specified: %s" % library_item.__class__.__name__
+        self.role = role
+
+class LibraryDatasetPermissions( object ):
+    def __init__( self, action, library_item, role ):
+        self.action = action
+        if isinstance( library_item, LibraryDataset ):
+            self.library_dataset = library_item
+        else:
+            raise "Invalid LibraryDataset specified: %s" % library_item.__class__.__name__
+        self.role = role
+
+class LibraryDatasetDatasetAssociationPermissions( object ):
+    def __init__( self, action, library_item, role ):
+        self.action = action
+        if isinstance( library_item, LibraryDatasetDatasetAssociation ):
+            self.library_dataset_dataset_association = library_item
+        else:
+            raise "Invalid LibraryDatasetDatasetAssociation specified: %s" % library_item.__class__.__name__
+        self.role = role
+
+class LibraryItemInfoPermissions( object ):
+    def __init__( self, action, library_item, role ):
+        self.action = action
+        if isinstance( library_item, LibraryItemInfo ):
+            self.library_item_info = library_item
+        else:
+            raise "Invalid LibraryItemInfo specified: %s" % library_item.__class__.__name__
+        self.role = role
+
+class LibraryItemInfoTemplatePermissions( object ):
+    def __init__( self, action, library_item, role ):
+        self.action = action
+        if isinstance( library_item, LibraryItemInfoTemplate ):
+            self.library_item_info_template = library_item
+        else:
+            raise "Invalid LibraryItemInfoTemplate specified: %s" % library_item.__class__.__name__
+        self.role = role
+
+class DefaultUserPermissions( object ):
+    def __init__( self, user, action, role ):
+        self.user = user
+        self.action = action
+        self.role = role
+
+class DefaultHistoryPermissions( object ):
+    def __init__( self, history, action, role ):
+        self.history = history
+        self.action = action
+        self.role = role
+
+class Dataset( object ):
+    states = Bunch( NEW = 'new',
+                    QUEUED = 'queued',
+                    RUNNING = 'running',
+                    OK = 'ok',
+                    EMPTY = 'empty',
+                    ERROR = 'error',
+                    DISCARDED = 'discarded' )
+    permitted_actions = get_permitted_actions( filter='DATASET' )
+    file_path = "/tmp/"
+    engine = None
+    def __init__( self, id=None, state=None, external_filename=None, extra_files_path=None, file_size=None, purgable=True ):
+        self.id = id
+        self.state = state
+        self.deleted = False
+        self.purged = False
+        self.purgable = purgable
+        self.external_filename = external_filename
+        self._extra_files_path = extra_files_path
+        self.file_size = file_size
+    def get_file_name( self ):
+        if not self.external_filename:
+            assert self.id is not None, "ID must be set before filename used (commit the object)"
+            # First try filename directly under file_path
+            filename = os.path.join( self.file_path, "dataset_%d.dat" % self.id )
+            # Only use that filename if it already exists (backward compatibility),
+            # otherwise construct hashed path
+            if not os.path.exists( filename ):
+                dir = os.path.join( self.file_path, *directory_hash_id( self.id ) )
+                # Create directory if it does not exist
+                try:
+                    os.makedirs( dir )
+                except OSError, e:
+                    # File Exists is okay, otherwise reraise
+                    if e.errno != errno.EEXIST:
+                        raise
+                # Return filename inside hashed directory
+                return os.path.abspath( os.path.join( dir, "dataset_%d.dat" % self.id ) )
+        else:
+            filename = self.external_filename
+        # Make filename absolute
+        return os.path.abspath( filename )
+    def set_file_name ( self, filename ):
+        if not filename:
+            self.external_filename = None
+        else:
+            self.external_filename = filename
+    file_name = property( get_file_name, set_file_name )
+    @property
+    def extra_files_path( self ):
+        if self._extra_files_path: 
+            path = self._extra_files_path
+        else:
+            path = os.path.join( self.file_path, "dataset_%d_files" % self.id )
+            #only use path directly under self.file_path if it exists
+            if not os.path.exists( path ):
+                path = os.path.join( os.path.join( self.file_path, *directory_hash_id( self.id ) ), "dataset_%d_files" % self.id )
+        # Make path absolute
+        return os.path.abspath( path )
+    def get_size( self ):
+        """Returns the size of the data on disk"""
+        if self.file_size:
+            return self.file_size
+        else:
+            try:
+                return os.path.getsize( self.file_name )
+            except OSError:
+                return 0
+    def set_size( self ):
+        """Returns the size of the data on disk"""
+        try:
+            if not self.file_size:
+                self.file_size = os.path.getsize( self.file_name )
+        except OSError:
+            self.file_size = 0
+    def has_data( self ):
+        """Detects whether there is any data"""
+        return self.get_size() > 0
+    def mark_deleted( self, include_children=True ):
+        self.deleted = True
+    # FIXME: sqlalchemy will replace this
+    def _delete(self):
+        """Remove the file that corresponds to this data"""
+        try:
+            os.remove(self.data.file_name)
+        except OSError, e:
+            log.critical('%s delete error %s' % (self.__class__.__name__, e))
+
+class DatasetInstance( object ):
+    """A base class for all 'dataset instances', HDAs, LDAs, etc"""
+    states = Dataset.states
+    permitted_actions = Dataset.permitted_actions
     def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None, 
                   dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None,
-                  parent_id=None, copied_from_history_dataset_association = None, validation_errors=None, visible=True, create_dataset = False ):
+                  parent_id=None, validation_errors=None, visible=True, create_dataset = False ):
         self.name = name or "Unnamed dataset"
         self.id = id
-        self.hid = hid
         self.info = info
         self.blurb = blurb
         self.peek = peek
@@ -148,46 +436,32 @@ class HistoryDatasetAssociation( object ):
         self.deleted = deleted
         self.visible = visible
         # Relationships
-        self.history = history
         if not dataset and create_dataset:
-            dataset = Dataset()
+            dataset = Dataset( state=Dataset.states.NEW )
             dataset.flush()
         self.dataset = dataset
         self.parent_id = parent_id
         self.validation_errors = validation_errors
-        self.copied_from_history_dataset_association = copied_from_history_dataset_association
-    
     @property
     def ext( self ):
         return self.extension
-    
-    @property
-    def states( self ):
-        return self.dataset.states
-    
     def get_dataset_state( self ):
         return self.dataset.state
     def set_dataset_state ( self, state ):
         self.dataset.state = state
         self.dataset.flush() #flush here, because hda.flush() won't flush the Dataset object
     state = property( get_dataset_state, set_dataset_state )
-    
     def get_file_name( self ):
         return self.dataset.get_file_name()
-            
     def set_file_name (self, filename):
         return self.dataset.set_file_name( filename )
-        
     file_name = property( get_file_name, set_file_name )
-    
     @property
     def extra_files_path( self ):
         return self.dataset.extra_files_path
-    
     @property
     def datatype( self ):
         return datatypes_registry.get_datatype_by_extension( self.extension )
-
     def get_metadata( self ):
         if not hasattr( self, '_metadata_collection' ) or self._metadata_collection.parent != self: #using weakref to store parent (to prevent circ ref), does a Session.clear() cause parent to be invalidated, while still copying over this non-database attribute?
             self._metadata_collection = MetadataCollection( self )
@@ -196,15 +470,11 @@ class HistoryDatasetAssociation( object ):
         # Needs to accept a MetadataCollection, a bunch, or a dict
         self._metadata = self.metadata.make_dict_copy( bunch )
     metadata = property( get_metadata, set_metadata )
-
-    """
-    This provide backwards compatibility with using the old dbkey
-    field in the database.  That field now maps to "old_dbkey" (see mapping.py).
-    """
+    # This provide backwards compatibility with using the old dbkey
+    # field in the database.  That field now maps to "old_dbkey" (see mapping.py).
     def get_dbkey( self ):
         dbkey = self.metadata.dbkey
         if not isinstance(dbkey, list): dbkey = [dbkey]
-        #if dbkey in [["?"], [None], []]: dbkey = [self.old_dbkey]
         if dbkey in [[None], []]: return "?"
         return dbkey[0]
     def set_dbkey( self, value ):
@@ -213,12 +483,7 @@ class HistoryDatasetAssociation( object ):
                 self.metadata.dbkey = [value]
             else: 
                 self.metadata.dbkey = value
-        #if isinstance(value, list): 
-        #    self.old_dbkey = value[0]
-        #else:
-        #    self.old_dbkey = value
     dbkey = property( get_dbkey, set_dbkey )
-
     def change_datatype( self, new_ext ):
         self.clear_associated_files()
         datatypes_registry.change_datatype( self, new_ext )
@@ -269,59 +534,26 @@ class HistoryDatasetAssociation( object ):
                 valid.append( assoc.dataset )
         return valid
     def clear_associated_files( self, metadata_safe = False, purge = False ):
-        #metadata_safe = True means to only clear when assoc.metadata_safe == False
-        for assoc in self.implicitly_converted_datasets:
-            if not metadata_safe or not assoc.metadata_safe:
-                assoc.clear( purge = purge )
+        raise 'Unimplemented'
     def get_child_by_designation(self, designation):
         for child in self.children:
             if child.designation == designation:
                 return child
         return None
-
     def get_converter_types(self):
         return self.datatype.get_converter_types( self, datatypes_registry)
-    
     def find_conversion_destination( self, accepted_formats, **kwd ):
         """Returns ( target_ext, exisiting converted dataset )"""
         return self.datatype.find_conversion_destination( self, accepted_formats, datatypes_registry, **kwd )
-    
-    def copy( self, copy_children = False, parent_id = None ):
-        des = HistoryDatasetAssociation( hid=self.hid,
-                                         name=self.name,
-                                         info=self.info,
-                                         blurb=self.blurb,
-                                         peek=self.peek,
-                                         extension=self.extension,
-                                         dbkey=self.dbkey,
-                                         dataset=self.dataset,
-                                         visible=self.visible,
-                                         deleted=self.deleted,
-                                         parent_id=parent_id,
-                                         copied_from_history_dataset_association=self )
-        des.flush()
-        des.set_size()
-        des.metadata = self.metadata #need to set after flushed, as MetadataFiles require dataset.id
-        if copy_children:
-            for child in self.children:
-                child_copy = child.copy( copy_children = copy_children, parent_id = des.id )
-        # In some instances peek relies on dataset_id ( e.g., gmaj.zip for viewing MAFs )
-        des.set_peek()
-        des.flush()
-        return des
-
     def add_validation_error( self, validation_error ):
         self.validation_errors.append( validation_error )
-
     def extend_validation_errors( self, validation_errors ):
         self.validation_errors.extend(validation_errors)
-
     def mark_deleted( self, include_children=True ):
         self.deleted = True
         if include_children:
             for child in self.children:
                 child.mark_deleted()
-
     def mark_undeleted( self, include_children=True ):
         self.deleted = False
         if include_children:
@@ -331,174 +563,311 @@ class HistoryDatasetAssociation( object ):
         if self.purged:
             return False
         return True
+    @property
+    def source_library_dataset( self ):
+        def get_source( dataset ):
+            if isinstance( dataset, LibraryDatasetDatasetAssociation ):
+                if dataset.library_dataset:
+                    return ( dataset, dataset.library_dataset )
+            if dataset.copied_from_library_dataset_dataset_association:
+                source = get_source( dataset.copied_from_library_dataset_dataset_association )
+                if source:
+                    return source
+            if dataset.copied_from_history_dataset_association:
+                source = get_source( dataset.copied_from_history_dataset_association )
+                if source:
+                    return source
+            return ( None, None )
+        return get_source( self )
 
-class History( object ):
-    def __init__( self, id=None, name=None, user=None ):
-        self.id = id
-        self.name = name or "Unnamed history"
-        self.deleted = False
-        self.purged = False
-        self.genome_build = None
+class HistoryDatasetAssociation( DatasetInstance ):
+    def __init__( self, 
+                  hid = None, 
+                  history = None, 
+                  copied_from_history_dataset_association = None, 
+                  copied_from_library_dataset_dataset_association = None, 
+                  **kwd ):
+        DatasetInstance.__init__( self, **kwd )
+        self.hid = hid
         # Relationships
-        self.user = user
-        self.datasets = []
-        self.galaxy_sessions = []
-        
-    def _next_hid( self ):
-        # TODO: override this with something in the database that ensures 
-        # better integrity
-        if len( self.datasets ) == 0:
-            return 1
-        else:
-            last_hid = 0
-            for dataset in self.datasets:
-                if dataset.hid > last_hid:
-                    last_hid = dataset.hid
-            return last_hid + 1
+        self.history = history
+        self.copied_from_history_dataset_association = copied_from_history_dataset_association
+        self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association
+    def copy( self, copy_children = False, parent_id = None, target_history = None ):
+        hda = HistoryDatasetAssociation( hid=self.hid, 
+                                         name=self.name, 
+                                         info=self.info, 
+                                         blurb=self.blurb, 
+                                         peek=self.peek, 
+                                         extension=self.extension, 
+                                         dbkey=self.dbkey, 
+                                         dataset = self.dataset, 
+                                         visible=self.visible, 
+                                         deleted=self.deleted, 
+                                         parent_id=parent_id, 
+                                         copied_from_history_dataset_association=self,
+                                         history = target_history )
+        hda.flush()
+        hda.set_size()
+        hda.metadata = self.metadata #need to set after flushed, as MetadataFiles require dataset.id
+        if copy_children:
+            for child in self.children:
+                child_copy = child.copy( copy_children = copy_children, parent_id = hda.id )
+        if not self.datatype.copy_safe_peek:
+            hda.set_peek() #in some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
+        hda.flush()
+        return hda
+    def to_library_dataset_dataset_association( self, target_folder, parent_id=None ):
+        # Create e new LibraryDataset
+        library_dataset = LibraryDataset( folder=target_folder, name=self.name, info=self.info )
+        library_dataset.flush()
+        ldda = LibraryDatasetDatasetAssociation( name=self.name, 
+                                                 info=self.info,
+                                                 blurb=self.blurb, 
+                                                 peek=self.peek, 
+                                                 extension=self.extension, 
+                                                 dbkey=self.dbkey, 
+                                                 dataset=self.dataset, 
+                                                 library_dataset=library_dataset,
+                                                 visible=self.visible, 
+                                                 deleted=self.deleted, 
+                                                 parent_id=parent_id,
+                                                 copied_from_history_dataset_association=self )
+        ldda.flush()
+        # Must set metadata after flushed, as MetadataFiles require dataset.id
+        ldda.metadata = self.metadata
+        library_dataset.library_dataset_dataset_association_id = ldda.id
+        library_dataset.flush()
+        target_folder.add_library_dataset( library_dataset, genome_build=ldda.dbkey )
+        for child in self.children:
+            child_copy = child.to_library_dataset_dataset_association( target_folder=target_folder, parent_id=ldda.id )
+        if not self.datatype.copy_safe_peek:
+            ldda.set_peek() #in some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
+        ldda.flush()
+        return ldda
+    def clear_associated_files( self, metadata_safe = False, purge = False ):
+        #metadata_safe = True means to only clear when assoc.metadata_safe == False
+        for assoc in self.implicitly_converted_datasets:
+            if not metadata_safe or not assoc.metadata_safe:
+                assoc.clear( purge = purge )
 
-    def add_galaxy_session( self, galaxy_session, association=None ):
-        if association is None:
-            self.galaxy_sessions.append( GalaxySessionToHistoryAssociation( galaxy_session, self ) )
-        else:
-            self.galaxy_sessions.append( association )
+class Library( object ):
+    permitted_actions = get_permitted_actions( filter='LIBRARY' )
+    def __init__( self, name = None, description = None, root_folder = None ):
+        self.name = name or "Unnamed library"
+        self.description = description
+        self.root_folder = root_folder
+    def get_library_item_info_templates( self, template_list=[], restrict=False ):
+        if self.library_info_template_associations:
+            template_list.extend( [ lita.library_item_info_template for lita in self.library_info_template_associations if lita.library_item_info_template not in template_list ] )
+        return template_list
 
-    def add_dataset( self, dataset, parent_id=None, genome_build=None, set_hid = True ):
-        if isinstance( dataset, Dataset ):
-            dataset = HistoryDatasetAssociation( dataset = dataset )
-            dataset.flush()
-        elif not isinstance( dataset, HistoryDatasetAssociation ):
-            raise TypeError, "You can only add Dataset and HistoryDatasetAssociation instances to a history ( you tried to add %s )." % str( dataset )
-        if parent_id:
-            for data in self.datasets:
-                if data.id == parent_id:
-                    dataset.hid = data.hid
-                    break
-            else:
-                if set_hid:
-                    dataset.hid = self._next_hid()
-        else:
-            if set_hid:
-                dataset.hid = self._next_hid()
+class LibraryFolder( object ):
+    def __init__( self, name=None, description=None, item_count=0, order_id=None ):
+        self.name = name or "Unnamed folder"
+        self.description = description
+        self.item_count = item_count
+        self.order_id = order_id
+        self.genome_build = None
+    def add_library_dataset( self, library_dataset, genome_build=None ):
+        library_dataset.folder_id = self.id
+        library_dataset.order_id = self.item_count
+        self.item_count += 1
         if genome_build not in [None, '?']:
             self.genome_build = genome_build
-        self.datasets.append( dataset )
-
-    def copy(self):
-        des = History()
-        des.flush()
-        des.name = self.name
-        des.user_id = self.user_id
-        for data in self.datasets:
-            new_data = data.copy( copy_children = True )
-            des.add_dataset( new_data )
-            new_data.flush()
-        des.hid_counter = self.hid_counter
-        des.flush()
-        return des
-
-# class Query( object ):
-#     def __init__( self, name=None, state=None, tool_parameters=None, history=None ):
-#         self.name = name or "Unnamed query"
-#         self.state = state
-#         self.tool_parameters = tool_parameters
-#         # Relationships
-#         self.history = history
-#         self.datasets = []
-
-class Dataset( object ):
-    states = Bunch( NEW = 'new',
-                    QUEUED = 'queued',
-                    RUNNING = 'running',
-                    OK = 'ok',
-                    EMPTY = 'empty',
-                    ERROR = 'error',
-                    DISCARDED = 'discarded' )
-    file_path = "/tmp/"
-    engine = None
-    def __init__( self, id=None, state=None, external_filename=None, extra_files_path=None, file_size=None, purgable=True ):
-        self.id = id
-        self.state = state
-        self.deleted = False
-        self.purged = False
-        self.purgable = purgable
-        self.external_filename = external_filename
-        self._extra_files_path = extra_files_path
-        self.file_size = file_size
-        
-    def get_file_name( self ):
-        if not self.external_filename:
-            assert self.id is not None, "ID must be set before filename used (commit the object)"
-            # First try filename directly under file_path
-            filename = os.path.join( self.file_path, "dataset_%d.dat" % self.id )
-            # Only use that filename if it already exists (backward compatibility),
-            # otherwise construct hashed path
-            if not os.path.exists( filename ):
-                dir = os.path.join( self.file_path, *directory_hash_id( self.id ) )
-                # Create directory if it does not exist
-                try:
-                    os.makedirs( dir )
-                except OSError, e:
-                    # File Exists is okay, otherwise reraise
-                    if e.errno != errno.EEXIST:
-                        raise
-                # Return filename inside hashed directory
-                return os.path.abspath( os.path.join( dir, "dataset_%d.dat" % self.id ) )
-        else:
-            filename = self.external_filename
-        # Make filename absolute
-        return os.path.abspath( filename )
-            
-    def set_file_name ( self, filename ):
-        if not filename:
-            self.external_filename = None
-        else:
-            self.external_filename = filename
-        
-    file_name = property( get_file_name, set_file_name )
-    
+    def add_folder( self, folder ):
+        folder.parent_id = self.id
+        folder.order_id = self.item_count
+        self.item_count += 1
+    def get_library_item_info_templates( self, template_list=[], restrict=False ):
+        # If restrict is True, we'll return only those templates directly associated with this Folder
+        if self.library_folder_info_template_associations:
+            template_list.extend( [ lfita.library_item_info_template for lfita in self.library_folder_info_template_associations if lfita.library_item_info_template not in template_list ] )
+        if restrict not in [ 'True', True ] and self.parent:
+            self.parent.get_library_item_info_templates( template_list )
+        elif restrict not in [ 'True', True, 'folder' ] and self.library_root:
+            for library_root in self.library_root:
+                library_root.get_library_item_info_templates( template_list )
+        return template_list
     @property
-    def extra_files_path( self ):
-        if self._extra_files_path: 
-            path = self._extra_files_path
+    def active_components( self ):
+        return list( self.active_folders ) + list( self.active_datasets )
+
+class LibraryDataset( object ):
+    # This class acts as a proxy to the currently selected LDDA
+    def __init__( self, folder=None, order_id=None, name=None, info=None, library_dataset_dataset_association=None, **kwd ):
+        self.folder = folder
+        self.order_id = order_id
+        self.name = name
+        self.info = info
+        self.library_dataset_dataset_association = library_dataset_dataset_association
+    def set_library_dataset_dataset_association( self, ldda ):
+        self.library_dataset_dataset_association = ldda
+        ldda.library_dataset = self
+        ldda.flush()
+        self.flush()
+    def get_info( self ):
+        if self.library_dataset_dataset_association:
+            return self.library_dataset_dataset_association.info
+        elif self._info:
+            return self._info
         else:
-            path = os.path.join( self.file_path, "dataset_%d_files" % self.id )
-            #only use path directly under self.file_path if it exists
-            if not os.path.exists( path ):
-                path = os.path.join( os.path.join( self.file_path, *directory_hash_id( self.id ) ), "dataset_%d_files" % self.id )
-        # Make path absolute
-        return os.path.abspath( path )
+            return 'no info'
+    def set_info( self, info ):
+        self._info = info
+    info = property( get_info, set_info )
+    def get_name( self ):
+        if self.library_dataset_dataset_association:
+            return self.library_dataset_dataset_association.name
+        elif self._name:
+            return self._name
+        else:
+            return 'Unnamed dataset'
+    def set_name( self, name ):
+        self._name = name
+    name = property( get_name, set_name )
+    def display_name( self ):
+        self.library_dataset_dataset_association.display_name()
+    def get_library_item_info_templates( self, template_list=[], restrict=False ):
+        # If restrict is True, we'll return only those templates directly associated with this LibraryDataset
+        if self.library_dataset_info_template_associations:
+            template_list.extend( [ ldita.library_item_info_template for ldita in self.library_dataset_info_template_associations if ldita.library_item_info_template not in template_list ] )
+        if restrict not in [ 'True', True ]:
+            self.folder.get_library_item_info_templates( template_list, restrict )
+        return template_list
     
-    def get_size( self ):
-        """Returns the size of the data on disk"""
-        if self.file_size:
-            return self.file_size
+class LibraryDatasetDatasetAssociation( DatasetInstance ):
+    def __init__( self, copied_from_history_dataset_association=None, copied_from_library_dataset_dataset_association=None, library_dataset=None, **kwd ):
+        DatasetInstance.__init__( self, **kwd )
+        self.copied_from_history_dataset_association = copied_from_history_dataset_association
+        self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association
+        self.library_dataset = library_dataset
+    def to_history_dataset_association( self, target_history, parent_id=None ):
+        hid = target_history._next_hid()
+        hda = HistoryDatasetAssociation( name=self.name, 
+                                         info=self.info,
+                                         blurb=self.blurb, 
+                                         peek=self.peek, 
+                                         extension=self.extension, 
+                                         dbkey=self.dbkey, 
+                                         dataset=self.dataset, 
+                                         visible=self.visible, 
+                                         deleted=self.deleted, 
+                                         parent_id=parent_id, 
+                                         copied_from_library_dataset_dataset_association=self,
+                                         history=target_history,
+                                         hid=hid )
+        hda.flush()
+        hda.metadata = self.metadata #need to set after flushed, as MetadataFiles require dataset.id
+        for child in self.children:
+            child_copy = child.to_history_dataset_association( target_history=target_history, parent_id=hda.id )
+        if not self.datatype.copy_safe_peek:
+            hda.set_peek() #in some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
+        hda.flush()
+        return hda
+    def copy( self, copy_children = False, parent_id = None, target_folder = None ):
+        ldda = LibraryDatasetDatasetAssociation( name=self.name, 
+                                                 info=self.info, 
+                                                 blurb=self.blurb, 
+                                                 peek=self.peek, 
+                                                 extension=self.extension, 
+                                                 dbkey=self.dbkey, 
+                                                 dataset=self.dataset, 
+                                                 visible=self.visible, 
+                                                 deleted=self.deleted, 
+                                                 parent_id=parent_id, 
+                                                 copied_from_library_dataset_dataset_association=self,
+                                                 folder=target_folder )
+        ldda.flush()
+         # Need to set after flushed, as MetadataFiles require dataset.id
+        ldda.metadata = self.metadata
+        if copy_children:
+            for child in self.children:
+                child_copy = child.copy( copy_children = copy_children, parent_id = ldda.id )
+        if not self.datatype.copy_safe_peek:
+             # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
+            ldda.set_peek()
+        ldda.flush()
+        return ldda
+    def clear_associated_files( self, metadata_safe = False, purge = False ):
+        return
+    def get_library_item_info_templates( self, template_list=[], restrict=False ):
+        # If restrict is True, we'll return only those templates directly associated with this LibraryDatasetDatasetAssociation
+        if self.library_dataset_dataset_info_template_associations:
+            template_list.extend( [ lddita.library_item_info_template for lddita in self.library_dataset_dataset_info_template_associations if lddita.library_item_info_template not in template_list ] )
+        self.library_dataset.get_library_item_info_templates( template_list, restrict )
+        return template_list
+
+class LibraryInfoTemplateAssociation( object ):
+    pass
+
+class LibraryFolderInfoTemplateAssociation( object ):
+    pass
+
+class LibraryDatasetInfoTemplateAssociation( object ):
+    pass
+
+class LibraryDatasetDatasetInfoTemplateAssociation( object ):
+    pass
+
+class LibraryItemInfoTemplate( object ):
+    def add_element( self, element = None, name = None, description = None ):
+        if element:
+            raise "undefined"
         else:
-            try:
-                return os.path.getsize( self.file_name )
-            except OSError:
-                return 0
-    def set_size( self ):
-        """Returns the size of the data on disk"""
-        try:
-            if not self.file_size:
-                self.file_size = os.path.getsize( self.file_name )
-        except OSError:
-            self.file_size = 0
-    def has_data( self ):
-        """Detects whether there is any data"""
-        return self.get_size() > 0
-    def mark_deleted( self, include_children=True ):
-        self.deleted = True
+            new_elem = LibraryItemInfoTemplateElement()
+            new_elem.name = name
+            new_elem.description = description
+            new_elem.order_id = self.item_count
+            self.item_count += 1
+            self.flush()
+            new_elem.library_item_info_template_id = self.id
+            new_elem.flush()
+            return new_elem
+    
+class LibraryItemInfoTemplateElement( object ):
+    pass
 
-    # FIXME: sqlalchemy will replace this
-    def _delete(self):
-        """Remove the file that corresponds to this data"""
-        try:
-            os.remove(self.data.file_name)
-        except OSError, e:
-            log.critical('%s delete error %s' % (self.__class__.__name__, e))
+class LibraryInfoAssociation( object ):
+    def set_library_item( self, library_item, user ):
+        if isinstance( library_item, Library ):
+            self.library = library_item
+            self.user = user
+        else:
+            raise "Invalid Library specified: %s" % library_item.__class__.__name__
 
-class Old_Dataset( Dataset ):
+class LibraryFolderInfoAssociation( object ):
+    def set_library_item( self, library_item, user ):
+        if isinstance( library_item, LibraryFolder ):
+            self.folder = library_item
+            self.user = user
+        else:
+            raise "Invalid Library specified: %s" % library_item.__class__.__name__
+
+class LibraryDatasetInfoAssociation( object ):
+    def set_library_item( self, library_item, user ):
+        if isinstance( library_item, LibraryDataset ):
+            self.library_dataset = library_item
+            self.user = user
+        else:
+            raise "Invalid Library specified: %s" % library_item.__class__.__name__
+
+class LibraryDatasetDatasetInfoAssociation( object ):
+    def set_library_item( self, library_item, user ):
+        if isinstance( library_item, LibraryDatasetDatasetAssociation ):
+            self.library_dataset_dataset_association = library_item
+            self.user = user
+        else:
+            raise "Invalid Library specified: %s" % library_item.__class__.__name__
+
+class LibraryItemInfo( object ):
+    def get_element_by_template_element( self, template_element ):
+        for element in self.elements:
+            if element.library_item_info_template_element == template_element:
+                return element
+        raise 'element not found'
+
+class LibraryItemInfoElement( object ):
     pass
             
 class ValidationError( object ):
@@ -541,7 +910,16 @@ class Event( object ):
         self.message = message
 
 class GalaxySession( object ):
-    def __init__( self, id=None, user=None, remote_host=None, remote_addr=None, referer=None, current_history_id=None, session_key=None, is_valid=False, prev_session_id=None ):
+    def __init__( self, 
+                  id=None, 
+                  user=None, 
+                  remote_host=None, 
+                  remote_addr=None, 
+                  referer=None, 
+                  current_history_id=None, 
+                  session_key=None, 
+                  is_valid=False, 
+                  prev_session_id=None ):
         self.id = id
         self.user = user
         self.remote_host = remote_host
@@ -611,7 +989,10 @@ class StoredWorkflowMenuEntry( object ):
 
 class MetadataFile( object ):
     def __init__( self, dataset = None, name = None ):
-        self.dataset = dataset
+        if isinstance( dataset, HistoryDatasetAssociation ):
+            self.history_dataset = dataset
+        elif isinstance( dataset, LibraryDatasetDatasetAssociation ):
+            self.library_dataset = dataset
         self.name = name
     @property
     def file_name( self ):
@@ -641,3 +1022,5 @@ def directory_hash_id( id ):
     padded = padded[:-3]
     # Break into chunks of three
     return [ padded[i*3:(i+1)*3] for i in range( len( padded ) // 3 ) ]
+
+

lib/galaxy/model/mapping.py

@@ -13,6 +13,7 @@ from galaxy.model.orm import *
 from galaxy.model.orm.ext.assignmapper import *
 from galaxy.model.custom_types import *
 from galaxy.util.bunch import Bunch
+from galaxy.security import GalaxyRBACAgent
 
 metadata = MetaData()
 context = Session = scoped_session( sessionmaker( autoflush=False, transactional=False ) )
@@ -42,7 +43,9 @@ User.table = Table( "galaxy_user", metadata,
     Column( "update_time", DateTime, default=now, onupdate=now ),
     Column( "email", TrimmedString( 255 ), nullable=False ),
     Column( "password", TrimmedString( 40 ), nullable=False ),
-    Column( "external", Boolean, default=False ) )
+    Column( "external", Boolean, default=False ),
+    Column( "deleted", Boolean, index=True, default=False ),
+    Column( "purged", Boolean, index=True, default=False ) )
             
 History.table = Table( "history", metadata,
     Column( "id", Integer, primary_key=True),
@@ -55,14 +58,6 @@ History.table = Table( "history", metadata,
     Column( "purged", Boolean, index=True, default=False ),
     Column( "genome_build", TrimmedString( 40 ) ) )
 
-# model.Query.table = Table( "query", engine,
-#             Column( "id", Integer, primary_key=True),
-#             Column( "history_id", Integer, ForeignKey( "history.id" ) ),
-#             Column( "name", String( 255 ) ),
-#             Column( "state", String( 64 ) ),
-#             Column( "tool_parameters", Pickle() ) )
-
-
 HistoryDatasetAssociation.table = Table( "history_dataset_association", metadata, 
     Column( "id", Integer, primary_key=True ),
     Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ),
@@ -70,6 +65,7 @@ HistoryDatasetAssociation.table = Table( "history_dataset_association", metadata
     Column( "create_time", DateTime, default=now ),
     Column( "update_time", DateTime, default=now, onupdate=now ),
     Column( "copied_from_history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ), nullable=True ),
+    Column( "copied_from_library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), nullable=True ),
     Column( "hid", Integer ),
     Column( "name", TrimmedString( 255 ) ),
     Column( "info", TrimmedString( 255 ) ),
@@ -111,6 +107,264 @@ ValidationError.table = Table( "validation_error", metadata,
     Column( "err_type", TrimmedString( 64 ) ),
     Column( "attributes", TEXT ) )
 
+Group.table = Table( "galaxy_group", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "name", String( 255 ), index=True, unique=True ),
+    Column( "deleted", Boolean, index=True, default=False ) )
+
+UserGroupAssociation.table = Table( "user_group_association", metadata, 
+    Column( "id", Integer, primary_key=True ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
+    Column( "group_id", Integer, ForeignKey( "galaxy_group.id" ), index=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ) )
+
+UserRoleAssociation.table = Table( "user_role_association", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
+    Column( "role_id", Integer, ForeignKey( "role.id" ), index=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ) )
+
+GroupRoleAssociation.table = Table( "group_role_association", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "group_id", Integer, ForeignKey( "galaxy_group.id" ), index=True ),
+    Column( "role_id", Integer, ForeignKey( "role.id" ), index=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ) )
+
+Role.table = Table( "role", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "name", String( 255 ), index=True, unique=True ),
+    Column( "description", TEXT ),
+    Column( "type", String( 40 ), index=True ),
+    Column( "deleted", Boolean, index=True, default=False ) )
+
+DatasetPermissions.table = Table( "dataset_permissions", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "action", TEXT ),
+    Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
+    Column( "role_id", Integer, ForeignKey( "role.id" ), index=True ) )
+
+LibraryPermissions.table = Table( "library_permissions", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "action", TEXT ),
+    Column( "library_id", Integer, ForeignKey( "library.id" ), nullable=True, index=True ),
+    Column( "role_id", Integer, ForeignKey( "role.id" ), index=True ) )
+
+LibraryFolderPermissions.table = Table( "library_folder_permissions", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "action", TEXT ),
+    Column( "library_folder_id", Integer, ForeignKey( "library_folder.id" ), nullable=True, index=True ),
+    Column( "role_id", Integer, ForeignKey( "role.id" ), index=True ) )
+
+LibraryDatasetPermissions.table = Table( "library_dataset_permissions", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "action", TEXT ),
+    Column( "library_dataset_id", Integer, ForeignKey( "library_dataset.id" ), nullable=True, index=True ),
+    Column( "role_id", Integer, ForeignKey( "role.id" ), index=True ) )
+
+LibraryDatasetDatasetAssociationPermissions.table = Table( "library_dataset_dataset_association_permissions", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "action", TEXT ),
+    Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), nullable=True, index=True ),
+    Column( "role_id", Integer, ForeignKey( "role.id" ), index=True ) )
+
+LibraryItemInfoPermissions.table = Table( "library_item_info_permissions", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "action", TEXT ),
+    Column( "library_item_info_id", Integer, ForeignKey( "library_item_info.id" ), nullable=True, index=True ),
+    Column( "role_id", Integer, ForeignKey( "role.id" ), index=True ) )
+
+LibraryItemInfoTemplatePermissions.table = Table( "library_item_info_template_permissions", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "action", TEXT ),
+    Column( "library_item_info_template_id", Integer, ForeignKey( "library_item_info_template.id" ), nullable=True, index=True ),
+    Column( "role_id", Integer, ForeignKey( "role.id" ), index=True ) )
+
+DefaultUserPermissions.table = Table( "default_user_permissions", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
+    Column( "action", TEXT ),
+    Column( "role_id", Integer, ForeignKey( "role.id" ), index=True ) )
+
+DefaultHistoryPermissions.table = Table( "default_history_permissions", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ),
+    Column( "action", TEXT ),
+    Column( "role_id", Integer, ForeignKey( "role.id" ), index=True ) )
+
+LibraryDataset.table = Table( "library_dataset", metadata, 
+    Column( "id", Integer, primary_key=True ),
+    Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fk" ), nullable=True, index=True ),#current version of dataset, if null, there is not a current version selected
+    Column( "folder_id", Integer, ForeignKey( "library_folder.id" ), index=True ),
+    Column( "order_id", Integer ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "name", TrimmedString( 255 ), key="_name" ), #when not None/null this will supercede display in library (but not when imported into user's history?)
+    Column( "info", TrimmedString( 255 ),  key="_info" ), #when not None/null this will supercede display in library (but not when imported into user's history?)
+    Column( "deleted", Boolean, index=True, default=False ) )
+
+LibraryDatasetDatasetAssociation.table = Table( "library_dataset_dataset_association", metadata, 
+    Column( "id", Integer, primary_key=True ),
+    Column( "library_dataset_id", Integer, ForeignKey( "library_dataset.id" ), index=True ),
+    Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "copied_from_history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id", use_alter=True, name='history_dataset_association_dataset_id_fkey' ), nullable=True ),
+    Column( "copied_from_library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id", use_alter=True, name='library_dataset_dataset_association_id_fkey' ), nullable=True ),
+    Column( "name", TrimmedString( 255 ) ),
+    Column( "info", TrimmedString( 255 ) ),
+    Column( "blurb", TrimmedString( 255 ) ),
+    Column( "peek" , TEXT ),
+    Column( "extension", TrimmedString( 64 ) ),
+    Column( "metadata", MetadataType(), key="_metadata" ),
+    Column( "parent_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), nullable=True ),
+    Column( "designation", TrimmedString( 255 ) ),
+    Column( "deleted", Boolean, index=True, default=False ),
+    Column( "visible", Boolean ) )
+
+Library.table = Table( "library", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "root_folder_id", Integer, ForeignKey( "library_folder.id" ), index=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "name", String( 255 ), index=True ),
+    Column( "deleted", Boolean, index=True, default=False ),
+    Column( "purged", Boolean, index=True, default=False ),
+    Column( "description", TEXT ) )
+
+LibraryFolder.table = Table( "library_folder", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "parent_id", Integer, ForeignKey( "library_folder.id" ), nullable = True, index=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "name", TEXT ),
+    Column( "description", TEXT ),
+    Column( "order_id", Integer ),
+    Column( "item_count", Integer ),
+    Column( "deleted", Boolean, index=True, default=False ),
+    Column( "purged", Boolean, index=True, default=False ),
+    Column( "genome_build", TrimmedString( 40 ) ) )
+
+LibraryItemInfoTemplateElement.table = Table( "library_item_info_template_element", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "optional", Boolean, index=True, default=True ),
+    Column( "deleted", Boolean, index=True, default=False ),
+    Column( "name", TEXT ),
+    Column( "description", TEXT ),
+    Column( "type", TEXT, default='string' ),
+    Column( "order_id", Integer ),
+    Column( "options", JSONType() ),
+    Column( "library_item_info_template_id", Integer, ForeignKey( "library_item_info_template.id" ), index=True ) )
+    
+LibraryItemInfoTemplate.table = Table( "library_item_info_template", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "optional", Boolean, index=True, default=True ),
+    Column( "deleted", Boolean, index=True, default=False ),
+    Column( "name", TEXT ),
+    Column( "description", TEXT ),
+    Column( "item_count", Integer, default=0 ) )
+
+LibraryInfoTemplateAssociation.table = Table( "library_info_template_association", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "library_id", Integer, ForeignKey( "library.id" ), nullable=True, index=True ),
+    Column( "library_item_info_template_id", Integer, ForeignKey( "library_item_info_template.id" ), index=True ) )
+
+LibraryFolderInfoTemplateAssociation.table = Table( "library_folder_info_template_association", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "library_folder_id", Integer, ForeignKey( "library_folder.id" ), nullable=True, index=True ),
+    Column( "library_item_info_template_id", Integer, ForeignKey( "library_item_info_template.id" ), index=True ) )
+
+LibraryDatasetInfoTemplateAssociation.table = Table( "library_dataset_info_template_association", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "library_dataset_id", Integer, ForeignKey( "library_dataset.id" ), nullable=True, index=True ),
+    Column( "library_item_info_template_id", Integer, ForeignKey( "library_item_info_template.id" ), index=True ) )
+
+LibraryDatasetDatasetInfoTemplateAssociation.table = Table( "library_dataset_dataset_info_template_association", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), nullable=True, index=True ),
+    Column( "library_item_info_template_id", Integer, ForeignKey( "library_item_info_template.id" ), index=True ) )
+
+LibraryItemInfoElement.table = Table( "library_item_info_element", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "contents", JSONType() ),
+    Column( "library_item_info_id", Integer, ForeignKey( "library_item_info.id" ), index=True ),
+    Column( "library_item_info_template_element_id", Integer, ForeignKey( "library_item_info_template_element.id" ), index=True ) )
+
+LibraryItemInfo.table = Table( "library_item_info", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "deleted", Boolean, index=True, default=False ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), nullable=True, index=True ),
+    Column( "library_item_info_template_id", Integer, ForeignKey( "library_item_info_template.id" ), nullable=True, index=True )
+     )
+
+LibraryInfoAssociation.table = Table( "library_info_association", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "library_id", Integer, ForeignKey( "library.id" ), nullable=True, index=True ),
+    Column( "library_item_info_id", Integer, ForeignKey( "library_item_info.id" ), index=True ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), nullable=True, index=True ) )
+
+LibraryFolderInfoAssociation.table = Table( "library_folder_info_association", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "library_folder_id", Integer, ForeignKey( "library_folder.id" ), nullable=True, index=True ),
+    Column( "library_item_info_id", Integer, ForeignKey( "library_item_info.id" ), index=True ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), nullable=True, index=True ) )
+
+LibraryDatasetInfoAssociation.table = Table( "library_dataset_info_association", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "library_dataset_id", Integer, ForeignKey( "library_dataset.id" ), nullable=True, index=True ),
+    Column( "library_item_info_id", Integer, ForeignKey( "library_item_info.id" ), index=True ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), nullable=True, index=True ) )
+
+LibraryDatasetDatasetInfoAssociation.table = Table( "library_dataset_dataset_info_association", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), nullable=True, index=True ),
+    Column( "library_item_info_id", Integer, ForeignKey( "library_item_info.id" ), index=True ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), nullable=True, index=True ) )
+
 Job.table = Table( "job", metadata,
     Column( "id", Integer, primary_key=True ),
     Column( "create_time", DateTime, default=now ),
@@ -118,7 +372,7 @@ Job.table = Table( "job", metadata,
     Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ),
     Column( "tool_id", String( 255 ) ),
     Column( "tool_version", TEXT, default="1.0.0" ),
-    Column( "state", String( 64 ) ),
+    Column( "state", String( 64 ), index=True ),
     Column( "info", TrimmedString( 255 ) ),
     Column( "command_line", TEXT ), 
     Column( "param_filename", String( 1024 ) ),
@@ -148,6 +402,17 @@ JobToOutputDatasetAssociation.table = Table( "job_to_output_dataset", metadata,
     Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
     Column( "name", String(255) ) )
     
+JobExternalOutputMetadata.table = Table( "job_external_output_metadata", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ),
+    Column( "history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True, nullable=True ),
+    Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), index=True, nullable=True ),
+    Column( "filename_in", String( 255 ) ),
+    Column( "filename_out", String( 255 ) ),
+    Column( "filename_results_code", String( 255 ) ),
+    Column( "filename_kwds", String( 255 ) ),
+    Column( "job_runner_external_pid", String( 255 ) ) )
+    
 Event.table = Table( "event", metadata, 
     Column( "id", Integer, primary_key=True ),
     Column( "create_time", DateTime, default=now ),
@@ -239,6 +504,7 @@ MetadataFile.table = Table( "metadata_file", metadata,
     Column( "id", Integer, primary_key=True ),
     Column( "name", TEXT ),
     Column( "hda_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True, nullable=True ),
+    Column( "lda_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), index=True, nullable=True ),
     Column( "create_time", DateTime, default=now ),
     Column( "update_time", DateTime, index=True, default=now, onupdate=now ),
     Column( "deleted", Boolean, index=True, default=False ),
@@ -258,28 +524,39 @@ assign_mapper( context, HistoryDatasetAssociation, HistoryDatasetAssociation.tab
         copied_to_history_dataset_associations=relation( 
             HistoryDatasetAssociation, 
             primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id == HistoryDatasetAssociation.table.c.id ),
-            backref=backref( "copied_from_history_dataset_association", primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id == HistoryDatasetAssociation.table.c.id ), remote_side=[HistoryDatasetAssociation.table.c.id] ) ),
+            backref=backref( "copied_from_history_dataset_association", primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id == HistoryDatasetAssociation.table.c.id ), remote_side=[HistoryDatasetAssociation.table.c.id], uselist=False ) ),
+        copied_to_library_dataset_dataset_associations=relation( 
+            LibraryDatasetDatasetAssociation, 
+            primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ),
+            backref=backref( "copied_from_history_dataset_association", primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id], uselist=False ) ),
         implicitly_converted_datasets=relation( 
             ImplicitlyConvertedDatasetAssociation, 
             primaryjoin=( ImplicitlyConvertedDatasetAssociation.table.c.hda_parent_id == HistoryDatasetAssociation.table.c.id ) ),
         children=relation( 
             HistoryDatasetAssociation, 
             primaryjoin=( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ),
-            backref=backref( "parent", primaryjoin=( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ), remote_side=[HistoryDatasetAssociation.table.c.id] ) )
+            backref=backref( "parent", primaryjoin=( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ), remote_side=[HistoryDatasetAssociation.table.c.id], uselist=False ) ),
+        visible_children=relation( 
+            HistoryDatasetAssociation, 
+            primaryjoin=( ( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ) & ( HistoryDatasetAssociation.table.c.visible == True ) ) )
             ) )
 
 assign_mapper( context, Dataset, Dataset.table,
     properties=dict( 
         history_associations=relation( 
             HistoryDatasetAssociation, 
-            primaryjoin=( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) )
+            primaryjoin=( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) ),
+        active_history_associations=relation( 
+            HistoryDatasetAssociation, 
+            primaryjoin=( ( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) & ( HistoryDatasetAssociation.table.c.deleted == False ) ) ),
+        library_associations=relation( 
+            LibraryDatasetDatasetAssociation, 
+            primaryjoin=( Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id ) ),
+        active_library_associations=relation( 
+            LibraryDatasetDatasetAssociation, 
+            primaryjoin=( ( Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id ) & ( LibraryDatasetDatasetAssociation.table.c.deleted == False ) ) )
             ) )
 
-
-# assign_mapper( model.Query, model.Query.table,
-#     properties=dict( datasets=relation( model.Dataset.mapper, backref="query") ) )
-
-
 assign_mapper( context, ImplicitlyConvertedDatasetAssociation, ImplicitlyConvertedDatasetAssociation.table, 
     properties=dict( parent=relation( 
                      HistoryDatasetAssociation, 
@@ -292,19 +569,220 @@ assign_mapper( context, ImplicitlyConvertedDatasetAssociation, ImplicitlyConvert
 assign_mapper( context, History, History.table,
     properties=dict( galaxy_sessions=relation( GalaxySessionToHistoryAssociation ),
                      datasets=relation( HistoryDatasetAssociation, backref="history", order_by=asc(HistoryDatasetAssociation.table.c.hid) ),
-                     active_datasets=relation( HistoryDatasetAssociation, primaryjoin=( ( HistoryDatasetAssociation.table.c.history_id == History.table.c.id ) & ( not_( HistoryDatasetAssociation.table.c.deleted ) ) ), order_by=asc( HistoryDatasetAssociation.table.c.hid ), lazy=False, viewonly=True ),
-                     activatable_datasets=relation( HistoryDatasetAssociation, primaryjoin=( ( HistoryDatasetAssociation.table.c.history_id == History.table.c.id ) & ( not_( Dataset.table.c.purged ) ) ), order_by=asc( HistoryDatasetAssociation.table.c.hid ), lazy=True, viewonly=True )
+                     active_datasets=relation( HistoryDatasetAssociation, primaryjoin=( ( HistoryDatasetAssociation.table.c.history_id == History.table.c.id ) & ( not_( HistoryDatasetAssociation.table.c.deleted ) ) ), order_by=asc( HistoryDatasetAssociation.table.c.hid ), lazy=False, viewonly=True )
                       ) )
 
 assign_mapper( context, User, User.table, 
-    properties=dict( histories=relation( History, backref="user", 
+    properties=dict( histories=relation( History, backref="user",
                                          order_by=desc(History.table.c.update_time) ),
+                     active_histories=relation( History, primaryjoin=( ( History.table.c.user_id == User.table.c.id ) & ( not_( History.table.c.deleted ) ) ), order_by=desc( History.table.c.update_time ) ),
                      galaxy_sessions=relation( GalaxySession, order_by=desc( GalaxySession.table.c.update_time ) ),
                      stored_workflow_menu_entries=relation( StoredWorkflowMenuEntry, backref="user",
                                                             cascade="all, delete-orphan",
                                                             collection_class=ordering_list( 'order_index' ) )
                      ) )
 
+assign_mapper( context, Group, Group.table,
+    properties=dict( users=relation( UserGroupAssociation ) ) )
+
+assign_mapper( context, UserGroupAssociation, UserGroupAssociation.table,
+    properties=dict( user=relation( User, backref = "groups" ),
+                     group=relation( Group, backref = "members" ) ) )
+
+assign_mapper( context, DefaultUserPermissions, DefaultUserPermissions.table,
+    properties=dict( user=relation( User, backref = "default_permissions" ),
+                     role=relation( Role ) ) )
+
+assign_mapper( context, DefaultHistoryPermissions, DefaultHistoryPermissions.table,
+    properties=dict( history=relation( History, backref = "default_permissions" ),
+                     role=relation( Role ) ) )
+
+assign_mapper( context, Role, Role.table,
+    properties=dict(
+        users=relation( UserRoleAssociation ),
+        groups=relation( GroupRoleAssociation )
+    )
+)
+
+assign_mapper( context, UserRoleAssociation, UserRoleAssociation.table,
+    properties=dict(
+        user=relation( User, backref="roles" ),
+        non_private_roles=relation( User, 
+                                    backref="non_private_roles",
+                                    primaryjoin=( ( User.table.c.id == UserRoleAssociation.table.c.user_id ) & ( UserRoleAssociation.table.c.role_id == Role.table.c.id ) & not_( Role.table.c.type == 'private' ) ) ),
+        role=relation( Role )
+    )
+)
+
+assign_mapper( context, GroupRoleAssociation, GroupRoleAssociation.table,
+    properties=dict(
+        group=relation( Group, backref="roles" ),
+        role=relation( Role )
+    )
+)
+
+assign_mapper( context, DatasetPermissions, DatasetPermissions.table,
+    properties=dict(
+        dataset=relation( Dataset, backref="actions" ),
+        role=relation( Role, backref="actions" )
+    )
+)
+
+assign_mapper( context, LibraryPermissions, LibraryPermissions.table,
+    properties=dict(
+        library=relation( Library, backref="actions" ),
+        role=relation( Role, backref="library_actions" )
+    )
+)
+
+assign_mapper( context, LibraryFolderPermissions, LibraryFolderPermissions.table,
+    properties=dict(
+        folder=relation( LibraryFolder, backref="actions" ),
+        role=relation( Role, backref="library_folder_actions" )
+    )
+)
+
+assign_mapper( context, LibraryDatasetPermissions, LibraryDatasetPermissions.table,
+    properties=dict(
+        library_dataset=relation( LibraryDataset, backref="actions" ),
+        role=relation( Role, backref="library_dataset_actions" )
+    )
+)
+
+assign_mapper( context, LibraryDatasetDatasetAssociationPermissions, LibraryDatasetDatasetAssociationPermissions.table,
+    properties=dict(
+        library_dataset_dataset_association = relation( LibraryDatasetDatasetAssociation, backref="actions" ),
+        role=relation( Role, backref="library_dataset_dataset_actions" )
+    )
+)
+
+assign_mapper( context, LibraryItemInfoPermissions, LibraryItemInfoPermissions.table,
+    properties=dict(
+        library_item_info = relation( LibraryItemInfo, backref="actions" ),
+        role=relation( Role, backref="library_item_info_actions" )
+    )
+)
+
+assign_mapper( context, LibraryItemInfoTemplatePermissions, LibraryItemInfoTemplatePermissions.table,
+    properties=dict(
+        library_item_info_template = relation( LibraryItemInfoTemplate, backref="actions" ),
+        role=relation( Role, backref="library_item_info_template_actions" )
+    )
+)
+
+assign_mapper( context, Library, Library.table,
+    properties=dict(
+        root_folder=relation( LibraryFolder,
+        backref=backref( "library_root" ) )
+        ) )
+
+assign_mapper( context, LibraryFolder, LibraryFolder.table,
+    properties=dict( 
+        folders=relation( 
+            LibraryFolder, 
+            primaryjoin=( LibraryFolder.table.c.parent_id == LibraryFolder.table.c.id ),
+            backref=backref( "parent", primaryjoin=( LibraryFolder.table.c.parent_id == LibraryFolder.table.c.id ), remote_side=[LibraryFolder.table.c.id] ) ),
+        active_folders=relation( LibraryFolder, 
+            primaryjoin=( ( LibraryFolder.table.c.parent_id == LibraryFolder.table.c.id ) & ( not_( LibraryFolder.table.c.deleted ) ) ), 
+            order_by=asc( LibraryFolder.table.c.order_id ), 
+            lazy=True, #"""sqlalchemy.exceptions.ArgumentError: Error creating eager relationship 'active_folders' on parent class '<class 'galaxy.model.LibraryFolder'>' to child class '<class 'galaxy.model.LibraryFolder'>': Cant use eager loading on a self referential relationship."""
+            viewonly=True ),
+        datasets=relation( LibraryDataset,
+            primaryjoin=( ( LibraryDataset.table.c.folder_id == LibraryFolder.table.c.id ) ), 
+            order_by=asc( LibraryDataset.table.c.order_id ), 
+            lazy=False, 
+            viewonly=True ),
+        active_datasets=relation( LibraryDataset,
+            primaryjoin=( ( LibraryDataset.table.c.folder_id == LibraryFolder.table.c.id ) & ( not_( LibraryDataset.table.c.deleted ) ) ), 
+            order_by=asc( LibraryDataset.table.c.order_id ), 
+            lazy=False, 
+            viewonly=True )
+    ) )
+
+assign_mapper( context, LibraryDataset, LibraryDataset.table,
+    properties=dict( 
+        folder=relation( LibraryFolder ),
+        library_dataset_dataset_association=relation( LibraryDatasetDatasetAssociation, primaryjoin=( LibraryDataset.table.c.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ) ),
+        expired_datasets = relation( LibraryDatasetDatasetAssociation, foreign_keys=[LibraryDataset.table.c.id,LibraryDataset.table.c.library_dataset_dataset_association_id ], primaryjoin=( ( LibraryDataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.library_dataset_id ) & ( not_( LibraryDataset.table.c.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ) ) ), viewonly=True, uselist=True )
+        ) )
+
+assign_mapper( context, LibraryDatasetDatasetAssociation, LibraryDatasetDatasetAssociation.table,
+    properties=dict( 
+        dataset=relation( Dataset ),
+        library_dataset = relation( LibraryDataset,
+        primaryjoin=( LibraryDatasetDatasetAssociation.table.c.library_dataset_id == LibraryDataset.table.c.id ) ),
+        copied_to_library_dataset_dataset_associations=relation( 
+            LibraryDatasetDatasetAssociation, 
+            primaryjoin=( LibraryDatasetDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ),
+            backref=backref( "copied_from_library_dataset_dataset_association", primaryjoin=( LibraryDatasetDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id] ) ),
+        copied_to_history_dataset_associations=relation( 
+            HistoryDatasetAssociation, 
+            primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ),
+            backref=backref( "copied_from_library_dataset_dataset_association", primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id], uselist=False ) ),
+        children=relation( 
+            LibraryDatasetDatasetAssociation, 
+            primaryjoin=( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ),
+            backref=backref( "parent", primaryjoin=( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id] ) ),
+        visible_children=relation( 
+            LibraryDatasetDatasetAssociation, 
+            primaryjoin=( ( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ) & ( LibraryDatasetDatasetAssociation.table.c.visible == True ) ) )
+        ) )
+
+assign_mapper( context, LibraryItemInfoTemplateElement, LibraryItemInfoTemplateElement.table,
+    properties=dict( library_item_info_template=relation( LibraryItemInfoTemplate, backref="elements" ),
+    ) )
+
+assign_mapper( context, LibraryItemInfoTemplate, LibraryItemInfoTemplate.table )
+
+assign_mapper( context, LibraryInfoTemplateAssociation, LibraryInfoTemplateAssociation.table,
+    properties=dict( library=relation( Library, backref="library_info_template_associations" ),
+                     library_item_info_template = relation( LibraryItemInfoTemplate, backref="library_info_template_associations" ),
+                     ) )
+
+assign_mapper( context, LibraryFolderInfoTemplateAssociation, LibraryFolderInfoTemplateAssociation.table,
+    properties=dict( folder=relation( LibraryFolder, backref="library_folder_info_template_associations" ),
+                     library_item_info_template = relation( LibraryItemInfoTemplate, backref="library_folder_info_template_associations" ),
+                     ) )
+
+assign_mapper( context, LibraryDatasetInfoTemplateAssociation, LibraryDatasetInfoTemplateAssociation.table,
+    properties=dict( library_dataset=relation( LibraryDataset, backref="library_dataset_info_template_associations" ),
+                     library_item_info_template = relation( LibraryItemInfoTemplate, backref="library_dataset_info_template_associations" ),
+                     ) )
+
+assign_mapper( context, LibraryDatasetDatasetInfoTemplateAssociation, LibraryDatasetDatasetInfoTemplateAssociation.table,
+    properties=dict( library_dataset_dataset_association = relation( LibraryDatasetDatasetAssociation, backref="library_dataset_dataset_info_template_associations" ),
+                     library_item_info_template = relation( LibraryItemInfoTemplate, backref="library_dataset_dataset_info_template_associations" ),
+                     ) )
+
+assign_mapper( context, LibraryItemInfoElement, LibraryItemInfoElement.table,
+    properties=dict( library_item_info=relation( LibraryItemInfo, backref="elements" ),
+                     library_item_info_template_element=relation( LibraryItemInfoTemplateElement )
+    ) )
+    
+assign_mapper( context, LibraryItemInfo, LibraryItemInfo.table,
+    properties=dict( library_item_info_template=relation( LibraryItemInfoTemplate, backref="library_item_infos" ),
+    ) )
+
+assign_mapper( context, LibraryInfoAssociation, LibraryInfoAssociation.table,
+    properties=dict( library=relation( Library, backref="library_info_associations" ),
+                     library_item_info = relation( LibraryItemInfo, backref="library_info_associations" ),
+                     ) )
+
+assign_mapper( context, LibraryFolderInfoAssociation, LibraryFolderInfoAssociation.table,
+    properties=dict( folder=relation( LibraryFolder, backref="library_folder_info_associations" ),
+                     library_item_info = relation( LibraryItemInfo, backref="library_folder_info_associations" ),
+                     ) )
+
+assign_mapper( context, LibraryDatasetInfoAssociation, LibraryDatasetInfoAssociation.table,
+    properties=dict( library_dataset=relation( LibraryDataset, backref="library_dataset_info_associations" ),
+                     library_item_info = relation( LibraryItemInfo, backref="library_dataset_info_associations" ),
+                     ) )
+
+assign_mapper( context, LibraryDatasetDatasetInfoAssociation, LibraryDatasetDatasetInfoAssociation.table,
+    properties=dict( library_dataset_dataset_association = relation( LibraryDatasetDatasetAssociation, backref="library_dataset_dataset_info_associations" ),
+                     library_item_info = relation( LibraryItemInfo, backref="library_dataset_dataset_info_associations" ),
+                     ) )
+
 assign_mapper( context, JobToInputDatasetAssociation, JobToInputDatasetAssociation.table,
     properties=dict( job=relation( Job ), dataset=relation( HistoryDatasetAssociation, lazy=False ) ) )
 
@@ -313,12 +791,18 @@ assign_mapper( context, JobToOutputDatasetAssociation, JobToOutputDatasetAssocia
 
 assign_mapper( context, JobParameter, JobParameter.table )
 
+assign_mapper( context, JobExternalOutputMetadata, JobExternalOutputMetadata.table,
+    properties=dict( job = relation( Job ), 
+                     history_dataset_association = relation( HistoryDatasetAssociation, lazy = False ),
+                     library_dataset_dataset_association = relation( LibraryDatasetDatasetAssociation, lazy = False ) ) )
+
 assign_mapper( context, Job, Job.table, 
     properties=dict( galaxy_session=relation( GalaxySession ),
                      history=relation( History ),
                      parameters=relation( JobParameter, lazy=False ),
                      input_datasets=relation( JobToInputDatasetAssociation, lazy=False ),
-                     output_datasets=relation( JobToOutputDatasetAssociation, lazy=False ) ) )
+                     output_datasets=relation( JobToOutputDatasetAssociation, lazy=False ),
+                     external_output_metadata = relation( JobExternalOutputMetadata, lazy = False ) ) )
 
 assign_mapper( context, Event, Event.table,
     properties=dict( history=relation( History ),
@@ -370,7 +854,7 @@ assign_mapper( context, StoredWorkflowMenuEntry, StoredWorkflowMenuEntry.table,
     properties=dict( stored_workflow=relation( StoredWorkflow ) ) )
 
 assign_mapper( context, MetadataFile, MetadataFile.table,
-    properties=dict( dataset=relation( HistoryDatasetAssociation ) ) )
+    properties=dict( history_dataset=relation( HistoryDatasetAssociation ), library_dataset=relation( LibraryDatasetDatasetAssociation ) ) )
 
 def db_next_hid( self ):
     """
@@ -390,13 +874,13 @@ def db_next_hid( self ):
         raise
 
 History._next_hid = db_next_hid
-    
-def init( file_path, url, engine_options={}, create_tables=False ):
-    """Connect mappings to the database"""
-    # Connect dataset to the file path
-    Dataset.file_path = file_path
+
+def guess_dialect_for_url( url ):
+    return (url.split(':', 1))[0]
+
+def load_egg_for_url( url ):
     # Load the appropriate db module
-    dialect = (url.split(':', 1))[0]
+    dialect = guess_dialect_for_url( url )
     try:
         egg = dialect_to_egg[dialect]
         try:
@@ -408,6 +892,13 @@ def init( file_path, url, engine_options={}, create_tables=False ):
     except KeyError:
         # Let this go, it could possibly work with db's we don't support
         log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect )
+
+def init( file_path, url, engine_options={}, create_tables=False ):
+    """Connect mappings to the database"""
+    # Connect dataset to the file path
+    Dataset.file_path = file_path
+    # Load the appropriate db module
+    load_egg_for_url( url )
     # Create the database engine
     engine = create_engine( url, **engine_options )
     # Connect the metadata to the database.
@@ -427,9 +918,12 @@ def init( file_path, url, engine_options={}, create_tables=False ):
     # For backward compatibility with "model.context.current"
     result.context = Session
     result.create_tables = create_tables
+    #load local galaxy security policy
+    result.security_agent = GalaxyRBACAgent( result )
     return result
     
 def get_suite():
     """Get unittest suite for this module"""
     import unittest, mapping_tests
     return unittest.makeSuite( mapping_tests.MappingTests )
+

lib/galaxy/security/__init__.py

@@ -0,0 +1,479 @@
+"""
+Galaxy Security
+
+"""
+import logging
+from galaxy.util.bunch import Bunch
+from galaxy.model.orm import *
+
+log = logging.getLogger(__name__)
+
+class Action( object ):
+    def __init__( self, action, description, model ):
+        self.action = action
+        self.description = description
+        self.model = model
+
+class RBACAgent:
+    """Class that handles galaxy security"""
+    permitted_actions = Bunch(
+        DATASET_MANAGE_PERMISSIONS = Action( "manage permissions", "Role members can manage the roles associated with this dataset", "grant" ),
+        DATASET_ACCESS = Action( "access", "Role members can import this dataset into their history for analysis", "restrict" ),
+        LIBRARY_ADD = Action( "add library item", "Role members can add library items to this folder", "grant" ),
+        LIBRARY_MODIFY = Action( "modify library item", "Role members can modify this library item", "grant" ),
+        LIBRARY_MANAGE = Action( "manage library permissions", "Role members can manage roles associated with this library item", "grant" )
+    )
+    def get_action( self, name, default=None ):
+        """Get a permitted action by its dict key or action name"""
+        for k, v in self.permitted_actions.items():
+            if k == name or v.action == name:
+                return v
+        return default
+    def get_actions( self ):
+        """Get all permitted actions as a list of Action objects"""
+        return self.permitted_actions.__dict__.values()
+    def allow_action( self, user, action, **kwd ):
+        raise 'No valid method of checking action (%s) on %s for user %s.' % ( action, kwd, user )
+    def guess_derived_permissions_for_datasets( self, datasets = [] ):
+        raise "Unimplemented Method"
+    def associate_components( self, **kwd ):
+        raise 'No valid method of associating provided components: %s' % kwd
+    def create_private_user_role( self, user ):
+        raise "Unimplemented Method"
+    def get_private_user_role( self, user ):
+        raise "Unimplemented Method"
+    def user_set_default_permissions( self, user, permissions={}, history=False, dataset=False ):
+        raise "Unimplemented Method"
+    def history_set_default_permissions( self, history, permissions=None, dataset=False, bypass_manage_permission=False ):
+        raise "Unimplemented Method"
+    def set_all_dataset_permissions( self, dataset, permissions ):
+        raise "Unimplemented Method"
+    def set_dataset_permission( self, dataset, permission ):
+        raise "Unimplemented Method"
+    def make_dataset_public( self, dataset ):
+        raise "Unimplemented Method"
+    def get_component_associations( self, **kwd ):
+        raise "Unimplemented Method"
+    def components_are_associated( self, **kwd ):
+        return bool( self.get_component_associations( **kwd ) )
+    def convert_permitted_action_strings( self, permitted_action_strings ):
+        """
+        When getting permitted actions from an untrusted source like a
+        form, ensure that they match our actual permitted actions.
+        """
+        return filter( lambda x: x is not None, [ self.permitted_actions.get( action_string ) for action_string in permitted_action_strings ] )
+
+class GalaxyRBACAgent( RBACAgent ):
+    def __init__( self, model, permitted_actions=None ):
+        self.model = model
+        if permitted_actions:
+            self.permitted_actions = permitted_actions
+        # List of "library_item" objects and their associated permissions and info template objects
+        self.library_item_assocs = ( 
+            ( self.model.Library, self.model.LibraryPermissions, self.model.LibraryInfoAssociation ),
+            ( self.model.LibraryFolder, self.model.LibraryFolderPermissions, self.model.LibraryFolderInfoAssociation ),
+            ( self.model.LibraryDataset, self.model.LibraryDatasetPermissions, self.model.LibraryDatasetInfoAssociation ),
+            ( self.model.LibraryDatasetDatasetAssociation, self.model.LibraryDatasetDatasetAssociationPermissions, self.model.LibraryDatasetDatasetInfoAssociation ),
+            ( self.model.LibraryItemInfo, self.model.LibraryItemInfoPermissions, None ),
+            ( self.model.LibraryItemInfoTemplate, self.model.LibraryItemInfoTemplatePermissions, None ) )
+    def allow_action( self, user, action, **kwd ):
+        if 'dataset' in kwd:
+            return self.allow_dataset_action( user, action, kwd[ 'dataset' ] )
+        elif 'library_item' in kwd:
+            return self.allow_library_item_action( user, action, kwd[ 'library_item' ] )
+        raise 'No valid method of checking action (%s) for user %s using kwd %s' % ( action, str( user ), str( kwd ) )
+    def allow_dataset_action( self, user, action, dataset ):
+        """Returns true when user has permission to perform an action"""
+        if not isinstance( dataset, self.model.Dataset ):
+            dataset = dataset.dataset
+        if not user:
+            if action == self.permitted_actions.DATASET_ACCESS and action.action not in [ dp.action for dp in dataset.actions ]:
+                return True # anons only get access, and only if there are no roles required for the access action
+            # other actions (or if the dataset has roles defined for the access action) fall through to the false below
+        elif action.action not in [ dp.action for dp in dataset.actions ]:
+            if action.model == 'restrict':
+                return True # implicit access to restrict-style actions if the dataset does not have the action
+            # grant-style actions fall through to the false below
+        else:
+            user_role_ids = sorted( [ r.id for r in user.all_roles() ] )
+            perms = self.get_dataset_permissions( dataset )
+            if action in perms.keys():
+                # The filter() returns a list of the dataset's role ids of which the user is not a member,
+                # so an empty list means the user has all of the required roles.
+                if not filter( lambda x: x not in user_role_ids, [ r.id for r in perms[ action ] ] ):
+                    return True # user has all of the roles required to perform the action
+                # Fall through to the false because the user is missing at least one required role
+        return False # default is to reject
+    def allow_library_item_action( self, user, action, library_item ):
+        if user is None:
+            # All permissions are granted, so non-users cannot have permissions
+            return False
+        if action.model == 'grant':
+            user_role_ids = [ r.id for r in user.all_roles() ]
+            # Check to see if user has access to any of the roles
+            allowed_role_assocs = []
+            for item_class, permission_class, info_association_class in self.library_item_assocs:
+                if isinstance( library_item, item_class ):
+                    if permission_class == self.model.LibraryPermissions:
+                        allowed_role_assocs = permission_class.filter_by( action=action.action, library_id=library_item.id ).all()
+                    elif permission_class == self.model.LibraryFolderPermissions:
+                        allowed_role_assocs = permission_class.filter_by( action=action.action, library_folder_id=library_item.id ).all()
+                    elif permission_class == self.model.LibraryDatasetPermissions:
+                        allowed_role_assocs = permission_class.filter_by( action=action.action, library_dataset_id=library_item.id ).all()
+                    elif permission_class == self.model.LibraryDatasetDatasetAssociationPermissions:
+                        allowed_role_assocs = permission_class.filter_by( action=action.action, library_dataset_dataset_association_id=library_item.id ).all()
+                    elif permission_class == self.model.LibraryItemInfoPermissions:
+                        allowed_role_assocs = permission_class.filter_by( action=action.action, library_item_info_id=library_item.id ).all()
+                    elif permission_class == self.model.LibraryItemInfoTemplatePermissions:
+                        allowed_role_assocs = permission_class.filter_by( action=action.action, library_item_info_template_id=library_item.id ).all()
+            for allowed_role_assoc in allowed_role_assocs:
+                if allowed_role_assoc.role_id in user_role_ids:
+                    return True
+            return False
+        else:
+            raise 'Unimplemented model (%s) specified for action (%s)' % ( action.model, action.action )
+    def guess_derived_permissions_for_datasets( self, datasets=[] ):
+        """Returns a dict of { action : [ role, role, ... ] } for the output dataset based upon provided datasets"""
+        perms = {}
+        for dataset in datasets:
+            if not isinstance( dataset, self.model.Dataset ):
+                dataset = dataset.dataset
+            these_perms = {}
+            # initialize blank perms
+            for action in self.get_actions():
+                these_perms[ action ] = []
+            # collect this dataset's perms
+            these_perms = self.get_dataset_permissions( dataset )
+            # join or intersect this dataset's permissions with others
+            for action, roles in these_perms.items():
+                if action not in perms.keys():
+                    perms[ action ] = roles
+                else:
+                    if action.model == 'grant':
+                        # intersect existing roles with new roles
+                        perms[ action ] = filter( lambda x: x in perms[ action ], roles )
+                    elif action.model == 'restrict':
+                        # join existing roles with new roles
+                        perms[ action ].extend( filter( lambda x: x not in perms[ action ], roles ) )
+        return perms
+    def associate_components( self, **kwd ):
+        if 'user' in kwd:
+            if 'group' in kwd:
+                return self.associate_user_group( kwd['user'], kwd['group'] )
+            elif 'role' in kwd:
+                return self.associate_user_role( kwd['user'], kwd['role'] )
+        elif 'role' in kwd:
+            if 'group' in kwd:
+                return self.associate_group_role( kwd['group'], kwd['role'] )
+        if 'action' in kwd:
+            if 'dataset' in kwd and 'role' in kwd:
+                return self.associate_action_dataset_role( kwd['action'], kwd['dataset'], kwd['role'] )
+        raise 'No valid method of associating provided components: %s' % kwd
+    def associate_user_group( self, user, group ):
+        assoc = self.model.UserGroupAssociation( user, group )
+        assoc.flush()
+        return assoc
+    def associate_user_role( self, user, role ):
+        assoc = self.model.UserRoleAssociation( user, role )
+        assoc.flush()
+        return assoc
+    def associate_group_role( self, group, role ):
+        assoc = self.model.GroupRoleAssociation( group, role )
+        assoc.flush()
+        return assoc
+    def associate_action_dataset_role( self, action, dataset, role ):
+        assoc = self.model.DatasetPermissions( action, dataset, role )
+        assoc.flush()
+        return assoc
+    def create_private_user_role( self, user ):
+        # Create private role
+        role = self.model.Role( name=user.email, description='Private Role for ' + user.email, type=self.model.Role.types.PRIVATE )
+        role.flush()
+        # Add user to role
+        self.associate_components( role=role, user=user )
+        return role
+    def get_private_user_role( self, user, auto_create=False ):
+        role = self.model.Role.filter( and_( self.model.Role.table.c.name == user.email, 
+                                             self.model.Role.table.c.type == self.model.Role.types.PRIVATE ) ).first()
+        if not role:
+            if auto_create:
+                return self.create_private_user_role( user )
+            else:
+                return None
+        return role
+    def user_set_default_permissions( self, user, permissions={}, history=False, dataset=False, bypass_manage_permission=False ):
+        # bypass_manage_permission is used to change permissions of datasets in a userless history when logging in
+        if user is None:
+            return None
+        if not permissions:
+            permissions = { self.permitted_actions.DATASET_MANAGE_PERMISSIONS : [ self.get_private_user_role( user, auto_create=True ) ] }
+        # Delete all of the current default permissions for the user
+        for dup in user.default_permissions:
+            dup.delete()
+            dup.flush()
+        # Add the new default permissions for the user
+        for action, roles in permissions.items():
+            if isinstance( action, Action ):
+                action = action.action
+            for dup in [ self.model.DefaultUserPermissions( user, action, role ) for role in roles ]:
+                dup.flush()
+        if history:
+            for history in user.active_histories:
+                self.history_set_default_permissions( history, permissions=permissions, dataset=dataset, bypass_manage_permission=bypass_manage_permission )
+    def user_get_default_permissions( self, user ):
+        permissions = {}
+        for dup in user.default_permissions:
+            action = self.get_action( dup.action )
+            if action in permissions:
+                permissions[ action ].append( dup.role )
+            else:
+                permissions[ action ] = [ dup.role ]
+        return permissions
+    def history_set_default_permissions( self, history, permissions={}, dataset=False, bypass_manage_permission=False ):
+        # bypass_manage_permission is used to change permissions of datasets in a user-less history when logging in
+        user = history.user
+        if not user:
+            # default permissions on a user-less history are None
+            return None
+        if not permissions:
+            permissions = self.user_get_default_permissions( user )
+        # Delete all of the current default permission for the history
+        for dhp in history.default_permissions:
+            dhp.delete()
+            dhp.flush()
+        # Add the new default permissions for the history
+        for action, roles in permissions.items():
+            if isinstance( action, Action ):
+                action = action.action
+            for dhp in [ self.model.DefaultHistoryPermissions( history, action, role ) for role in roles ]:
+                dhp.flush()
+        if dataset:
+            # Only deal with datasets that are not purged
+            for hda in history.activatable_datasets:
+                dataset = hda.dataset
+                if dataset.library_associations:
+                    # Don't change permissions on a dataset associated with a library
+                    continue
+                if [ assoc for assoc in dataset.history_associations if assoc.history not in user.histories ]:
+                    # Don't change permissions on a dataset associated with a history not owned by the user
+                    continue
+                if bypass_manage_permission or self.allow_action( user, self.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset=dataset ):
+                    self.set_all_dataset_permissions( dataset, permissions )
+    def history_get_default_permissions( self, history ):
+        permissions = {}
+        for dhp in history.default_permissions:
+            action = self.get_action( dhp.action )
+            if action in permissions:
+                permissions[ action ].append( dhp.role )
+            else:
+                permissions[ action ] = [ dhp.role ]
+        return permissions
+    def set_all_dataset_permissions( self, dataset, permissions={} ):
+        # Set new permissions on a dataset, eliminating all current permissions        
+        # Delete all of the current permissions on the dataset
+        for dp in dataset.actions:
+            dp.delete()
+            dp.flush()
+        # Add the new permissions on the dataset
+        for action, roles in permissions.items():
+            if isinstance( action, Action ):
+                action = action.action
+            for dp in [ self.model.DatasetPermissions( action, dataset, role ) for role in roles ]:
+                dp.flush()
+    def set_dataset_permission( self, dataset, permission={} ):
+        # Set a specific permission on a dataset, leaving all other current permissions on the dataset alone
+        for action, roles in permission.items():
+            if isinstance( action, Action ):
+                action = action.action
+            # Delete the current specific permission on the dataset if one exists
+            for dp in dataset.actions:
+                if dp.action == action:
+                    dp.delete()
+                    dp.flush()
+            # Add the new specific permission on the dataset
+            for dp in [ self.model.DatasetPermissions( action, dataset, role ) for role in roles ]:
+                dp.flush()
+    def make_dataset_public( self, dataset ):
+        # A dataset is considered public if there are no "access" actions associated with it.  Any
+        # other actions ( 'manage permissions', 'edit metadata' ) are irrelevant.
+        for dp in dataset.actions:
+            if dp.action == self.permitted_actions.DATASET_ACCESS.action:
+                dp.delete()
+                dp.flush()
+    def get_dataset_permissions( self, dataset ):
+        if not isinstance( dataset, self.model.Dataset ):
+            dataset = dataset.dataset
+        permissions = {}
+        for dp in dataset.actions:
+            action = self.get_action( dp.action )
+            if action in permissions:
+                permissions[ action ].append( dp.role )
+            else:
+                permissions[ action ] = [ dp.role ]
+        return permissions
+    def copy_dataset_permissions( self, src, dst ):
+        if not isinstance( src, self.model.Dataset ):
+            src = src.dataset
+        if not isinstance( dst, self.model.Dataset ):
+            dst = dst.dataset
+        self.set_all_dataset_permissions( dst, self.get_dataset_permissions( src ) )
+    def privately_share_dataset( self, dataset, users = [] ):
+        intersect = None
+        for user in users:
+            roles = [ ura.role for ura in user.roles if ura.role.type == self.model.Role.types.SHARING ]
+            if intersect is None:
+                intersect = roles
+            else:
+                new_intersect = []
+                for role in roles:
+                    if role in intersect:
+                        new_intersect.append( role )
+                intersect = new_intersect
+        sharing_role = None
+        if intersect:
+            for role in intersect:
+                if not filter( lambda x: x not in users, [ ura.user for ura in role.users ] ):
+                    # only use a role if it contains ONLY the users we're sharing with
+                    sharing_role = role
+                    break
+        if sharing_role is None:
+            sharing_role = self.model.Role( name = "Sharing role for: " + ", ".join( [ u.email for u in users ] ),
+                                            type = self.model.Role.types.SHARING )
+            sharing_role.flush()
+            for user in users:
+                self.associate_components( user=user, role=sharing_role )
+        self.set_dataset_permission( dataset, { self.permitted_actions.DATASET_ACCESS : [ sharing_role ] } )
+    def set_all_library_permissions( self, library_item, permissions={} ):
+        # Set new permissions on library_item, eliminating all current permissions
+        for role_assoc in library_item.actions:
+            role_assoc.delete()
+            role_assoc.flush()
+        # Add the new permissions on library_item
+        for item_class, permission_class, info_association_class in self.library_item_assocs:
+            if isinstance( library_item, item_class ):
+                for action, roles in permissions.items():
+                    if isinstance( action, Action ):
+                        action = action.action
+                    for role_assoc in [ permission_class( action, library_item, role ) for role in roles ]:
+                        role_assoc.flush()
+    def get_library_dataset_permissions( self, library_dataset ):
+        # Permissions will always be the same for LibraryDatasets and associated
+        # LibraryDatasetDatasetAssociations
+        if isinstance( library_dataset, self.model.LibraryDatasetDatasetAssociation ):
+            library_dataset = library_dataset.library_dataset
+        permissions = {}
+        for library_dataset_permission in library_dataset.actions:
+            action = self.get_action( library_dataset_permission.action )
+            if action in permissions:
+                permissions[ action ].append( library_dataset_permission.role )
+            else:
+                permissions[ action ] = [ library_dataset_permission.role ]
+        return permissions
+    def copy_library_permissions( self, source_library_item, target_library_item, user=None ):
+        # Copy all permissions from source
+        permissions = {}
+        for role_assoc in source_library_item.actions:
+            if role_assoc.action in permissions:
+                permissions[role_assoc.action].append( role_assoc.role )
+            else:
+                permissions[role_assoc.action] = [ role_assoc.role ]
+        self.set_all_library_permissions( target_library_item, permissions )
+        if user:
+            # Make sure user's private role is included
+            item_class = None
+            for item_class, permission_class, info_association_class in self.library_item_assocs:
+                if isinstance( target_library_item, item_class ):
+                    break
+            if item_class:
+                private_role = self.model.security_agent.get_private_user_role( user )
+                for name, action in self.permitted_actions.items():
+                    if not permission_class.filter_by( role_id = private_role.id, action = action.action ).first():
+                        lp = permission_class( action.action, target_library_item, private_role )
+                        lp.flush()
+            else:
+                raise 'Invalid class (%s) specified for target_library_item (%s)' % ( target_library_item.__class__, target_library_item.__class__.__name__ )
+    def show_library_item( self, user, library_item ):
+        # TODO: possibly needs to support other library item types
+        if self.allow_action( user, self.permitted_actions.LIBRARY_MODIFY, library_item=library_item ) or \
+            self.allow_action( user, self.permitted_actions.LIBRARY_MANAGE, library_item=library_item ) or \
+            self.allow_action( user, self.permitted_actions.LIBRARY_ADD, library_item=library_item ):
+            return True
+        if isinstance( library_item, self.model.Library ):
+            return self.show_library_item( user, library_item.root_folder )
+        elif isinstance( library_item, self.model.LibraryFolder ):
+            for folder in library_item.folders:
+                if self.show_library_item( user, folder ):
+                    return True
+        return False
+    def set_entity_user_associations( self, users=[], roles=[], groups=[], delete_existing_assocs=True ):
+        for user in users:
+            if delete_existing_assocs:
+                for a in user.non_private_roles + user.groups:
+                    a.delete()
+                    a.flush()
+            for role in roles:
+                self.associate_components( user=user, role=role )
+            for group in groups:
+                self.associate_components( user=user, group=group )
+    def set_entity_group_associations( self, groups=[], users=[], roles=[], delete_existing_assocs=True ):
+        for group in groups:
+            if delete_existing_assocs:
+                for a in group.roles + group.users:
+                    a.delete()
+                    a.flush()
+            for role in roles:
+                self.associate_components( group=group, role=role )
+            for user in users:
+                self.associate_components( group=group, user=user )
+    def set_entity_role_associations( self, roles=[], users=[], groups=[], delete_existing_assocs=True ):
+        for role in roles:
+            if delete_existing_assocs:
+                for a in role.users + role.groups:
+                    a.delete()
+                    a.flush()
+            for user in users:
+                self.associate_components( user=user, role=role )
+            for group in groups:
+                self.associate_components( group=group, role=role )
+    def get_component_associations( self, **kwd ):
+        assert len( kwd ) == 2, 'You must specify exactly 2 Galaxy security components to check for associations.'
+        if 'dataset' in kwd:
+            if 'action' in kwd:
+                return self.model.DatasetPermissions.filter_by( action = kwd['action'].action, dataset_id = kwd['dataset'].id ).first()
+        elif 'user' in kwd:
+            if 'group' in kwd:
+                return self.model.UserGroupAssociation.filter_by( group_id = kwd['group'].id, user_id = kwd['user'].id ).first()
+            elif 'role' in kwd:
+                return self.model.UserRoleAssociation.filter_by( role_id = kwd['role'].id, user_id = kwd['user'].id ).first()
+        elif 'group' in kwd:
+            if 'role' in kwd:
+                return self.model.GroupRoleAssociation.filter_by( role_id = kwd['role'].id, group_id = kwd['group'].id ).first()
+        raise 'No valid method of associating provided components: %s' % kwd
+    def check_folder_contents( self, user, entry ):
+        """
+        Return true if there are any datasets under 'folder' that the
+        user has access permission on.  We do this a lot and it's a
+        pretty inefficient method, optimizations are welcomed.
+        """
+        if isinstance( entry, self.model.Library ):
+            return self.check_folder_contents( user, entry.root_folder )
+        elif isinstance( entry, self.model.LibraryFolder ):
+            for library_dataset in entry.active_datasets:
+                if self.allow_action( user, self.permitted_actions.DATASET_ACCESS, dataset=library_dataset.library_dataset_dataset_association.dataset ):
+                    return True
+            for folder in entry.active_folders:
+                if self.check_folder_contents( user, folder ):
+                    return True
+            return False
+        elif isinstance( entry, self.model.LibraryDatasetDatasetAssociation ):
+            return self.allow_action( user, self.permitted_actions.DATASET_ACCESS, dataset=entry.dataset )
+        else:
+            raise 'Passed an illegal object to check_folder_contents: %s' % type( entry )
+
+def get_permitted_actions( filter=None ):
+    '''Utility method to return a subset of RBACAgent's permitted actions'''
+    if filter is None:
+        return RBACAgent.permitted_actions
+    tmp_bunch = Bunch()
+    [ tmp_bunch.__dict__.__setitem__(k, v) for k, v in RBACAgent.permitted_actions.items() if k.startswith( filter ) ]
+    return tmp_bunch

lib/galaxy/tools/__init__.py

@@ -1194,7 +1194,7 @@ class Tool:
         redirect_url_params = redirect_url_params.replace( "\n", " " ).replace( "\r", " " )
         return redirect_url_params
 
-    def parse_redirect_url( self, inp_data, param_dict ):
+    def parse_redirect_url( self, data, param_dict ):
         """Parse the REDIRECT_URL tool param"""
         # Tools that send data to an external application via a redirect must include the following 3 tool params:
         # REDIRECT_URL - the url to which the data is being sent
@@ -1213,9 +1213,6 @@ class Tool:
             rup_dict[ p_name ] = p_val
         DATA_URL = param_dict.get( 'DATA_URL', None )
         assert DATA_URL is not None, "DATA_URL parameter missing in tool config."
-        # Get the dataset - there should only be 1
-        for name in inp_data.keys():
-            data = inp_data[ name ]
         DATA_URL += "/%s/display" % str( data.id )
         redirect_url += "?DATA_URL=%s" % DATA_URL
         # Add the redirect_url_params to redirect_url
@@ -1328,6 +1325,7 @@ class Tool:
                 else: visible = False
                 ext = fields.pop(0).lower()
                 child_dataset = self.app.model.HistoryDatasetAssociation( extension=ext, parent_id=outdata.id, designation=designation, visible=visible, dbkey=outdata.dbkey, create_dataset=True )
+                self.app.security_agent.copy_dataset_permissions( outdata.dataset, child_dataset.dataset )
                 # Move data from temp location to dataset location
                 shutil.move( filename, child_dataset.file_name )
                 child_dataset.flush()
@@ -1364,6 +1362,7 @@ class Tool:
                 ext = fields.pop(0).lower()
                 # Create new primary dataset
                 primary_data = self.app.model.HistoryDatasetAssociation( extension=ext, designation=designation, visible=visible, dbkey=outdata.dbkey, create_dataset=True )
+                self.app.security_agent.copy_dataset_permissions( outdata.dataset, primary_data.dataset )
                 primary_data.flush()
                 # Move data from temp location to dataset location
                 shutil.move( filename, primary_data.file_name )

lib/galaxy/tools/actions/__init__.py

@@ -47,6 +47,9 @@ class DefaultToolAction( object ):
                             assoc.dataset = new_data
                             assoc.flush()
                             data = new_data
+                # TODO, Nate: Make sure the permitted actions here are appropriate.
+                if data and not trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_ACCESS, dataset=data ):
+                    raise "User does not have permission to use a dataset (%s) provided for input." % data.id
                 return data
             if isinstance( input, DataToolParameter ):
                 if isinstance( value, list ):
@@ -116,6 +119,14 @@ class DefaultToolAction( object ):
                 data = NoneDataset( datatypes_registry = trans.app.datatypes_registry )
             if data.dbkey not in [None, '?']:
                 input_dbkey = data.dbkey
+        
+        # Determine output dataset permission/roles list
+        existing_datasets = [ inp for inp in inp_data.values() if inp ]
+        if existing_datasets:
+            output_permissions = trans.app.security_agent.guess_derived_permissions_for_datasets( existing_datasets )
+        else:
+            # No valid inputs, we will use history defaults
+            output_permissions = trans.app.security_agent.history_get_default_permissions( trans.history )
         # Build name for output datasets based on tool name and input names
         if len( input_names ) == 1:
             on_text = input_names[0]
@@ -169,6 +180,7 @@ class DefaultToolAction( object ):
                 data = trans.app.model.HistoryDatasetAssociation( extension=ext, create_dataset=True )
                 # Commit the dataset immediately so it gets database assigned unique id
                 data.flush()
+                trans.app.security_agent.set_all_dataset_permissions( data.dataset, output_permissions )
             # Create an empty file immediately
             open( data.file_name, "w" ).close()
             # This may not be neccesary with the new parent/child associations
@@ -233,6 +245,9 @@ class DefaultToolAction( object ):
             job.add_parameter( name, value )
         for name, dataset in inp_data.iteritems():
             if dataset:
+                # TODO, Nate: Make sure the permitted actions here are appropriate.
+                if not trans.app.security_agent.allow_action( trans.user, dataset.permitted_actions.DATASET_ACCESS, dataset=dataset ):
+                    raise "User does not have permission to use a dataset (%s) provided for input." % data.id
                 job.add_input_dataset( name, dataset )
             else:
                 job.add_input_dataset( name, None )
@@ -244,7 +259,15 @@ class DefaultToolAction( object ):
         # include something that can be retrieved from the params ( e.g., REDIRECT_URL ) to keep the job
         # from being queued.
         if 'REDIRECT_URL' in incoming:
-            redirect_url = tool.parse_redirect_url( inp_data, incoming )
+            # Get the dataset - there should only be 1
+            for name in inp_data.keys():
+                dataset = inp_data[ name ]
+            redirect_url = tool.parse_redirect_url( dataset, incoming )
+            # GALAXY_URL should be include in the tool params to enable the external application 
+            # to send back to the current Galaxy instance
+            GALAXY_URL = incoming.get( 'GALAXY_URL', None )
+            assert GALAXY_URL is not None, "GALAXY_URL parameter missing in tool config."
+            redirect_url += "&GALAXY_URL=%s" % GALAXY_URL
             # Job should not be queued, so set state to ok
             job.state = JOB_OK
             job.info = "Redirected to: %s" % redirect_url

lib/galaxy/tools/actions/upload.py

@@ -72,10 +72,24 @@ class UploadToolAction( object ):
                 return self.upload_empty( trans, job, "Error:", str( e ) )
         if url_paste not in [ None, "" ]:
             if url_paste.lower().find( 'http://' ) >= 0 or url_paste.lower().find( 'ftp://' ) >= 0:
+                # If we were sent a DATA_URL from an external application in a post, NAME and INFO
+                # values should be in the request
+                if 'NAME' in incoming and incoming[ 'NAME' ] not in [ "None", None ]:
+                    NAME = incoming[ 'NAME' ]
+                else:
+                    NAME = ''
+                if 'INFO' in incoming and incoming[ 'INFO' ] not in [ "None", None ]:
+                    INFO = incoming[ 'INFO' ]
+                else:
+                    INFO = "uploaded url"
                 url_paste = url_paste.replace( '\r', '' ).split( '\n' )
+                name_set_from_line = False #if we are setting the name from the line, it needs to be the line that creates that dataset
                 for line in url_paste:
                     line = line.rstrip( '\r\n' )
                     if line:
+                        if not NAME or name_set_from_line:
+                            NAME = line
+                            name_set_from_line = True
                         try:
                             temp_name = sniff.stream_to_file( urllib.urlopen( line ), prefix='url_paste' )
                         except Exception, e:
@@ -83,7 +97,7 @@ class UploadToolAction( object ):
                             self.remove_tempfile( temp_name )
                             return self.upload_empty( trans, job, "Error:", str( e ) )
                         try:
-                            data_list.append( self.add_file( trans, temp_name, line, file_type, dbkey, info="uploaded url", space_to_tab=space_to_tab ) )
+                            data_list.append( self.add_file( trans, temp_name, NAME, file_type, dbkey, info="uploaded url", space_to_tab=space_to_tab ) )
                         except Exception, e:
                             log.exception( 'exception in add_file using url_paste temp_name %s: %s' % ( str( temp_name ), str( e ) ) )
                             self.remove_tempfile( temp_name )
@@ -114,7 +128,8 @@ class UploadToolAction( object ):
             return self.upload_empty( trans, job, "Empty file error:", "you attempted to upload an empty file." )
         elif len( data_list ) < 1:
             return self.upload_empty( trans, job, "No data error:", "either you pasted no data, the url you specified is invalid, or you have not specified a file." )
-        hda = data_list[0]
+        #if we could make a 'real' job here, then metadata could be set before job.finish() is called
+        hda = data_list[0] #only our first hda is being added as input for the job, why?
         job.state = trans.app.model.Job.states.OK
         file_size_str = datatypes.data.nice_size( hda.dataset.file_size )
         job.info = "%s, size: %s" % ( hda.info, file_size_str )
@@ -124,9 +139,10 @@ class UploadToolAction( object ):
         trans.log_event( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ), tool_id=tool.id )
         return dict( output=hda )
 
-    def upload_empty( self, trans, job, err_code, err_msg ):
-        data = trans.app.model.HistoryDatasetAssociation( create_dataset = True )
-        data.name = err_code 
+    def upload_empty(self, trans, job, err_code, err_msg):
+        data = trans.app.model.HistoryDatasetAssociation( create_dataset=True )
+        trans.app.security_agent.set_all_dataset_permissions( data.dataset, trans.app.security_agent.history_get_default_permissions( trans.history ) )
+        data.name = err_code
         data.extension = "txt"
         data.dbkey = "?"
         data.info = err_msg
@@ -151,12 +167,12 @@ class UploadToolAction( object ):
         if not os.path.getsize( temp_name ) > 0:
             raise BadFileException( "you attempted to upload an empty file." )
         
-        # See if we have a gzipped file, which, if it passes our restrictions, we'll decompress on the fly.
+        # See if we have a gzipped file, which, if it passes our restrictions, we'll uncompress on the fly.
         is_gzipped, is_valid = self.check_gzip( temp_name )
         if is_gzipped and not is_valid:
             raise BadFileException( "you attempted to upload an inappropriate file." )
         elif is_gzipped and is_valid:
-            #We need to decompress the temp_name file
+            # We need to uncompress the temp_name file
             CHUNK_SIZE = 2**20 # 1Mb   
             fd, uncompressed = tempfile.mkstemp()   
             gzipped_file = gzip.GzipFile( temp_name )
@@ -216,7 +232,6 @@ class UploadToolAction( object ):
             else:
                 self.line_count = sniff.convert_newlines( temp_name )
             if file_type == 'auto':
-                log.debug("In upload, in if file_type == 'auto':")
                 ext = sniff.guess_ext( temp_name, sniff_order=trans.app.datatypes_registry.sniff_order )    
             else:
                 ext = file_type
@@ -226,6 +241,7 @@ class UploadToolAction( object ):
             info = 'uploaded %s file' %data_type
 
         data = trans.app.model.HistoryDatasetAssociation( history = trans.history, extension = ext, create_dataset = True )
+        trans.app.security_agent.set_all_dataset_permissions( data.dataset, trans.app.security_agent.history_get_default_permissions( trans.history ) )
         data.name = file_name
         data.dbkey = dbkey
         data.info = info

lib/galaxy/tools/parameters/basic.py

@@ -3,17 +3,14 @@ Basic tool parameters.
 """
 
 import logging, string, sys, os, os.path
-
 from elementtree.ElementTree import XML, Element
-
 from galaxy import config, datatypes, util
 from galaxy.web import form_builder
 from galaxy.util.bunch import Bunch
-
 import validation, dynamic_options
-
 # For BaseURLToolParameter
 from galaxy.web import url_for
+import galaxy.model
 
 log = logging.getLogger(__name__)
 
@@ -444,10 +441,12 @@ class SelectToolParameter( ToolParameter ):
     >>> print p.name
     blah
     >>> print p.get_html()
+    <div class="checkUncheckAllPlaceholder" checkbox_name="blah"></div>
     <div><input type="checkbox" name="blah" value="x">I am X</div>
     <div class="odd_row"><input type="checkbox" name="blah" value="y" checked>I am Y</div>
     <div><input type="checkbox" name="blah" value="z" checked>I am Z</div>
     >>> print p.get_html( value=["x","y"])
+    <div class="checkUncheckAllPlaceholder" checkbox_name="blah"></div>
     <div><input type="checkbox" name="blah" value="x" checked>I am X</div>
     <div class="odd_row"><input type="checkbox" name="blah" value="y" checked>I am Y</div>
     <div><input type="checkbox" name="blah" value="z">I am Z</div>
@@ -528,7 +527,13 @@ class SelectToolParameter( ToolParameter ):
         if self.is_dynamic and ( trans and trans.workflow_building_mode ) \
            and ( self.options is None or self.options.has_dataset_dependencies ):
             if self.multiple:
-                value = value.split( "\n" )
+                #While it is generally allowed that a select value can be '', 
+                #we do not allow this to be the case in a dynamically generated multiple select list being set in workflow building mode
+                #we instead treat '' as 'No option Selected' (None)
+                if value == '':
+                    value = None
+                else:
+                    value = value.split( "\n" )
             return UnvalidatedValue( value )
         legal_values = self.get_legal_values( trans, other_values )
         if isinstance( value, list ):
@@ -566,7 +571,7 @@ class SelectToolParameter( ToolParameter ):
         return value
     def get_initial_value( self, trans, context ):
         # More working around dynamic options for workflow
-        if self.is_dynamic and trans.workflow_building_mode \
+        if self.is_dynamic and ( trans is None or trans.workflow_building_mode )\
            and ( self.options is None or self.options.has_dataset_dependencies ):
             # Really the best we can do?
             return UnvalidatedValue( None )
@@ -1005,30 +1010,16 @@ class DrillDownSelectToolParameter( ToolParameter ):
 
 
 class DataToolParameter( ToolParameter ):
+    # TODO, Nate: Make sure the following unit tests appropriately test the dataset security
+    # components.  Add as many additional tests as necessary.
     """
     Parameter that takes on one (or many) or a specific set of values.
 
     TODO: There should be an alternate display that allows single selects to be 
           displayed as radio buttons and multiple selects as a set of checkboxes
 
-    >>> # Mock up a history (not connected to database)
-    >>> from galaxy.model import History, HistoryDatasetAssociation
-    >>> from galaxy.util.bunch import Bunch
-    >>> hist = History()
-    >>> hist.flush()
-    >>> hist.add_dataset( HistoryDatasetAssociation( id=1, extension='txt', create_dataset=True ) )
-    >>> hist.add_dataset( HistoryDatasetAssociation( id=2, extension='bed', create_dataset=True ) )
-    >>> hist.add_dataset( HistoryDatasetAssociation( id=3, extension='fasta', create_dataset=True ) )
-    >>> hist.add_dataset( HistoryDatasetAssociation( id=4, extension='png', create_dataset=True ) )
-    >>> hist.add_dataset( HistoryDatasetAssociation( id=5, extension='interval', create_dataset=True ) )
-    >>> p = DataToolParameter( None, XML( '<param name="blah" type="data" format="interval"/>' ) )
-    >>> print p.name
-    blah
-    >>> print p.get_html( trans=Bunch( history=hist ) )
-    <select name="blah">
-    <option value="2">2: Unnamed dataset</option>
-    <option value="5" selected>5: Unnamed dataset</option>
-    </select>
+    TODO: The following must be fixed to test correctly for the new security_check tag in the DataToolParameter ( the last test below is broken )
+    Nate's next passs at the dataset security stuff will dramatically alter this anyway.
     """
 
     def __init__( self, tool, elem ):
@@ -1075,28 +1066,38 @@ class DataToolParameter( ToolParameter ):
                 value = [ value ]
         field = form_builder.SelectField( self.name, self.multiple, None, self.refresh_on_change )
         # CRUCIAL: the dataset_collector function needs to be local to DataToolParameter.get_html_field()
-        def dataset_collector( datasets, parent_hid ):
-            for i, data in enumerate( datasets ):
+        def dataset_collector( hdas, parent_hid ):
+            for i, hda in enumerate( hdas ):
                 if parent_hid is not None:
                     hid = "%s.%d" % ( parent_hid, i + 1 )
                 else:
-                    hid = str( data.hid )
-                if not data.deleted and data.state not in [data.states.ERROR, data.states.DISCARDED] and data.visible:
-                    if self.options and data.get_dbkey() != filter_value:
+                    hid = str( hda.hid )
+                if not hda.dataset.state in [galaxy.model.Dataset.states.ERROR, galaxy.model.Dataset.states.DISCARDED] and \
+                    hda.visible and \
+                    trans.app.security_agent.allow_action( trans.user, hda.permitted_actions.DATASET_ACCESS, dataset=hda ):
+                    # If we are sending data to an external application, then we need to make sure there are no roles
+                    # associated with the dataset that restrict it's access from "public".  We determine this by sending
+                    # None as the user to the allow_action method.
+                    if self.tool and self.tool.tool_type == 'data_destination':
+                        if not trans.app.security_agent.allow_action( None, hda.permitted_actions.DATASET_ACCESS, dataset=hda ):
+                            continue
+                    if self.options and hda.get_dbkey() != filter_value:
                         continue
-                    if isinstance( data.datatype, self.formats):
-                        selected = ( value and ( data in value ) )
-                        field.add_option( "%s: %s" % ( hid, data.name[:30] ), data.id, selected )
+                    if isinstance( hda.datatype, self.formats):
+                        selected = ( value and ( hda in value ) )
+                        field.add_option( "%s: %s" % ( hid, hda.name[:30] ), hda.id, selected )
                     else:
-                        target_ext, converted_dataset = data.find_conversion_destination( self.formats, converter_safe = self.converter_safe( other_values, trans ) )
+                        target_ext, converted_dataset = hda.find_conversion_destination( self.formats, converter_safe = self.converter_safe( other_values, trans ) )
                         if target_ext:
                             if converted_dataset:
-                                data = converted_dataset
-                            selected = ( value and ( data in value ) )
-                            field.add_option( "%s: (as %s) %s" % ( hid, target_ext, data.name[:30] ), data.id, selected )
+                                hda = converted_dataset
+                            if not trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.DATASET_ACCESS, dataset=hda.dataset ):
+                                continue
+                            selected = ( value and ( hda in value ) )
+                            field.add_option( "%s: (as %s) %s" % ( hid, target_ext, hda.name[:30] ), hda.id, selected )
                 # Also collect children via association object
-                dataset_collector( data.children, hid )
-        dataset_collector( history.datasets, None )
+                dataset_collector( hda.children, hid )
+        dataset_collector( history.active_datasets, None )
         some_data = bool( field.options )
         if some_data:
             if value is None or len( field.options ) == 1:

lib/galaxy/tools/parameters/grouping.py

@@ -52,7 +52,10 @@ class Repeat( Group ):
             rval_dict['__index__'] = d.get( '__index__', i )
             # Restore child inputs
             for input in self.inputs.itervalues():
-                rval_dict[ input.name ] = input.value_from_basic( d[input.name], app, ignore_errors )
+                if ignore_errors and input.name not in d: #this wasn't tested
+                    rval_dict[ input.name ] = input.get_initial_value( None, d )
+                else:
+                    rval_dict[ input.name ] = input.value_from_basic( d[input.name], app, ignore_errors )
             rval.append( rval_dict )
         return rval 
     def visit_inputs( self, prefix, value, callback ):
@@ -90,7 +93,13 @@ class Conditional( Group ):
         current_case = rval['__current_case__'] = value['__current_case__']
         rval[ self.test_param.name ] = self.test_param.value_from_basic( value[ self.test_param.name ], app, ignore_errors )
         for input in self.cases[current_case].inputs.itervalues():
-            rval[ input.name ] = input.value_from_basic( value[ input.name ], app, ignore_errors )
+            if ignore_errors and input.name not in value:
+                #two options here, either try to use unvalidated None or use initial==default value
+                #using unvalidated values here will cause, i.e., integer fields within groupings to be filled in workflow building mode like '<galaxy.tools.parameters.basic.UnvalidatedValue object at 0x981818c>'
+                #we will go with using the default value
+                rval[ input.name ] = input.get_initial_value( None, value ) #use default value
+            else:
+                rval[ input.name ] = input.value_from_basic( value[ input.name ], app, ignore_errors )
         return rval
     def visit_inputs( self, prefix, value, callback ):
         current_case = value['__current_case__']

lib/galaxy/tools/util/maf_utilities.py

@@ -54,12 +54,12 @@ class RegionAlignment( object ):
     
     #sets a position for a species
     def set_position( self, index, species, base ):
-        if len( base ) != 1: raise "A genomic position can only have a length of 1."
+        if len( base ) != 1: raise Exception( "A genomic position can only have a length of 1." )
         return self.set_range( index, species, base )
     #sets a range for a species
     def set_range( self, index, species, bases ):
-        if index >= self.size or index < 0: raise "Your index (%i) is out of range (0 - %i)." % ( index, self.size - 1 )
-        if len( bases ) == 0: raise "A set of genomic positions can only have a positive length."
+        if index >= self.size or index < 0: raise Exception( "Your index (%i) is out of range (0 - %i)." % ( index, self.size - 1 ) )
+        if len( bases ) == 0: raise Exception( "A set of genomic positions can only have a positive length." )
         if species not in self.sequences.keys(): self.add_species( species )
         self.sequences[species].seek( index )
         self.sequences[species].write( bases )
@@ -141,7 +141,7 @@ def maf_index_by_uid( maf_uid, index_location_file ):
                     maf_files = fields[4].replace( "\n", "" ).replace( "\r", "" ).split( "," )
                     return bx.align.maf.MultiIndexed( maf_files, keep_open = True, parse_e_rows = False )
                 except Exception, e:
-                    raise 'MAF UID (%s) found, but configuration appears to be malformed: %s' % ( maf_uid, e )
+                    raise Exception( 'MAF UID (%s) found, but configuration appears to be malformed: %s' % ( maf_uid, e ) )
         except:
             pass
     return None
@@ -296,7 +296,7 @@ def get_starts_ends_fields_from_gene_bed( line ):
     fields = line.split()
     #Requires atleast 12 BED columns
     if len(fields) < 12:
-        raise Exception, "Not a proper 12 column BED line (%s)." % line
+        raise Exception( "Not a proper 12 column BED line (%s)." % line )
     chrom     = fields[0]
     tx_start  = int( fields[1] )
     tx_end    = int( fields[2] )
@@ -343,12 +343,19 @@ def get_species_in_maf( maf_filename ):
         except:
             return []
 
-
+def parse_species_option( species ):
+    if species:
+        species = species.split( ',' )
+        if 'None' not in species:
+            return species
+    return None #provided species was '', None, or had 'None' in it
 
 def remove_temp_index_file( index_filename ):
     try: os.unlink( index_filename )
     except: pass
 
+#Below are methods to deal with FASTA files
+
 def get_fasta_header( component, attributes = {}, suffix = None ):
     header = ">%s(%s):%i-%i|" % ( component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end() )
     for key, value in attributes.iteritems():

lib/galaxy/util/__init__.py

@@ -3,7 +3,7 @@ Utility functions used systemwide.
 
 """
 import logging
-import threading, sets, random, string, md5, re, binascii, pickle, time, datetime, math, re, os
+import threading, sets, random, string, md5, re, binascii, pickle, time, datetime, math, re, os, sys
 
 import pkg_resources
 
@@ -85,6 +85,7 @@ mapped_chars = { '>' :'__gt__',
                  ']' :'__cb__',
                  '{' :'__oc__',
                  '}' :'__cc__',
+                 '@' : '__at__'
                  }
 
 def restore_text(text):
@@ -123,7 +124,7 @@ class Params:
     
     Operates on string or list values only (HTTP parameters).
     
-    >>> values = { 'status':'on', 'symbols':[  'alpha', '<>', '$rm&#@!' ]  }
+    >>> values = { 'status':'on', 'symbols':[  'alpha', '<>', '$rm&#!' ]  }
     >>> par = Params(values)
     >>> par.status
     'on'
@@ -132,9 +133,9 @@ class Params:
     >>> par.get('price', 0)
     0
     >>> par.symbols            # replaces unknown symbols with X
-    ['alpha', '__lt____gt__', 'XrmXXX!']
+    ['alpha', '__lt____gt__', 'XrmXX!']
     >>> par.flatten()          # flattening to a list
-    [('status', 'on'), ('symbols', 'alpha'), ('symbols', '__lt____gt__'), ('symbols', 'XrmXXX!')]
+    [('status', 'on'), ('symbols', 'alpha'), ('symbols', '__lt____gt__'), ('symbols', 'XrmXX!')]
     """
     
     # HACK: Need top prevent sanitizing certain parameter types. The 
@@ -264,6 +265,20 @@ def string_as_bool( string ):
     else:
         return False
 
+def listify( item ):
+    """
+    Make a single item a single item list, or return a list if passed a
+    list.  Passing a None returns an empty list.
+    """
+    if not item:
+        return []
+    elif isinstance( item, list ):
+        return item
+    elif isinstance( item, str ) and item.count( ',' ):
+        return item.split( ',' )
+    else:
+        return [ item ]
+
 def commaify(amount):
     orig = amount
     new = re.sub("^(-?\d+)(\d{3})", '\g<1>,\g<2>', amount)
@@ -361,6 +376,50 @@ def read_build_sites(filename):
         print "ERROR: Unable to read builds for site file %s" %filename
     return build_sites
 
+def relpath( path, start = None ):
+    """Return a relative version of a path"""
+    #modified from python 2.6.1 source code
+    
+    #version 2.6+ has it built in, we'll use the 'official' copy
+    if sys.version_info[:2] >= ( 2, 6 ):
+        if start is not None:
+            return os.path.relpath( path, start )
+        return os.path.relpath( path )
+    
+    #we need to initialize some local parameters
+    curdir = os.curdir
+    pardir = os.pardir
+    sep = os.sep
+    commonprefix = os.path.commonprefix
+    join = os.path.join
+    if start is None:
+        start = curdir
+    
+    #below is the unedited (but formated) relpath() from posixpath.py of 2.6.1
+    #this will likely not function properly on non-posix systems, i.e. windows
+    if not path:
+        raise ValueError( "no path specified" )
+    
+    start_list = os.path.abspath( start ).split( sep )
+    path_list = os.path.abspath( path ).split( sep )
+    
+    # Work out how much of the filepath is shared by start and path.
+    i = len( commonprefix( [ start_list, path_list ] ) )
+    
+    rel_list = [ pardir ] * ( len( start_list )- i ) + path_list[ i: ]
+    if not rel_list:
+        return curdir
+    return join( *rel_list )
+
+def stringify_dictionary_keys( in_dict ):
+    #returns a new dictionary
+    #changes unicode keys into strings, only works on top level (does not recurse)
+    #unicode keys are not valid for expansion into keyword arguments on method calls
+    out_dict = {}
+    for key, value in in_dict.iteritems():
+        out_dict[ str( key ) ] = value
+    return out_dict
+
 galaxy_root_path = os.path.join(__path__[0], "..","..","..")
 dbnames = read_dbnames( os.path.join( galaxy_root_path, "tool-data", "shared", "ucsc", "builds.txt" ) ) #this list is used in edit attributes and the upload tool
 ucsc_build_sites = read_build_sites( os.path.join( galaxy_root_path, "tool-data", "shared", "ucsc", "ucsc_build_sites.txt" ) ) #this list is used in history.tmpl

lib/galaxy/web/__init__.py

@@ -2,5 +2,5 @@
 The Galaxy web application.
 """
 
-from framework import expose, json, require_login, url_for, error, form, FormBuilder
+from framework import expose, json, require_login, require_admin, url_for, error, form, FormBuilder
 

lib/galaxy/web/base/controller.py

@@ -28,4 +28,7 @@ class BaseController( object ):
 Root = BaseController
 """
 Deprecated: `BaseController` used to be available under the name `Root`
-"""
+"""
+
+class ControllerUnavailable( Exception ):
+    pass

lib/galaxy/web/buildapp.py

@@ -28,13 +28,18 @@ def add_controllers( webapp, app ):
     them to the webapp.
     """
     from galaxy.web.base.controller import BaseController
+    from galaxy.web.base.controller import ControllerUnavailable
     import galaxy.web.controllers
     controller_dir = galaxy.web.controllers.__path__[0]
     for fname in os.listdir( controller_dir ):
         if not( fname.startswith( "_" ) ) and fname.endswith( ".py" ):
             name = fname[:-3]
             module_name = "galaxy.web.controllers." + name
-            module = __import__( module_name )
+            try:
+                module = __import__( module_name )
+            except ControllerUnavailable, exc:
+                log.debug("%s could not be loaded: %s" % (module_name, str(exc)))
+                continue
             for comp in module_name.split( "." )[1:]:
                 module = getattr( module, comp )
             # Look for a controller inside the modules
@@ -62,6 +67,8 @@ def app_factory( global_conf, **kwargs ):
     # Create the universe WSGI application
     webapp = galaxy.web.framework.WebApplication( app, session_cookie='galaxysession' )
     add_controllers( webapp, app )
+    # Force /history to go to /root/history -- needed since the tests assume this
+    webapp.add_route( '/history', controller='root', action='history' )
     # These two routes handle our simple needs at the moment
     webapp.add_route( '/async/:tool_id/:data_id/:data_secret', controller='async', action='index', tool_id=None, data_id=None, data_secret=None )
     webapp.add_route( '/:controller/:action', action='index' )

lib/galaxy/web/controllers/async.py

@@ -60,7 +60,7 @@ class ASync( BaseController ):
             if STATUS == 'OK':
                 key = hmac.new( trans.app.config.tool_secret, "%d:%d" % ( data.id, data.history_id), sha ).hexdigest()
                 if key != data_secret:
-                    return "You do not have permision to alter data %s." % data_id
+                    return "You do not have permission to alter data %s." % data_id
                 # push the job into the queue
                 data.state = data.blurb = data.states.RUNNING
                 log.debug('executing tool %s' % tool.id)
@@ -104,6 +104,7 @@ class ASync( BaseController ):
             #history.datasets.add_dataset( data )
             
             data = trans.app.model.HistoryDatasetAssociation( create_dataset = True, extension = GALAXY_TYPE )
+            trans.app.security_agent.set_all_dataset_permissions( data.dataset, trans.app.security_agent.history_get_default_permissions( trans.history ) )
             data.name = GALAXY_NAME
             data.dbkey = GALAXY_BUILD
             data.info = GALAXY_INFO

lib/galaxy/web/controllers/dataset.py

@@ -1,14 +1,9 @@
+import logging, os, mimetypes, smtplib
+
 from galaxy.web.base.controller import *
-
-import logging, os, sets, string, shutil
-import re, socket
-import mimetypes
-
-from galaxy import util, datatypes, jobs, web, util, model
-
+from galaxy import web, model
 from cgi import escape, FieldStorage
 
-import smtplib
 from email.MIMEText import MIMEText
 
 import pkg_resources; 
@@ -95,68 +90,66 @@ class DatasetInterface( BaseController ):
             s.close()
             return trans.show_ok_message( "Your error report has been sent" )
         except:
-            return trans.show_error_message( "An error occurred sending the report by email" )
-    
-    @web.expose
-    def default(self, trans, dataset_id=None, **kwd):
-        return 'This link may not be followed from within Galaxy.'
-    
-    @web.expose
-    def display(self, trans, dataset_id=None, filename=None, **kwd):
-        """Catches the dataset id and displays file contents as directed"""
-        if filename is None or filename.lower() == "index":
-            try:
-                data = trans.app.model.HistoryDatasetAssociation.get( dataset_id )
-                if data:
-                    mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() )
-                    trans.response.set_content_type(mime)
-                    trans.log_event( "Display dataset id: %s" % str(dataset_id) )
-                    try:
-                        return open( data.file_name )
-                    except: 
-                        return "This item contains no content"
-            except:
-                pass
-            return "Invalid dataset specified"
-        else:
-            #display files from directory here
-            try:
-                file_path = os.path.join(trans.app.model.HistoryDatasetAssociation.get( dataset_id ).extra_files_path, filename)
-                mime, encoding = mimetypes.guess_type(file_path)
-                if mime is None:
-                    mime = trans.app.datatypes_registry.get_mimetype_by_extension(".".split(file_path)[-1])
-                trans.response.set_content_type(mime)
-                return open(file_path)
-            except:
-                raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % (filename) )
+            return trans.show_error_message( "An error occurred sending the report by email" )
+    
+    @web.expose
+    def default(self, trans, dataset_id=None, **kwd):
+        return 'This link may not be followed from within Galaxy.'
+    
+    @web.expose
+    def display(self, trans, dataset_id=None, filename=None, **kwd):
+        """Catches the dataset id and displays file contents as directed"""
+        data = trans.app.model.HistoryDatasetAssociation.get( dataset_id )
+        if not data:
+            raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) )
+        if trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_ACCESS, dataset = data ):
+            if filename is None or filename.lower() == "index":
+                mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() )
+                trans.response.set_content_type(mime)
+                trans.log_event( "Display dataset id: %s" % str( dataset_id ) )
+                try:
+                    return open( data.file_name )
+                except:
+                    raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % ( filename ) )
+            else:
+                file_path = os.path.join( data.extra_files_path, filename )
+                mime, encoding = mimetypes.guess_type( file_path )
+                if mime is None:
+                    mime = trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( file_path )[-1] )
+                trans.response.set_content_type( mime )
+                try:
+                    return open( file_path )
+                except:
+                    raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % ( filename ) )
+        else:
+            return trans.show_error_message( "You are not allowed to access this dataset" )
     
     def _undelete( self, trans, id ):
-        history = trans.get_history()
-        data = self.app.model.HistoryDatasetAssociation.get( id )
-        if data and data.undeletable:
-            # Walk up parent datasets to find the containing history
-            topmost_parent = data
-            while topmost_parent.parent:
-                topmost_parent = topmost_parent.parent
-            assert topmost_parent in history.datasets, "Data does not belong to current history"
-            # Mark undeleted
-            data.mark_undeleted()
-            self.app.model.flush()
-            trans.log_event( "Dataset id %s has been undeleted" % str(id) )
+        history = trans.get_history()
+        data = self.app.model.HistoryDatasetAssociation.get( id )
+        if data and data.undeletable:
+            # Walk up parent datasets to find the containing history
+            topmost_parent = data
+            while topmost_parent.parent:
+                topmost_parent = topmost_parent.parent
+            assert topmost_parent in history.datasets, "Data does not belong to current history"
+            # Mark undeleted
+            data.mark_undeleted()
+            self.app.model.flush()
+            trans.log_event( "Dataset id %s has been undeleted" % str(id) )
             return True
         return False
-    
-    @web.expose
-    def undelete( self, trans, id ):
-        self._undelete( trans, id )
-        return trans.response.send_redirect( web.url_for( controller='root', action='history', show_deleted = True ) )
     
-    @web.expose
-    def undelete_async( self, trans, id ):
+    @web.expose
+    def undelete( self, trans, id ):
+        self._undelete( trans, id )
+        return trans.response.send_redirect( web.url_for( controller='root', action='history', show_deleted = True ) )
+    
+    @web.expose
+    def undelete_async( self, trans, id ):
         if self._undelete( trans, id ):
             return "OK"
         raise "Error undeleting"
-        
     
     @web.expose
     def copy_datasets( self, trans, source_dataset_ids = "", target_history_ids = "", new_history_name="", do_copy = False ):
@@ -165,7 +158,7 @@ class DatasetInterface( BaseController ):
         create_new_history = False
         if source_dataset_ids:
             if not isinstance( source_dataset_ids, list ):
-            	source_dataset_ids = source_dataset_ids.split( "," )
+                source_dataset_ids = source_dataset_ids.split( "," )
             source_dataset_ids = map( int, source_dataset_ids )
         else:
             source_dataset_ids = []
@@ -219,4 +212,11 @@ class DatasetInterface( BaseController ):
         if user:
            target_histories = user.histories 
         
-        return trans.fill_template( "/dataset/copy_view.mako", source_dataset_ids = source_dataset_ids, target_history_ids = target_history_ids, source_datasets = source_datasets, target_histories = target_histories, new_history_name = new_history_name, done_msg = done_msg, error_msg = error_msg )
+        return trans.fill_template( "/dataset/copy_view.mako",
+                                    source_dataset_ids = source_dataset_ids,
+                                    target_history_ids = target_history_ids,
+                                    source_datasets = source_datasets,
+                                    target_histories = target_histories,
+                                    new_history_name = new_history_name,
+                                    done_msg = done_msg,
+                                    error_msg = error_msg )

lib/galaxy/web/controllers/genetrack.py

@@ -0,0 +1,294 @@
+import time, glob, os
+from itertools import cycle
+import sha
+
+from mako import exceptions
+from mako.template import Template
+from mako.lookup import TemplateLookup
+from galaxy.web.base.controller import *
+
+try:
+    import pkg_resources
+    pkg_resources.require("GeneTrack")
+    import atlas
+    from atlas import sql
+    from atlas import hdf
+    from atlas import util as atlas_utils
+    from atlas.web import formlib, feature_query, feature_filter
+    from atlas.web import label_cache as atlas_label_cache
+    from atlas.plotting.const import *
+    from atlas.plotting.tracks import prefab
+    from atlas.plotting.tracks import chart
+    from atlas.plotting import tracks
+except Exception, exc:
+    raise ControllerUnavailable("GeneTrack could not import a required dependency: %s" % str(exc))
+
+pkg_resources.require( "Paste" )
+import paste.httpexceptions
+
+# Database helpers
+SHOW_LABEL_LIMIT = 10000
+
+def list_labels(session):
+    """
+    Returns a list of labels that will be plotted in order.
+    """
+    labels = sql.Label
+    query = session.query(labels).order_by("-id")
+    return query
+
+def open_databases( conf ):
+    """
+    A helper function that returns handles to the hdf and sql databases
+    """
+    db = hdf.hdf_open( conf.HDF_DATABASE, mode='r' )
+    session = sql.get_session( conf.SQL_URI )
+    return db, session
+
+def hdf_query(db, name, param, autosize=False ):
+    """
+    Schema specific hdf query. 
+    Note that returns data as columns not rows.
+    """
+    if not hdf.has_node(db=db, name=name):
+        atlas.warn( 'missing label %s' % name )
+        return [], [], [], []
+    data  = hdf.GroupData( db=db, name=name)
+    istart, iend = data.get_indices(label=param.chrom, start=param.start, stop=param.end)
+    table = data.get_table(label=param.chrom)
+    if autosize:
+        # attempts to reduce the number of points
+        size = len( table.cols.ix[istart:iend] )
+        step = max( [1, size/1200] )
+    else:
+        step = 1
+
+    ix = table.cols.ix[istart:iend:step].tolist()
+    wx = table.cols.wx[istart:iend:step].tolist()
+    cx = table.cols.cx[istart:iend:step].tolist()
+    ax = table.cols.ax[istart:iend:step].tolist()
+    return ix, wx, cx, ax
+
+# Chart helpers
+def build_tracks( param, conf, data_label, fit_label, pred_label, strand, show=False ):
+    """
+    Builds tracks
+    """
+    # gets all the labels for a fast lookup
+    label_cache = atlas_label_cache( conf )       
+
+    # get database handles for hdf and sql
+    db, session = open_databases( conf )
+
+    # fetching x and y coordinates for bar and fit (line) for 
+    # each strand plus (p), minus (m), all (a) 
+    bix, bpy, bmy, bay = hdf_query( db=db, name=data_label, param=param )
+    fix, fpy, fmy, fay = hdf_query( db=db, name=fit_label, param=param )
+
+    # close the hdf database
+    db.close()
+
+    # get all features within the range
+    all = feature_query( session=session,  param=param )
+
+    # draws the barchart and the nucleosome chart below it
+    if strand == 'composite':
+        bar = prefab.composite_bartrack( fix=fix, fay=fay, bix=bix, bay=bay, param=param)
+    else:
+        bar = prefab.twostrand_bartrack( fix=fix, fmy=fmy, fpy=fpy, bix=bix, bmy=bmy, bpy=bpy, param=param)
+    
+    charts = list()
+    charts.append( bar )            
+
+    return charts
+
+def feature_chart(param=None, session=None, label=None, label_dict={}, color=cycle( [LIGHT, WHITE] ) ):
+    all = feature_filter(feature_query(session=session,  param=param), name=label, kdict=label_dict)
+    flipped = []
+    for feature in all:
+        if feature.strand == "-":
+            feature.start, feature.end = feature.end, feature.start
+        flipped.append(feature)
+    opts  = track_options( 
+        xscale=param.xscale, w=param.width, fgColor=PURPLE,
+        show_labels=param.show_labels, ylabel=str(label),
+        bgColor=color.next()
+    )
+    return [
+       tracks.split_tracks(features=flipped, options=opts, split=param.show_labels, track_type='vector')
+    ]
+
+def consolidate_charts( charts, param ):
+    # create the multiplot
+    opt = chart_options( w=param.width )
+    multi = chart.MultiChart(options=opt, charts=charts)
+    return multi
+
+# SETUP Track Builders
+import functools
+def twostrand_tracks( param=None, conf=None ):
+    return build_tracks( data_label=conf.LABEL, fit_label=conf.FIT_LABEL, pred_label=conf.PRED_LABEL, param=param, conf=conf, strand='twostrand')
+def composite_tracks( param=None, conf=None ):
+    return build_tracks( data_label=conf.LABEL, fit_label=conf.FIT_LABEL, pred_label=conf.PRED_LABEL, param=param, conf=conf, strand='composite')
+
+class BaseConf( object ):
+    """
+    Fake web_conf for atlas.
+    """
+    IMAGE_DIR = "static/genetrack/plots/"
+    LEVELS = [str(x) for x in [ 50, 100, 250, 500, 1000, 2500, 5000, 10000, 20000, 50000, 100000, 200000 ]]
+    ZOOM_LEVELS = zip(LEVELS, LEVELS)
+    PLOT_SETUP = [
+        ('comp-id', 'Composite' ,  'genetrack/index.html', composite_tracks ),
+        ('two-id' , 'Two Strand',  'genetrack/index.html', twostrand_tracks ),
+    ]
+    PLOT_CHOICES = [ (id, name) for (id, name, page, func) in PLOT_SETUP ]
+    PLOT_MAPPER = dict( [ (id, (page, func)) for (id, name, page, func) in PLOT_SETUP ] )
+    
+    def __init__(self, **kwds):
+        for key,value in kwds.items():
+            setattr( self, key, value)
+            
+class WebRoot(BaseController):   
+    @web.expose
+    def search(self, trans, word='', dataset_id=None, submit=''):
+        """
+        Default search page
+        """
+        data = trans.app.model.HistoryDatasetAssociation.get( dataset_id )
+        if not data:
+            raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) )
+        # the main configuration file
+        conf = BaseConf(
+            TITLE = "<i>%s</i>: %s" % (data.metadata.dbkey, data.metadata.label),
+            HDF_DATABASE = os.path.join( data.extra_files_path, data.metadata.hdf ),
+            SQL_URI = "sqlite:///%s" % os.path.join( data.extra_files_path, data.metadata.sqlite ),
+            LABEL = data.metadata.label,
+            FIT_LABEL = "%s-SIGMA-%d" % (data.metadata.label, 20),
+            PRED_LABEL = "PRED-%s-SIGMA-%d" % (data.metadata.label, 20),
+            )
+
+        param = atlas.Param( word=word )
+        
+        # search for a given 
+        try:
+            session = sql.get_session( conf.SQL_URI )
+        except:
+            return trans.fill_template_mako('genetrack/invalid.html', dataset_id=dataset_id)
+
+        if param.word:
+            def search_query( word, text ):
+                query = session.query(sql.Feature).filter( "name LIKE :word or freetext LIKE :text" ).params(word=word, text=text)
+                query = list(query[:20])
+                return query
+
+            # a little heuristics to match most likely target
+            targets = [ 
+                (param.word+'%', 'No match'), # match beginning
+                ('%'+param.word+'%', 'No match'), # match name anywhere
+                ('%'+param.word+'%', '%'+param.word+'%'), # match json anywhere
+            ]
+            for word, text in targets:
+                query = search_query( word=word, text=text)
+                if query:
+                    break
+        else:
+            query = []
+
+        return trans.fill_template_mako('genetrack/search.html', param=param, query=query, dataset_id=dataset_id)
+
+    @web.expose
+    def index(self, trans, dataset_id=None, **kwds):
+        """
+        Main request handler
+        """
+        color = cycle( [LIGHT, WHITE] )
+        data = trans.app.model.HistoryDatasetAssociation.get( dataset_id )
+        if not data:
+            raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) )
+        # the main configuration file
+        conf = BaseConf(
+            TITLE = "<i>%s</i>: %s" % (data.metadata.dbkey, data.metadata.label),
+            HDF_DATABASE = os.path.join( data.extra_files_path, data.metadata.hdf ),
+            SQL_URI = "sqlite:///%s" % os.path.join( data.extra_files_path, data.metadata.sqlite ),
+            LABEL = data.metadata.label,
+            FIT_LABEL = "%s-SIGMA-%d" % (data.metadata.label, 20),
+            PRED_LABEL = "PRED-%s-SIGMA-%d" % (data.metadata.label, 20),
+            )
+
+        try:
+            session = sql.get_session( conf.SQL_URI )
+        except:
+            return trans.fill_template_mako('genetrack/invalid.html', dataset_id=dataset_id)
+
+        if os.path.exists( conf.HDF_DATABASE ):
+            db = hdf.hdf_open( conf.HDF_DATABASE, mode='r' )
+            conf.CHROM_FIELDS = [(x,x) for x in hdf.GroupData(db=db, name=conf.LABEL).labels]
+            db.close()
+        else:
+            query = session.execute(sql.select([sql.feature_table.c.chrom]).distinct())
+            conf.CHROM_FIELDS = [(x.chrom,x.chrom) for x in query]
+
+        # generate a new form based on the configuration
+        form = formlib.main_form( conf )
+        
+        # clear the tempdir every once in a while
+        atlas_utils.clear_tempdir( dir=conf.IMAGE_DIR, days=1, chance=10)
+
+        incoming = form.defaults()
+        incoming.update( kwds )
+        
+        # manage the zoom and pan requests
+        incoming = formlib.zoom_change( kdict=incoming, levels=conf.LEVELS)
+        incoming = formlib.pan_view( kdict=incoming )
+        
+        # process the form
+        param = atlas.Param( **incoming )
+        form.process( incoming )
+
+        if kwds and form.isSuccessful():
+            # adds the sucessfull parameters
+            param.update( form.values() )
+
+        # if it was a search word not a number go to search page
+        try:
+            center = int( param.feature )
+        except ValueError:
+            # go and search for these
+            return trans.response.send_redirect( web.url_for( controller='genetrack', action='search', word=param.feature, dataset_id=dataset_id ) )
+
+        param.width  = min( [2000, int(param.img_size)] )
+        param.xscale = [ param.start, param.end ] 
+        param.show_labels = ( param.end - param.start ) <= SHOW_LABEL_LIMIT    
+        
+        # get the template and the function used to generate the tracks
+        tmpl_name, track_maker  = conf.PLOT_MAPPER[param.plot]
+        
+        # check against a hash, display an image that already exists if it was previously created.
+        hash = sha.new()
+        hash.update(str(dataset_id))
+        for key in sorted(kwds.keys()):
+            hash.update(str(kwds[key]))
+        fname = "%s.png" % hash.hexdigest()
+        fpath = os.path.join(conf.IMAGE_DIR, fname)
+
+        charts = []
+        param.fname  = fname
+        
+        # The SHA1 hash should uniquely identify the qs that created the plot...
+        if os.path.exists(fpath):
+            os.utime(fpath, (time.time(), time.time()))
+            return trans.fill_template_mako(tmpl_name, conf=conf, form=form, param=param, dataset_id=dataset_id)
+        
+        # If the hashed filename doesn't exist, create it.
+        if track_maker is not None and os.path.exists( conf.HDF_DATABASE ):
+            # generate the fit track
+            charts = track_maker( param=param, conf=conf )
+            
+        for label in list_labels( session ):
+            charts.extend( feature_chart(param=param, session=session, label=label.name, label_dict={label.name:label.id}, color=color))
+
+        track_chart = consolidate_charts( charts, param )
+        track_chart.save(fname=fpath)
+
+        return trans.fill_template_mako(tmpl_name, conf=conf, form=form, param=param, dataset_id=dataset_id)

lib/galaxy/web/controllers/history.py

@@ -0,0 +1,341 @@
+from galaxy.web.base.controller import *
+from galaxy.web.framework.helpers.grids import *
+import webhelpers
+from datetime import datetime
+from cgi import escape
+
+log = logging.getLogger( __name__ )
+
+# States for passing messages
+SUCCESS, INFO, WARNING, ERROR = "done", "info", "warning", "error"
+
+def time_ago( x ):
+    return webhelpers.date.distance_of_time_in_words( x, datetime.utcnow() )
+
+def iff( a, b, c ):
+    if a:
+        return b
+    else:
+        return c
+
+class HistoryListGrid( Grid ):
+    
+    title = "Stored histories"
+    model_class = model.History
+    default_sort_key = "-create_time"
+    columns = [
+        GridColumn( "Name", key="name",
+                    link=( lambda item: iff( item.deleted, None, dict( operation="switch", id=item.id ) ) ),
+                    attach_popup=True ),
+        GridColumn( "Datasets (by state)", method='_build_datasets_by_state', ncells=4 ),
+        GridColumn( "Status", method='_build_status' ),
+        GridColumn( "Age", key="create_time", format=time_ago ),
+        GridColumn( "Last update", key="update_time", format=time_ago ),
+        # Valid for filtering but invisible
+        GridColumn( "Deleted", key="deleted", visible=False )
+    ]
+    operations = [
+        GridOperation( "Switch", allow_multiple=False, condition=( lambda item: not item.deleted ) ),
+        GridOperation( "Share", condition=( lambda item: not item.deleted )  ),
+        GridOperation( "Rename", condition=( lambda item: not item.deleted )  ),
+        GridOperation( "Delete", condition=( lambda item: not item.deleted ) ),
+        GridOperation( "Undelete", condition=( lambda item: item.deleted ) )
+    ]
+    standard_filters = [
+        GridColumnFilter( "Active", args=dict( deleted=False ) ),
+        GridColumnFilter( "Deleted", args=dict( deleted=True ) ),
+        GridColumnFilter( "All", args=dict( deleted='All' ) )
+    ]
+    default_filter = dict( deleted=False )
+    
+    def get_current_item( self, trans ):
+        return trans.history
+    
+    def apply_default_filter( self, trans, query ):
+        return query.filter_by( user=trans.user, purged=False )
+            
+    def handle_operation( self, trans, operation, history_ids ):
+        # Display no message by default
+        status, message = None, None
+        refresh_history = False
+        # Load the histories and ensure they all belong to the current user
+        histories = []
+        for hid in history_ids:            
+            history = model.History.get( hid )
+            if history:
+                # Ensure history is owned by current user
+                if history.user_id != None and trans.user:
+                    assert trans.user.id == history.user_id, "History does not belong to current user"
+                histories.append( history )
+            else:
+                log.warn( "Invalid history id '%r' passed to list", hid )
+        operation = operation.lower()
+        if operation == "switch":
+            status, message = self._list_switch( trans, histories )
+            refresh_history = True
+        elif operation == "share":
+            ## Caught above for now
+            pass
+        elif operation == "rename":
+            ## Caught above for now
+            pass
+        elif operation == "delete":
+            status, message = self._list_delete( trans, histories )
+        elif operation == "undelete":
+            status, message = self._list_undelete( trans, histories )
+        trans.sa_session.flush()
+        # Render the list view
+        return status, message
+    
+    def _build_datasets_by_state( self, trans, history ):
+        rval = []
+        for state in ( 'ok', 'running', 'queued', 'error' ):
+            total = sum( 1 for d in history.active_datasets if d.state == state )
+            if total:
+                rval.append( '<div class="count-box state-color-%s">%s</div>' % ( state, total ) )
+            else:
+                rval.append( '' )
+        return rval
+        
+    def _build_status( self, trans, history ):
+        if history.deleted:
+            return "deleted"
+        return ""
+    
+    def _list_delete( self, trans, histories ):
+        """Delete histories"""
+        n_deleted = 0
+        deleted_current = False
+        for history in histories:
+            if not history.deleted:
+                # Delete DefaultHistoryPermissions
+                for dhp in history.default_permissions:
+                    dhp.delete()
+                    dhp.flush()
+                # Mark history as deleted in db
+                history.deleted = True
+                # If deleting the current history, make a new current.
+                if history == trans.history:
+                    deleted_current = True
+                    trans.new_history()
+                trans.log_event( "History id %d marked as deleted" % history.id )
+                n_deleted += 1
+        status = SUCCESS
+        message_parts = []
+        if n_deleted:
+            message_parts.append( "Deleted %d histories." % n_deleted )
+        if deleted_current:
+            message_parts.append( "Your active history was deleted, a new empty history is now active.")
+            status = INFO
+        return ( status, " ".join( message_parts ) )
+            
+    def _list_undelete( self, trans, histories ):
+        """Undelete histories"""
+        n_undeleted = 0
+        n_already_purged = 0
+        for history in histories:
+            if history.purged:
+                n_already_purged += 1
+            if history.deleted:
+                history.deleted = False
+                n_undeleted += 1
+                trans.log_event( "History id %d marked as undeleted" % history.id )
+        status = SUCCESS
+        message_parts = []
+        if n_undeleted:
+            message_parts.append( "Undeleted %d histories." % n_undeleted )
+        if n_already_purged:
+            message_parts.append( "%d have already been purged and cannot be undeleted." % n_already_purged )
+            status = WARNING
+        return status, "".join( message_parts )
+
+    def _list_switch( self, trans, histories ):
+        """Switch to a new different history"""
+        new_history = histories[0]
+        galaxy_session = trans.get_galaxy_session()
+        try:
+            association = trans.app.model.GalaxySessionToHistoryAssociation.filter_by( session_id=galaxy_session.id, history_id=new_history.id ).first()
+        except:
+            association = None
+        new_history.add_galaxy_session( galaxy_session, association=association )
+        new_history.flush()
+        trans.set_history( new_history )
+        trans.log_event( "History switched to id: %s, name: '%s'" % (str(new_history.id), new_history.name ) )
+        # No message
+        return None, None
+
+class HistoryController( BaseController ):
+    
+    @web.expose
+    def index( self, trans ):
+        return ""
+    
+    @web.expose
+    def list_as_xml( self, trans ):
+        """
+        XML history list for functional tests
+        """
+        return trans.fill_template( "/history/list_as_xml.mako" )
+            
+    _list_grid = HistoryListGrid()
+    
+    @web.expose
+    @web.require_login( "work with multiple histories" )
+    def list( self, trans, *args, **kwargs ):
+        """
+        List all available histories
+        """
+        # TODO: these two operations need to be updates still
+        operation = kwargs.get( 'operation', None )
+        if operation:
+            operation = operation.lower()
+        if operation == "share":
+            return self.share( trans, **kwargs )
+        elif operation == "rename":
+            return self.rename( trans, **kwargs )
+        return self._list_grid( trans, *args, **kwargs )
+    
+    @web.expose
+    def rename_async( self, trans, id=None, new_name=None ):
+        history = model.History.get( id )
+        # Check that the history exists, and is either owned by the current
+        # user (if logged in) or the current history
+        assert history is not None
+        if history.user is None:
+            assert history == trans.history
+        else:
+            assert history.user == trans.user
+        # Rename
+        history.name = new_name
+        trans.sa_session.flush()
+    
+    ## These have been moved from 'root' but not cleaned up
+    
+    @web.expose
+    @web.require_login( "share histories with other users" )
+    def share( self, trans, id=None, email="", **kwd ):
+        send_to_err = ""
+        if not id:
+            id = trans.get_history().id
+        if not isinstance( id, list ):
+            id = [ id ]
+        histories = []
+        history_names = []
+        for hid in id:
+            histories.append( trans.app.model.History.get( hid ) )
+            history_names.append(histories[-1].name) 
+        if not email:
+            return trans.fill_template("/history/share.mako", histories=histories, email=email, send_to_err=send_to_err)
+        user = trans.get_user()  
+        send_to_user = trans.app.model.User.filter( trans.app.model.User.table.c.email==email ).first()
+        params = util.Params( kwd )
+        action = params.get( 'action', None )
+        if action == "no_share":
+            trans.response.send_redirect( url_for( action='history_options' ) )
+        if not send_to_user:
+            send_to_err = "No such user"
+        elif user.email == email:
+            send_to_err = "You can't send histories to yourself"
+        else:
+            if 'history_share_btn' in kwd or action != 'share':
+                # The user is attempting to share a history whose datasets cannot all be accessed by the other user.  In this case,
+                # the user sharing the history can chose to make the datasets public ( action == 'public' ) if he has the authority
+                # to do so, or automatically create a new "sharing role" that allows the user to share his private datasets only with the 
+                # desired user ( action == 'private' ).
+                can_change = {}
+                cannot_change = {}
+                for history in histories:
+                    for hda in history.activatable_datasets:
+                        # Only deal with datasets that have not been purged
+                        if not trans.app.security_agent.allow_action( send_to_user, 
+                                                                      trans.app.security_agent.permitted_actions.DATASET_ACCESS, 
+                                                                      dataset=hda ):
+                            # The user with which we are sharing the history does not have access permission on the current dataset
+                            if trans.app.security_agent.allow_action( user, 
+                                                                      trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS, 
+                                                                      dataset=hda ) and not hda.dataset.library_associations:
+                                # The current user has authority to change permissions on the current dataset because
+                                # they have permission to manage permissions on the dataset and the dataset is not associated 
+                                # with a library.
+                                if action == "private":
+                                    trans.app.security_agent.privately_share_dataset( hda.dataset, users=[ user, send_to_user ] )
+                                elif action == "public":
+                                    trans.app.security_agent.make_dataset_public( hda.dataset )
+                                elif history not in can_change:
+                                    # Build the set of histories / datasets on which the current user has authority
+                                    # to "manage permissions".  This is used in /history/share.mako
+                                    can_change[ history ] = [ hda ]
+                                else:
+                                    can_change[ history ].append( hda )
+                            else:
+                                if action in [ "private", "public" ]:
+                                    # Don't change stuff that the user doesn't have permission to change
+                                    continue
+                                elif history not in cannot_change:
+                                    # Build the set of histories / datasets on which the current user does
+                                    # not have authority to "manage permissions".  This is used in /history/share.mako
+                                    cannot_change[ history ] = [ hda ]
+                                else:
+                                    cannot_change[ history ].append( hda )
+                if can_change or cannot_change:
+                    return trans.fill_template( "/history/share.mako", 
+                                                histories=histories, 
+                                                email=email, 
+                                                send_to_err=send_to_err, 
+                                                can_change=can_change, 
+                                                cannot_change=cannot_change )
+            for history in histories:
+                new_history = history.copy( target_user=send_to_user )
+                new_history.name = history.name + " from " + user.email
+                new_history.user_id = send_to_user.id
+                trans.log_event( "History share, id: %s, name: '%s': to new id: %s" % ( str( history.id ), history.name, str( new_history.id ) ) )
+            self.app.model.flush()
+            return trans.show_message( "History (%s) has been shared with: %s" % ( ",".join( history_names ),email ) )
+        return trans.fill_template( "/history/share.mako", histories=histories, email=email, send_to_err=send_to_err )
+        
+    @web.expose
+    @web.require_login( "rename histories" )
+    def rename( self, trans, id=None, name=None, **kwd ):
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = self.app.memory_usage.memory()
+        user = trans.get_user()
+
+        if not isinstance( id, list ):
+            if id != None:
+                id = [ id ]
+        if not isinstance( name, list ):
+            if name != None:
+                name = [ name ]
+        histories = []
+        cur_names = []
+        if not id:
+            if not trans.get_history().user:
+                return trans.show_error_message( "You must save your history before renaming it." )
+            id = [trans.get_history().id]
+        for history_id in id:
+            history = trans.app.model.History.get( history_id )
+            if history and history.user_id == user.id:
+                histories.append(history)
+                cur_names.append(history.name)
+        if not name or len(histories)!=len(name):
+            return trans.fill_template( "/history/rename.mako",histories=histories )
+        change_msg = ""
+        for i in range(len(histories)):
+            if histories[i].user_id == user.id:
+                if name[i] == histories[i].name:
+                    change_msg = change_msg + "<p>History: "+cur_names[i]+" is already named: "+name[i]+"</p>"
+                elif name[i] not in [None,'',' ']:
+                    name[i] = escape(name[i])
+                    histories[i].name = name[i]
+                    histories[i].flush()
+                    change_msg = change_msg + "<p>History: "+cur_names[i]+" renamed to: "+name[i]+"</p>"
+                    trans.log_event( "History renamed: id: %s, renamed to: '%s'" % (str(histories[i].id), name[i] ) )
+                else:
+                    change_msg = change_msg + "<p>You must specify a valid name for History: "+cur_names[i]+"</p>"
+            else:
+                change_msg = change_msg + "<p>History: "+cur_names[i]+" does not appear to belong to you.</p>"
+        if self.app.memory_usage:
+            m1 = trans.app.memory_usage.memory( m0, pretty=True )
+            log.info( "End of root/history_rename, memory used increased by %s"  % m1 )
+        return trans.show_message( "<p>%s" % change_msg, refresh_frames=['history'] ) 

lib/galaxy/web/controllers/library_dataset.py

@@ -0,0 +1,355 @@
+import os, shutil, urllib, StringIO, re, gzip, tempfile, shutil, zipfile
+from galaxy.web.base.controller import *
+from galaxy import util, jobs
+from galaxy.datatypes import sniff
+from galaxy.security import RBACAgent
+
+log = logging.getLogger( __name__ )
+
+class UploadLibraryDataset( BaseController ):
+    def remove_tempfile( self, filename ):
+        try:
+            os.unlink( filename )
+        except:
+            log.exception( 'failure removing temporary file: %s' % filename )
+    def add_file( self, trans, folder_id, file_obj, name, file_format, dbkey, roles, info='no info', space_to_tab=False, replace_dataset=None ):
+        folder = trans.app.model.LibraryFolder.get( folder_id )
+        data_type = None
+        line_count = 0
+        temp_name = sniff.stream_to_file( file_obj )
+        # See if we have an empty file
+        if not os.path.getsize( temp_name ) > 0:
+            raise BadFileException( "you attempted to upload an empty file." )
+        # See if we have a gzipped file, which, if it passes our restrictions, we'll uncompress on the fly.
+        is_gzipped, is_valid = self.check_gzip( temp_name )
+        if is_gzipped and not is_valid:
+            raise BadFileException( "you attempted to upload an inappropriate file." )
+        elif is_gzipped and is_valid:
+            # We need to uncompress the temp_name file
+            CHUNK_SIZE = 2**20 # 1Mb   
+            fd, uncompressed = tempfile.mkstemp()   
+            gzipped_file = gzip.GzipFile( temp_name )
+            while 1:
+                try:
+                    chunk = gzipped_file.read( CHUNK_SIZE )
+                except IOError:
+                    os.close( fd )
+                    os.remove( uncompressed )
+                    raise BadFileException( 'problem uncompressing gzipped data.' )
+                if not chunk:
+                    break
+                os.write( fd, chunk )
+            os.close( fd )
+            gzipped_file.close()
+            # Replace the gzipped file with the decompressed file
+            shutil.move( uncompressed, temp_name )
+            name = name.rstrip( '.gz' )
+            data_type = 'gzip'
+        if not data_type:
+            # See if we have a zip archive
+            is_zipped, is_valid, test_ext = self.check_zip( temp_name )
+            if is_zipped and not is_valid:
+                raise BadFileException( "you attempted to upload an inappropriate file." )
+            elif is_zipped and is_valid:
+                # Currently, we force specific tools to handle this case.  We also require the user
+                # to manually set the incoming file_format
+                if ( test_ext == 'ab1' or test_ext == 'scf' ) and file_format != 'binseq.zip':
+                    raise BadFileException( "Invalid 'File Format' for archive consisting of binary files - use 'Binseq.zip'." )
+                elif test_ext == 'txt' and file_format != 'txtseq.zip':
+                    raise BadFileException( "Invalid 'File Format' for archive consisting of text files - use 'Txtseq.zip'." )
+                if not ( file_format == 'binseq.zip' or file_format == 'txtseq.zip' ):
+                    raise BadFileException( "you must manually set the 'File Format' to either 'Binseq.zip' or 'Txtseq.zip' when uploading zip files." )
+                data_type = 'zip'
+                ext = file_format
+        if not data_type:
+            if self.check_binary( temp_name ):
+                ext = file_name.split( "." )[1].strip().lower()
+                if not( ext == 'ab1' or ext == 'scf' ):
+                    raise BadFileException( "you attempted to upload an inappropriate file." )
+                if ext == 'ab1' and file_format != 'ab1':
+                    raise BadFileException( "you must manually set the 'File Format' to 'Ab1' when uploading ab1 files." )
+                elif ext == 'scf' and file_format != 'scf':
+                    raise BadFileException( "you must manually set the 'File Format' to 'Scf' when uploading scf files." )
+                data_type = 'binary'
+        if not data_type:
+            # We must have a text file
+            if self.check_html( temp_name ):
+                raise BadFileException( "you attempted to upload an inappropriate file." )
+        if data_type != 'binary' and data_type != 'zip':
+            if space_to_tab:
+                line_count = sniff.convert_newlines_sep2tabs( temp_name )
+            elif os.stat( temp_name ).st_size < 262144000: # 250MB
+                line_count = sniff.convert_newlines( temp_name )
+            else:
+                if sniff.check_newlines( temp_name ):
+                    line_count = sniff.convert_newlines( temp_name )
+                else:
+                    line_count = None
+            if file_format == 'auto':
+                ext = sniff.guess_ext( temp_name, sniff_order=trans.app.datatypes_registry.sniff_order )    
+            else:
+                ext = file_format
+            data_type = ext
+        if info is None:
+            info = 'uploaded %s file' % data_type
+        if file_format == 'auto':
+            data_type = sniff.guess_ext( temp_name, sniff_order=trans.app.datatypes_registry.sniff_order )    
+        else:
+            data_type = file_format
+        if replace_dataset:
+            # The replace_dataset param ( when not None ) refers to a LibraryDataset that is being replaced with a new version.
+            # In this case, all of the permissions on the expired LibraryDataset will be applied to the new version.
+            library_dataset = replace_dataset
+        else:
+            # If replace_dataset is None, the Library level permissions will be taken from the folder and applied to the new 
+            # LibraryDataset, and the current user's DefaultUserPermissions will be applied to the associated Dataset.
+            library_dataset = trans.app.model.LibraryDataset( folder=folder, name=name, info=info )
+            library_dataset.flush()
+            trans.app.security_agent.copy_library_permissions( folder, library_dataset )
+        ldda = trans.app.model.LibraryDatasetDatasetAssociation( name=name, 
+                                                                 info=info, 
+                                                                 extension=data_type, 
+                                                                 dbkey=dbkey, 
+                                                                 library_dataset=library_dataset,
+                                                                 create_dataset=True )
+        ldda.flush()
+        # Permissions must be the same on the LibraryDatasetDatasetAssociation and the associated LibraryDataset
+        trans.app.security_agent.copy_library_permissions( library_dataset, ldda )
+        if replace_dataset:
+            # Copy the Dataset level permissions from replace_dataset to the new LibraryDatasetDatasetAssociation.dataset
+            trans.app.security_agent.copy_dataset_permissions( replace_dataset.library_dataset_dataset_association.dataset, ldda.dataset )
+        else:
+            # Copy the current user's DefaultUserPermissions to the new LibraryDatasetDatasetAssociation.dataset
+            trans.app.security_agent.set_all_dataset_permissions( ldda.dataset, trans.app.security_agent.user_get_default_permissions( trans.get_user() ) )
+            folder.add_library_dataset( library_dataset, genome_build=dbkey )
+        library_dataset.library_dataset_dataset_association_id = ldda.id
+        library_dataset.flush()
+        # If roles were selected upon upload, restrict access to the Dataset to those roles
+        if roles:
+            for role in roles:
+                dp = trans.app.model.DatasetPermissions( RBACAgent.permitted_actions.DATASET_ACCESS.action, ldda.dataset, role )
+                dp.flush()
+        shutil.move( temp_name, ldda.dataset.file_name )
+        ldda.state = ldda.states.OK
+        ldda.init_meta()
+        if line_count:
+            try:
+                ldda.set_peek( line_count=line_count )
+            except:
+                ldda.set_peek()
+        else:
+            ldda.set_peek()
+        ldda.set_size()
+        if ldda.missing_meta():
+            ldda.datatype.set_meta( ldda )
+        ldda.flush()
+        return ldda
+    @web.expose
+    def upload_dataset( self, trans, controller, library_id, folder_id, replace_dataset=None, **kwd ):
+        # This method is called from both the admin and library controllers.  The replace_dataset param ( when
+        # not None ) refers to a LibraryDataset that is being replaced with a new version.
+        params = util.Params( kwd )
+        msg = util.restore_text( params.get( 'msg', ''  ) )
+        messagetype = params.get( 'messagetype', 'done' )
+        dbkey = params.get( 'dbkey', '?' )
+        file_format = params.get( 'file_format', 'auto' )
+        data_file = params.get( 'file_data', '' )
+        url_paste = params.get( 'url_paste', '' )
+        server_dir = params.get( 'server_dir', 'None' )
+        if replace_dataset is not None:
+            replace_id = replace_dataset.id
+        else:
+            replace_id = None
+        if data_file == '' and url_paste == '' and server_dir in [ 'None', '' ]:
+            if trans.app.config.library_import_dir is not None:
+                msg = 'Select a file, enter a URL or Text, or select a server directory.'
+            else:
+                msg = 'Select a file, enter a URL or enter Text.'
+            trans.response.send_redirect( web.url_for( controller=controller,
+                                                       action='library_dataset_dataset_association',
+                                                       library_id=library_id,
+                                                       folder_id=folder_id,
+                                                       replace_id=replace_id, 
+                                                       msg=util.sanitize_text( msg ),
+                                                       messagetype='done' ) )
+        space_to_tab = params.get( 'space_to_tab', False )
+        if space_to_tab and space_to_tab not in [ "None", None ]:
+            space_to_tab = True
+        roles = []
+        for role_id in util.listify( params.get( 'roles', [] ) ):
+            roles.append( trans.app.model.Role.get( role_id ) )
+        data_list = []
+        created_ldda_ids = ''
+        if 'filename' in dir( data_file ):
+            file_name = data_file.filename
+            file_name = file_name.split( '\\' )[-1]
+            file_name = file_name.split( '/' )[-1]
+            try:
+                created_ldda = self.add_file( trans,
+                                              folder_id,
+                                              data_file.file,
+                                              file_name,
+                                              file_format,
+                                              dbkey,
+                                              roles,
+                                              info="uploaded file",
+                                              space_to_tab=space_to_tab,
+                                              replace_dataset=replace_dataset )
+                created_ldda_ids = str( created_ldda.id )
+            except Exception, e:
+                log.exception( 'exception in upload_dataset using file_name %s: %s' % ( str( file_name ), str( e ) ) )
+                return self.upload_empty( trans, controller, library_id, "Error:", str( e ) )
+        elif url_paste not in [ None, "" ]:
+            if url_paste.lower().find( 'http://' ) >= 0 or url_paste.lower().find( 'ftp://' ) >= 0:
+                url_paste = url_paste.replace( '\r', '' ).split( '\n' )
+                for line in url_paste:
+                    line = line.rstrip( '\r\n' )
+                    if line:
+                        try:
+                            created_ldda = self.add_file( trans,
+                                                          folder_id,
+                                                          urllib.urlopen( line ),
+                                                          line,
+                                                          file_format,
+                                                          dbkey,
+                                                          roles,
+                                                          info="uploaded url",
+                                                          space_to_tab=space_to_tab,
+                                                          replace_dataset=replace_dataset )
+                            created_ldda_ids = '%s,%s' % ( created_ldda_ids, str( created_ldda.id ) )
+                        except Exception, e:
+                            log.exception( 'exception in upload_dataset using url_paste %s' % str( e ) )
+                            return self.upload_empty( trans, controller, library_id, "Error:", str( e ) )
+            else:
+                is_valid = False
+                for line in url_paste:
+                    line = line.rstrip( '\r\n' )
+                    if line:
+                        is_valid = True
+                        break
+                if is_valid:
+                    try:
+                        created_ldda = self.add_file( trans,
+                                                      folder_id,
+                                                      StringIO.StringIO( url_paste ),
+                                                      'Pasted Entry',
+                                                      file_format,
+                                                      dbkey,
+                                                      roles,
+                                                      info="pasted entry",
+                                                      space_to_tab=space_to_tab,
+                                                      replace_dataset=replace_dataset )
+                        created_ldda_ids = '%s,%s' % ( created_ldda_ids, str( created_ldda.id ) )
+                    except Exception, e:
+                        log.exception( 'exception in add_file using StringIO.StringIO( url_paste ) %s' % str( e ) )
+                        return self.upload_empty( trans, controller, library_id, "Error:", str( e ) )
+        elif server_dir not in [ None, "", "None" ]:
+            full_dir = os.path.join( trans.app.config.library_import_dir, server_dir )
+            try:
+                files = os.listdir( full_dir )
+            except:
+                log.debug( "Unable to get file list for %s" % full_dir )
+            for file in files:
+                full_file = os.path.join( full_dir, file )
+                if not os.path.isfile( full_file ):
+                    continue
+                try:
+                    created_ldda = self.add_file( trans,
+                                                  folder_id,
+                                                  open( full_file, 'rb' ),
+                                                  file,
+                                                  file_format,
+                                                  dbkey,
+                                                  roles,
+                                                  info="imported file",
+                                                  space_to_tab=space_to_tab,
+                                                  replace_dataset=replace_dataset )
+                    created_ldda_ids = '%s,%s' % ( created_ldda_ids, str( created_ldda.id ) )
+                except Exception, e:
+                    log.exception( 'exception in add_file using server_dir %s' % str( e ) )
+                    return self.upload_empty( trans, controller, library_id, "Error:", str( e ) )
+        if created_ldda_ids:
+            created_ldda_ids = created_ldda_ids.lstrip( ',' )
+            return created_ldda_ids
+        else:
+            return ''
+    def check_gzip( self, temp_name ):
+        temp = open( temp_name, "U" )
+        magic_check = temp.read( 2 )
+        temp.close()
+        if magic_check != util.gzip_magic:
+            return ( False, False )
+        CHUNK_SIZE = 2**15 # 32Kb
+        gzipped_file = gzip.GzipFile( temp_name )
+        chunk = gzipped_file.read( CHUNK_SIZE )
+        gzipped_file.close()
+        if self.check_html( temp_name, chunk=chunk ) or self.check_binary( temp_name, chunk=chunk ):
+            return( True, False )
+        return ( True, True )
+    def check_zip( self, temp_name ):
+        if not zipfile.is_zipfile( temp_name ):
+            return ( False, False, None )
+        zip_file = zipfile.ZipFile( temp_name, "r" )
+        # Make sure the archive consists of valid files.  The current rules are:
+        # 1. Archives can only include .ab1, .scf or .txt files
+        # 2. All file file_formats within an archive must be the same
+        name = zip_file.namelist()[0]
+        test_ext = name.split( "." )[1].strip().lower()
+        if not ( test_ext == 'scf' or test_ext == 'ab1' or test_ext == 'txt' ):
+            return ( True, False, test_ext )
+        for name in zip_file.namelist():
+            ext = name.split( "." )[1].strip().lower()
+            if ext != test_ext:
+                return ( True, False, test_ext )
+        return ( True, True, test_ext )
+    def check_html( self, temp_name, chunk=None ):
+        if chunk is None:
+            temp = open(temp_name, "U")
+        else:
+            temp = chunk
+        regexp1 = re.compile( "<A\s+[^>]*HREF[^>]+>", re.I )
+        regexp2 = re.compile( "<IFRAME[^>]*>", re.I )
+        regexp3 = re.compile( "<FRAMESET[^>]*>", re.I )
+        regexp4 = re.compile( "<META[^>]*>", re.I )
+        lineno = 0
+        for line in temp:
+            lineno += 1
+            matches = regexp1.search( line ) or regexp2.search( line ) or regexp3.search( line ) or regexp4.search( line )
+            if matches:
+                if chunk is None:
+                    temp.close()
+                return True
+            if lineno > 100:
+                break
+        if chunk is None:
+            temp.close()
+        return False
+    def check_binary( self, temp_name, chunk=None ):
+        if chunk is None:
+            temp = open( temp_name, "U" )
+        else:
+            temp = chunk
+        lineno = 0
+        for line in temp:
+            lineno += 1
+            line = line.strip()
+            if line:
+                for char in line:
+                    if ord( char ) > 128:
+                        if chunk is None:
+                            temp.close()
+                        return True
+            if lineno > 10:
+                break
+        if chunk is None:
+            temp.close()
+        return False
+    def upload_empty( self, trans, controller, library_id, err_code, err_msg ):
+        msg = err_code + err_msg
+        return trans.response.send_redirect( web.url_for( controller=controller,
+                                                          action='browse_library',
+                                                          id=library_id,
+                                                          msg=util.sanitize_text( msg ),
+                                                          messagetype='error' ) )
+class BadFileException( Exception ):
+    pass

lib/galaxy/web/controllers/mobile.py

@@ -0,0 +1,60 @@
+from galaxy.web.base.controller import *
+
+class Mobile( BaseController ):
+    @web.expose
+    def index( self, trans, **kwargs ):
+        if trans.user is None:
+            return self.__login( trans, **kwargs )
+        else:
+            return self.history_list( trans, **kwargs )
+        
+    @web.expose
+    def history_list( self, trans ):
+        if trans.user is None: trans.response.send_redirect( url_for( action='index' ) )
+        return trans.fill_template( "mobile/history/list.mako" )
+        
+    @web.expose
+    def history_detail( self, trans, id ):
+        if trans.user is None: trans.response.send_redirect( url_for( action='index' ) )
+        history = trans.app.model.History.get( id )
+        assert history.user == trans.user
+        return trans.fill_template( "mobile/history/detail.mako", history=history )
+
+    @web.expose
+    def dataset_detail( self, trans, id ):
+        if trans.user is None: trans.response.send_redirect( url_for( action='index' ) )
+        dataset = trans.app.model.HistoryDatasetAssociation.get( id )
+        assert dataset.history.user == trans.user
+        return trans.fill_template( "mobile/dataset/detail.mako", dataset=dataset )
+
+    @web.expose
+    def dataset_peek( self, trans, id ):
+        if trans.user is None: trans.response.send_redirect( url_for( action='index' ) )
+        dataset = trans.app.model.HistoryDatasetAssociation.get( id )
+        assert dataset.history.user == trans.user
+        yield "<html><body>"
+        yield dataset.display_peek()
+        yield "</body></html>"
+        
+    def __login( self, trans, email="", password="" ):
+        email_error = password_error = None
+        if email or password:
+            user = model.User.filter( model.User.table.c.email==email ).first()
+            if not user:
+                email_error = "No such user"
+            elif user.deleted:
+                email_error = "This account has been marked deleted, contact your Galaxy administrator to restore the account."
+            elif user.external:
+                email_error = "This account was created for use with an external authentication method, contact your local Galaxy administrator to activate it."
+            elif not user.check_password( password ):
+                password_error = "Invalid password"
+            else:
+                trans.handle_user_login( user )
+                trans.log_event( "User logged in" )
+                trans.response.send_redirect( url_for( action='index' ) )
+        form = web.FormBuilder( web.url_for(), "Login", submit_text="Login" ) \
+                .add_text( "email", "Email", value=email, error=email_error ) \
+                .add_password( "password", "Password", value='', error=password_error, 
+                                help="<a href='%s'>Forgot password? Reset here</a>" % web.url_for( action='reset_password' ) )
+        return trans.show_form( form, template="mobile/form.mako" )
+        

lib/galaxy/web/controllers/root.py

@@ -1,15 +1,11 @@
 """
 Contains the main interface in the Universe class
 """
-from galaxy.web.base.controller import *
-
-import logging, os, sets, string, shutil
-import re, socket
-
-from galaxy import util, datatypes, jobs, web, util
-
+import logging, os, sets, string, shutil, urllib, re, socket
 from cgi import escape, FieldStorage
-import urllib
+from galaxy import util, datatypes, jobs, web, util
+from galaxy.web.base.controller import *
+from galaxy.model.orm import *
 
 log = logging.getLogger( __name__ )
 
@@ -20,16 +16,20 @@ class RootController( BaseController ):
         return 'This link may not be followed from within Galaxy.'
     
     @web.expose
-    def index(self, trans, id=None, tool_id=None, mode=None, m_c=None, m_a=None, **kwd):
+    def index(self, trans, id=None, tool_id=None, mode=None, workflow_id=None, m_c=None, m_a=None, **kwd):
         return trans.fill_template( "root/index.mako",
                                     tool_id=tool_id,
+                                    workflow_id=workflow_id,
                                     m_c=m_c, m_a=m_a )
         
     ## ---- Tool related -----------------------------------------------------
     
     @web.expose
     def tool_menu( self, trans ):
-        return trans.fill_template('/root/tool_menu.mako', toolbox=self.get_toolbox() )
+        if trans.app.config.require_login and not trans.user:
+            return trans.fill_template( '/no_access.mako', message = 'Please log in to access Galaxy tools.' )
+        else:
+            return trans.fill_template('/root/tool_menu.mako', toolbox=self.get_toolbox() )
 
     @web.expose
     def tool_help( self, trans, id ):
@@ -55,6 +55,8 @@ class RootController( BaseController ):
         NOTE: No longer accepts "id" or "template" options for security reasons.
         """
         history = trans.get_history()
+        if trans.app.config.require_login and not trans.user:
+            return trans.fill_template( '/no_access.mako', message = 'Please log in to access Galaxy histories.' )
         if as_xml:
             trans.response.set_content_type('text/xml')
             return trans.fill_template_mako( "root/history_as_xml.mako", history=history )
@@ -88,7 +90,7 @@ class RootController( BaseController ):
             return trans.fill_template("root/history_item.mako", data=data, hid=hid)
         else:
             return trans.show_error_message( "Must specify a dataset id.")
-        
+
     @web.json
     def history_item_updates( self, trans, ids=None, states=None ):
         # Avoid caching
@@ -134,22 +136,25 @@ class RootController( BaseController ):
             except:
                 return "Dataset id '%s' is invalid" %str( id )
         if data:
-            mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() )
-            trans.response.set_content_type(mime)
-            if tofile:
-                fStat = os.stat(data.file_name)
-                trans.response.headers['Content-Length'] = int(fStat.st_size)
-                if toext[0:1] != ".":
-                    toext = "." + toext
-                valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-                fname = data.name
-                fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150]
-                trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, toext)
-            trans.log_event( "Display dataset id: %s" % str(id) )
-            try:
-                return open( data.file_name )
-            except: 
-                return "This dataset contains no content"
+            if trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_ACCESS, dataset = data ):
+                mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() )
+                trans.response.set_content_type(mime)
+                if tofile:
+                    fStat = os.stat(data.file_name)
+                    trans.response.headers['Content-Length'] = int(fStat.st_size)
+                    if toext[0:1] != ".":
+                        toext = "." + toext
+                    valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+                    fname = data.name
+                    fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150]
+                    trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, toext)
+                trans.log_event( "Display dataset id: %s" % str(id) )
+                try:
+                    return open( data.file_name )
+                except: 
+                    return "This dataset contains no content"
+            else:
+                return "You are not allowed to access this dataset"
         else:
             return "No dataset with id '%s'" % str( id )
 
@@ -161,9 +166,12 @@ class RootController( BaseController ):
         try:
             data = self.app.model.HistoryDatasetAssociation.get( parent_id )
             if data:
-                child = data.get_child_by_designation(designation)
+                child = data.get_child_by_designation( designation )
                 if child:
-                    return self.display(trans, id=child.id, tofile=tofile, toext=toext)
+                    if trans.app.security_agent.allow_action( trans.user, child.permitted_actions.DATASET_ACCESS, dataset = child ):
+                        return self.display( trans, id=child.id, tofile=tofile, toext=toext )
+                    else:
+                        return "You are not privileged to access this dataset."
         except Exception:
             pass
         return "A child named %s could not be found for data %s" % ( designation, parent_id )
@@ -173,9 +181,12 @@ class RootController( BaseController ):
         """Returns a file in a format that can successfully be displayed in display_app"""
         data = self.app.model.HistoryDatasetAssociation.get( id )
         if data:
-            trans.response.set_content_type(data.get_mime())
-            trans.log_event( "Formatted dataset id %s for display at %s" % ( str(id), display_app ) )
-            return data.as_display_type(display_app, **kwd)
+            if trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_ACCESS, dataset = data ):
+                trans.response.set_content_type( data.get_mime() )
+                trans.log_event( "Formatted dataset id %s for display at %s" % ( str( id ), display_app ) )
+                return data.as_display_type( display_app, **kwd )
+            else:
+                return "You are not privileged to access this dataset."
         else:
             return "No data with id=%d" % id
 
@@ -197,71 +208,89 @@ class RootController( BaseController ):
             history = trans.get_history()
             # TODO: hid handling
             data = history.datasets[ int( hid ) - 1 ]
-        elif id is None: 
-            return trans.show_error_message( "Problem loading dataset id %s with history id %s." % ( str( id ), str( hid ) ) )
-        else:
+        elif id is not None: 
             data = self.app.model.HistoryDatasetAssociation.get( id )
+        else:
+            trans.log_event( "Problem loading dataset id %s with history id %s." % ( str( id ), str( hid ) ) )
+            return trans.show_error_message( "Problem loading dataset." )
         if data is None:
-            return trans.show_error_message( "Problem retrieving dataset id %s with history id %s." % ( str( id ), str( hid ) ) )
+            trans.log_event( "Problem retrieving dataset id %s with history id." % ( str( id ), str( hid ) ) )
+            return trans.show_error_message( "Problem retrieving dataset." )
+        if id is not None and data.history.user is not None and data.history.user != trans.user:
+            return trans.show_error_message( "This instance of a dataset (%s) in a history does not belong to you." % ( data.id ) )
+        if trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_ACCESS, dataset=data ):
+            params = util.Params( kwd, safe=False )
+            if params.change:
+                # The user clicked the Save button on the 'Change data type' form
+                trans.app.datatypes_registry.change_datatype( data, params.datatype )
+                trans.app.model.flush()
+            elif params.save:
+                # The user clicked the Save button on the 'Edit Attributes' form
+                data.name  = params.name
+                data.info  = params.info
+                
+                # The following for loop will save all metadata_spec items
+                for name, spec in data.datatype.metadata_spec.items():
+                    if spec.get("readonly"):
+                        continue
+                    optional = params.get("is_"+name, None)
+                    if optional and optional == 'true':
+                        # optional element... == 'true' actually means it is NOT checked (and therefore omitted)
+                        setattr(data.metadata, name, None)
+                    else:
+                        setattr( data.metadata, name, spec.unwrap( params.get (name, None) ) )
 
-        p = util.Params(kwd, safe=False)
-        
-        if p.change:
-            # The user clicked the Save button on the 'Change data type' form
-            trans.app.datatypes_registry.change_datatype( data, p.datatype )
-            trans.app.model.flush()
-        elif p.save:
-            # The user clicked the Save button on the 'Edit Attributes' form
-            data.name  = p.name
-            data.info  = p.info
-            
-            # The following for loop will save all metadata_spec items
-            for name, spec in data.metadata.spec.items():
-                if spec.get("readonly"):
-                    continue
-                optional = p.get("is_"+name, None)
-                if optional and optional == 'true':
-                    # optional element... == 'true' actually means it is NOT checked (and therefore ommitted)
-                    setattr(data.metadata, name, None)
+                data.datatype.after_edit( data )
+                trans.app.model.flush()
+                return trans.show_ok_message( "Attributes updated", refresh_frames=['history'] )
+            elif params.detect:
+                # The user clicked the Auto-detect button on the 'Edit Attributes' form
+                for name, spec in data.metadata.spec.items():
+                    # We need to be careful about the attributes we are resetting
+                    if name not in [ 'name', 'info', 'dbkey' ]:
+                        if spec.get( 'default' ):
+                            setattr( data.metadata, name, spec.unwrap( spec.get( 'default' ) ) )
+                data.set_meta()
+                data.datatype.after_edit( data )
+                trans.app.model.flush()
+                return trans.show_ok_message( "Attributes updated", refresh_frames=['history'] )
+            elif params.convert_data:
+                target_type = kwd.get("target_type", None)
+                if target_type:
+                    msg = data.datatype.convert_dataset(trans, data, target_type)
+                    return trans.show_ok_message( msg, refresh_frames=['history'] )
+            elif params.update_roles_button:
+                if not trans.user:
+                    return trans.show_error_message( "You must be logged in if you want to change permissions." )
+                if trans.app.security_agent.allow_action( trans.user, data.dataset.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset = data.dataset ):
+                    permissions = {}
+                    for k, v in trans.app.model.Dataset.permitted_actions.items():
+                        in_roles = params.get( k + '_in', [] )
+                        if not isinstance( in_roles, list ):
+                            in_roles = [ in_roles ]
+                        in_roles = [ trans.app.model.Role.get( x ) for x in in_roles ]
+                        permissions[ trans.app.security_agent.get_action( v.action ) ] = in_roles
+                    trans.app.security_agent.set_all_dataset_permissions( data.dataset, permissions )
+                    data.dataset.refresh()
                 else:
-                    setattr( data.metadata, name, spec.unwrap( p.get (name, None) ) )
-
-            data.datatype.after_edit( data )
-            trans.app.model.flush()
-            return trans.show_ok_message( "Attributes updated", refresh_frames=['history'] )
-        elif p.detect:
-            # The user clicked the Auto-detect button on the 'Edit Attributes' form
-            for name, spec in data.metadata.spec.items():
-                # We need to be careful about the attributes we are resetting
-                if name not in [ 'name', 'info', 'dbkey' ]:
-                    if spec.get( 'default' ):
-                        setattr( data.metadata, name, spec.unwrap( spec.get( 'default' ) ) )
-            data.datatype.set_meta( data )
-            data.datatype.after_edit( data )
-            trans.app.model.flush()
-            return trans.show_ok_message( "Attributes updated", refresh_frames=['history'] )
-        elif p.convert_data:
-            """The user clicked the Convert button on the 'Convert to new format' form"""
-            target_type = kwd.get("target_type", None)
-            if target_type:
-                msg = data.datatype.convert_dataset(trans, data, target_type)
-                return trans.show_ok_message( msg, refresh_frames=['history'] )
-        data.datatype.before_edit( data )
-        
-        if "dbkey" in data.datatype.metadata_spec and not data.metadata.dbkey:
-            # Copy dbkey into metadata, for backwards compatability
-            # This looks like it does nothing, but getting the dbkey
-            # returns the metadata dbkey unless it is None, in which
-            # case it resorts to the old dbkey.  Setting the dbkey
-            # sets it properly in the metadata
-            data.metadata.dbkey = data.dbkey
-        # let's not overwrite the imported datatypes module with the variable datatypes?
-        ### the built-in 'id' is overwritten in lots of places as well
-        ldatatypes = [x for x in trans.app.datatypes_registry.datatypes_by_extension.iterkeys()]
-        ldatatypes.sort()
-        trans.log_event( "Opened edit view on dataset %s" % str(id) )
-        return trans.fill_template( "/dataset/edit_attributes.mako", data=data,
-                                    datatypes=ldatatypes, err=None )
+                    return trans.show_error_message( "You are not authorized to change this dataset's permissions" )
+            data.datatype.before_edit( data )
+            
+            if "dbkey" in data.datatype.metadata_spec and not data.metadata.dbkey:
+                # Copy dbkey into metadata, for backwards compatability
+                # This looks like it does nothing, but getting the dbkey
+                # returns the metadata dbkey unless it is None, in which
+                # case it resorts to the old dbkey.  Setting the dbkey
+                # sets it properly in the metadata
+                data.metadata.dbkey = data.dbkey
+            # let's not overwrite the imported datatypes module with the variable datatypes?
+            # the built-in 'id' is overwritten in lots of places as well
+            ldatatypes = [x for x in trans.app.datatypes_registry.datatypes_by_extension.iterkeys()]
+            ldatatypes.sort()
+            trans.log_event( "Opened edit view on dataset %s" % str(id) )
+            return trans.fill_template( "/dataset/edit_attributes.mako", data=data, datatypes=ldatatypes )
+        else:
+            return trans.show_error_message( "You do not have permission to edit this dataset's ( id: %s ) information." % str( id ) )
 
     def __delete_dataset( self, trans, id ):
         data = self.app.model.HistoryDatasetAssociation.get( id )
@@ -305,7 +334,7 @@ class RootController( BaseController ):
     def delete_async( self, trans, id = None, **kwd):
         if id:
             try:
-                int( id )
+                id = int( id )
             except:
                 return "Dataset id '%s' is invalid" %str( id )
             self.__delete_dataset( trans, id )
@@ -318,75 +347,13 @@ class RootController( BaseController ):
         """Displays a list of history related actions"""            
         return trans.fill_template( "/history/options.mako",
                                     user = trans.get_user(), history = trans.get_history() )
-
+        
     @web.expose
-    def history_delete( self, trans, id=None, **kwd):
-        """Deletes a list of histories, ensures that histories are owned by current user"""
-        history_names = []
-        if id:
-            if isinstance( id, list ):
-                history_ids = id
-            else:
-                history_ids = [ id ]
-            user = trans.get_user()
-            for hid in history_ids:
-                try:
-                    int( hid )
-                except:
-                    return trans.show_message( "Invalid history: %s" % str( hid ) )
-                history = self.app.model.History.get( hid )
-                if history:
-                    if history.user_id != None and user:
-                        assert user.id == history.user_id, "History does not belong to current user"
-                    history_names.append(history.name)
-                    history.deleted = True
-                    # If deleting the current history, make a new current.
-                    if history == trans.get_history():
-                        trans.new_history()
-                else:
-                    return trans.show_message( "Not able to find history %s" % str( hid ) )
-                self.app.model.flush()
-                trans.log_event( "History id %s marked as deleted" % str(hid) )
-        else:
-            return trans.show_message( "You must select at least one history to delete." )
-        return trans.show_message( "History deleted: %s" % ",".join(history_names),
-                                           refresh_frames=['history'])
-
-    @web.expose
-    def history_undelete( self, trans, id=[], **kwd):
-        """Undeletes a list of histories, ensures that histories are owned by current user"""
-        history_names = []
-        errors = []
-        ok_msg = ""
-        if id:
-            if not isinstance( id, list ):
-                id = id.split( "," )
-            user = trans.get_user()
-            for hid in id:
-                try:
-                    int( hid )
-                except:
-                    errors.append( "Invalid history: %s" % str( hid ) )
-                    continue
-                history = self.app.model.History.get( hid )
-                if history:
-                    if history.user != user:
-                        errors.append( "History does not belong to current user." )
-                        continue
-                    if history.purged:
-                        errors.append( "History has already been purged and can not be undeleted." )
-                        continue
-                    history_names.append( history.name )
-                    history.deleted = False
-                else:
-                    errors.append( "Not able to find history %s." % str( hid ) )
-                trans.log_event( "History id %s marked as undeleted" % str(hid) )
-            self.app.model.flush()
-            if history_names:
-                ok_msg = "Histories (%s) have been undeleted." % ", ".join( history_names )
-        else:
-            errors.append( "You must select at least one history to undelete." )
-        return self.history_available( trans, id=','.join( id ), show_deleted=True, ok_msg = ok_msg, error_msg = "  ".join( errors )  )
+    def history_delete( self, trans, id ):
+        """
+        Backward compatibility with check_galaxy script.
+        """
+        return trans.webapp.controllers['history'].list( trans, id, operation='delete' )
     
     @web.expose
     def clear_history( self, trans ):
@@ -398,60 +365,6 @@ class RootController( BaseController ):
         self.app.model.flush()
         trans.log_event( "History id %s cleared" % (str(history.id)) )
         trans.response.send_redirect( url_for("/index" ) )
-
-    @web.expose
-    @web.require_login( "share histories with other users" )
-    def history_share( self, trans, id=None, email="", **kwd ):
-        send_to_err = ""
-        if not id:
-            id = trans.get_history().id
-        if not isinstance( id, list ):
-            id = [ id ]
-        histories = []
-        history_names = []
-        for hid in id:
-            histories.append( trans.app.model.History.get( hid ) )
-            history_names.append(histories[-1].name) 
-        if not email:
-            return trans.fill_template("/history/share.mako", histories=histories, email=email, send_to_err=send_to_err)
-        user = trans.get_user()  
-        send_to_user = trans.app.model.User.filter_by( email=email ).first()
-        if not send_to_user:
-            send_to_err = "No such user"
-        elif user.email == email:
-            send_to_err = "You can't send histories to yourself"
-        else:
-            for history in histories:
-                new_history = history.copy()
-                new_history.name = history.name+" from "+user.email
-                new_history.user_id = send_to_user.id
-                trans.log_event( "History share, id: %s, name: '%s': to new id: %s" % (str(history.id), history.name, str(new_history.id)) )
-            self.app.model.flush()
-            return trans.show_message( "History (%s) has been shared with: %s" % (",".join(history_names),email) )
-        return trans.fill_template( "/history/share.mako", histories=histories, email=email, send_to_err=send_to_err)
-
-    @web.expose
-    @web.require_login( "work with multiple histories" )
-    def history_available( self, trans, id=[], do_operation = "view", show_deleted = False, ok_msg = "", error_msg="", as_xml=False, **kwd ):
-        """
-        List all available histories
-        """
-        if as_xml:
-            trans.response.set_content_type('text/xml')
-            return trans.fill_template( "/history/list_as_xml.mako" )
-        if not isinstance( id, list ):
-            id = id.split( "," )
-        trans.log_event( "History id %s available" % str( id ) )
-        history_operations = dict( share=self.history_share, rename=self.history_rename, delete=self.history_delete, undelete=self.history_undelete )
-        if do_operation in history_operations:
-            return history_operations[do_operation]( trans, id=id, show_deleted=show_deleted, ok_msg=ok_msg, error_msg=error_msg, **kwd  )
-        return trans.fill_template( "/history/list.mako", ids=id,
-                                    user=trans.get_user(),
-                                    current_history=trans.get_history(),
-                                    show_deleted=util.string_as_bool( show_deleted ),
-                                    ok_msg=ok_msg, error_msg=error_msg )
-        
-        
         
     @web.expose
     def history_import( self, trans, id=None, confirm=False, **kwd ):
@@ -466,7 +379,7 @@ class RootController( BaseController ):
         if user:
             if import_history.user_id == user.id:
                 return trans.show_error_message( "You cannot import your own history.")
-            new_history = import_history.copy()
+            new_history = import_history.copy( target_user=trans.user )
             new_history.name = "imported: "+new_history.name
             new_history.user_id = user.id
             galaxy_session = trans.get_galaxy_session()
@@ -502,28 +415,6 @@ class RootController( BaseController ):
             Warning! If you import this history, you will lose your current
             history. Click <a href="%s">here</a> to confirm.
             """ % web.url_for( id=id, confirm=True ) )
-
-    @web.expose
-    @web.require_login( "switch histories" )
-    def history_switch( self, trans, id=None ):
-        if not id:
-            return trans.response.send_redirect( web.url_for( action='history_available' ) )
-        else:
-            new_history = trans.app.model.History.get( id )
-            if new_history:
-                galaxy_session = trans.get_galaxy_session()
-                try:
-                    association = trans.app.model.GalaxySessionToHistoryAssociation.filter_by( session_id=galaxy_session.id, history_id=new_history.id ).first()
-                except:
-                    association = None
-                new_history.add_galaxy_session( galaxy_session, association=association )
-                new_history.flush()
-                trans.set_history( new_history )
-                trans.log_event( "History switched to id: %s, name: '%s'" % (str(new_history.id), new_history.name ) )
-                return trans.show_message( "History switched to: %s" % new_history.name,
-                                           refresh_frames=['history'])
-            else:
-                return trans.show_error_message( "History not found" )
                 
     @web.expose
     def history_new( self, trans ):
@@ -532,52 +423,17 @@ class RootController( BaseController ):
         return trans.show_message( "New history created", refresh_frames = ['history'] )
 
     @web.expose
-    @web.require_login( "renames histories" )
-    def history_rename( self, trans, id=None, name=None, **kwd ):
-        user = trans.get_user()
-
-        if not isinstance( id, list ):
-            if id != None:
-                id = [ id ]
-        if not isinstance( name, list ):
-            if name != None:
-                name = [ name ]
-        histories = []
-        cur_names = []
-        if not id:
-            if not trans.get_history().user:
-                return trans.show_error_message( "You must save your history before renaming it." )
-            id = [trans.get_history().id]
-        for history_id in id:
-            history = trans.app.model.History.get( history_id )
-            if history and history.user_id == user.id:
-                histories.append(history)
-                cur_names.append(history.name)
-        if not name or len(histories)!=len(name):
-            return trans.fill_template( "/history/rename.mako",histories=histories )
-        change_msg = ""
-        for i in range(len(histories)):
-            if histories[i].user_id == user.id:
-                if name[i] == histories[i].name:
-                    change_msg = change_msg + "<p>History: "+cur_names[i]+" is already named: "+name[i]+"</p>"
-                elif name[i] not in [None,'',' ']:
-                    name[i] = escape(name[i])
-                    histories[i].name = name[i]
-                    histories[i].flush()
-                    change_msg = change_msg + "<p>History: "+cur_names[i]+" renamed to: "+name[i]+"</p>"
-                    trans.log_event( "History renamed: id: %s, renamed to: '%s'" % (str(histories[i].id), name[i] ) )
-                else:
-                    change_msg = change_msg + "<p>You must specify a valid name for History: "+cur_names[i]+"</p>"
-            else:
-                change_msg = change_msg + "<p>History: "+cur_names[i]+" does not appear to belong to you.</p>"
-        return trans.show_message( "<p>%s" % change_msg, refresh_frames=['history'] ) 
-
-    @web.expose
-    def history_add_to( self, trans, history_id=None, file_data=None, name="Data Added to History",info=None,ext="txt",dbkey="?",**kwd ):
+    def history_add_to( self, trans, history_id=None, file_data=None, name="Data Added to History",info=None,ext="txt",dbkey="?",copy_access_from=None,**kwd ):
         """Adds a POSTed file to a History"""
         try:
             history = trans.app.model.History.get( history_id )
             data = trans.app.model.HistoryDatasetAssociation( name = name, info = info, extension = ext, dbkey = dbkey, create_dataset = True )
+            if copy_access_from:
+                copy_access_from = trans.app.model.HistoryDatasetAssociation.get( copy_access_from )
+                trans.app.security_agent.copy_dataset_permissions( copy_access_from.dataset, data.dataset )
+            else:
+                permissions = trans.app.security_agent.history_get_default_permissions( history )
+                trans.app.security_agent.set_all_dataset_permissions( data.dataset, permissions )
             data.flush()
             data_file = open( data.file_name, "wb" )
             file_data.file.seek( 0 )
@@ -594,9 +450,33 @@ class RootController( BaseController ):
             data.flush()
             trans.log_event("Added dataset %d to history %d" %(data.id, trans.history.id))
             return trans.show_ok_message("Dataset "+str(data.hid)+" added to history "+str(history_id)+".")
-        except:
+        except Exception, e:
+            trans.log_event( "Failed to add dataset to history: %s" % ( e ) )
             return trans.show_error_message("Adding File to History has Failed")
 
+    @web.expose
+    def history_set_default_permissions( self, trans, **kwd ):
+        """Sets the user's default permissions for the current history"""
+        if trans.user:
+            if 'update_roles_button' in kwd:
+                history = trans.get_history()
+                p = util.Params( kwd )
+                permissions = {}
+                for k, v in trans.app.model.Dataset.permitted_actions.items():
+                    in_roles = p.get( k + '_in', [] )
+                    if not isinstance( in_roles, list ):
+                        in_roles = [ in_roles ]
+                    in_roles = [ trans.app.model.Role.get( x ) for x in in_roles ]
+                    permissions[ trans.app.security_agent.get_action( v.action ) ] = in_roles
+                dataset = 'dataset' in kwd
+                bypass_manage_permission = 'bypass_manage_permission' in kwd
+                trans.app.security_agent.history_set_default_permissions( history, permissions, dataset=dataset, bypass_manage_permission=bypass_manage_permission )
+                return trans.show_ok_message( 'Default history permissions have been changed.' )
+            return trans.fill_template( 'history/permissions.mako' )
+        else:
+            #user not logged in, history group must be only public
+            return trans.show_error_message( "You must be logged in to change a history's default permissions." )
+
     @web.expose
     def dataset_make_primary( self, trans, id=None):
         """Copies a dataset and makes primary"""
@@ -612,17 +492,24 @@ class RootController( BaseController ):
         except:
             return trans.show_error_message( "<p>Failed to make secondary dataset primary.</p>" ) 
 
-    @web.expose
-    def masthead( self, trans ):
-        brand = trans.app.config.get( "brand", "" )
-        if brand:
-            brand ="<span class='brand'>/%s</span>" % brand
-        wiki_url = trans.app.config.get( "wiki_url", "http://g2.trac.bx.psu.edu/" )
-        bugs_email = trans.app.config.get( "bugs_email", "mailto:galaxy-bugs@bx.psu.edu"  )
-        blog_url = trans.app.config.get( "blog_url", "http://g2.trac.bx.psu.edu/blog"   )
-        screencasts_url = trans.app.config.get( "screencasts_url", "http://g2.trac.bx.psu.edu/wiki/ScreenCasts" )
-        return trans.fill_template( "/root/masthead.mako", brand=brand, wiki_url=wiki_url, 
-          blog_url=blog_url,bugs_email=bugs_email, screencasts_url=screencasts_url )
+    # @web.expose
+    # def masthead( self, trans, active_view=None ):
+    #     brand = trans.app.config.get( "brand", "" )
+    #     if brand:
+    #         brand ="<span class='brand'>/%s</span>" % brand
+    #     wiki_url = trans.app.config.get( "wiki_url", "http://g2.trac.bx.psu.edu/" )
+    #     bugs_email = trans.app.config.get( "bugs_email", "mailto:galaxy-bugs@bx.psu.edu"  )
+    #     blog_url = trans.app.config.get( "blog_url", "http://g2.trac.bx.psu.edu/blog"   )
+    #     screencasts_url = trans.app.config.get( "screencasts_url", "http://g2.trac.bx.psu.edu/wiki/ScreenCasts" )
+    #     admin_user = "false"
+    #     admin_users = trans.app.config.get( "admin_users", "" ).split( "," )
+    #     user = trans.get_user()
+    #     if user:
+    #         user_email = trans.get_user().email
+    #         if user_email in admin_users:
+    #             admin_user = "true"
+    #     return trans.fill_template( "/root/masthead.mako", brand=brand, wiki_url=wiki_url, 
+    #       blog_url=blog_url,bugs_email=bugs_email, screencasts_url=screencasts_url, admin_user=admin_user, active_view=active_view )
 
     @web.expose
     def dataset_errors( self, trans, id=None, **kwd ):
@@ -665,3 +552,7 @@ class RootController( BaseController ):
     @web.expose
     def generate_error( self, trans ):
         raise Exception( "Fake error!" )
+
+
+
+

lib/galaxy/web/controllers/user.py

@@ -1,14 +1,25 @@
 """
 Contains the user interface in the Universe class
 """
-
 from galaxy.web.base.controller import *
-
+from galaxy.model.orm import *
+from galaxy import util
 import logging, os, string
 from random import choice
 
 log = logging.getLogger( __name__ )
 
+require_login_template = """
+<h1>Welcome to Galaxy</h1>
+
+<p>
+    This installation of Galaxy has been configured such that only users who are logged in may use it.%s
+</p>
+<p/>
+"""
+require_login_nocreation_template = require_login_template % ""
+require_login_creation_template = require_login_template % "  If you don't already have an account, <a href='%s'>you may create one</a>."
+
 class User( BaseController ):
     
     @web.expose
@@ -72,40 +83,69 @@ class User( BaseController ):
     def login( self, trans, email='', password='' ):
         email_error = password_error = None
         # Attempt login
+        if trans.app.config.require_login:
+            refresh_frames = [ 'masthead', 'history', 'tools' ]
+        else:
+            refresh_frames = [ 'masthead', 'history' ]
         if email or password:
-            user = trans.app.model.User.filter_by( email=email ).first()
+            user = trans.app.model.User.filter( trans.app.model.User.table.c.email==email ).first()
             if not user:
                 email_error = "No such user"
+            elif user.deleted:
+                email_error = "This account has been marked deleted, contact your Galaxy administrator to restore the account."
             elif user.external:
-                return trans.show_error_message( "This account was created for use with an external authentication "
-                                               + "method.  Please contact your local Galaxy administrator to activate it." )
+                email_error = "This account was created for use with an external authentication method, contact your local Galaxy administrator to activate it."
             elif not user.check_password( password ):
                 password_error = "Invalid password"
             else:
                 trans.handle_user_login( user )
                 trans.log_event( "User logged in" )
-                return trans.show_ok_message( "Now logged in as " + user.email, refresh_frames=['masthead', 'history'] )
-        return trans.show_form( 
-            web.FormBuilder( web.url_for(), "Login", submit_text="Login" )
-                .add_text( "email", "Email address", value=email, error=email_error )
+                msg = "Now logged in as " + user.email + "."
+                if trans.app.config.require_login:
+                    msg += '  <a href="%s">Click here</a> to continue to the front page.' % web.url_for( '/static/welcome.html' )
+                return trans.show_ok_message( msg, refresh_frames=refresh_frames )
+        form = web.FormBuilder( web.url_for(), "Login", submit_text="Login" ) \
+                .add_text( "email", "Email address", value=email, error=email_error ) \
                 .add_password( "password", "Password", value='', error=password_error, 
-                                help="<a href='%s'>Forgot password? Reset here</a>" % web.url_for( action='reset_password' ) ) )
+                                help="<a href='%s'>Forgot password? Reset here</a>" % web.url_for( action='reset_password' ) )
+        if trans.app.config.require_login:
+            if trans.app.config.allow_user_creation:
+                return trans.show_form( form, header = require_login_creation_template % web.url_for( action = 'create' ) )
+            else:
+                return trans.show_form( form, header = require_login_nocreation_template )
+        else:
+            return trans.show_form( form )
+
     @web.expose
     def logout( self, trans ):
+        if trans.app.config.require_login:
+            refresh_frames = [ 'masthead', 'history', 'tools' ]
+        else:
+            refresh_frames = [ 'masthead', 'history' ]
         # Since logging an event requires a session, we'll log prior to ending the session
         trans.log_event( "User logged out" )
         trans.handle_user_logout()
-        return trans.show_ok_message( "You are no longer logged in", refresh_frames=['masthead', 'history'] )
-            
+        msg = "You are no longer logged in."
+        if trans.app.config.require_login:
+            msg += '  <a href="%s">Click here</a> to return to the login page.' % web.url_for( controller='user', action='login' )
+        return trans.show_ok_message( msg, refresh_frames=refresh_frames )
+
     @web.expose
-    def create( self, trans, email='', password='', confirm='',subscribe=False ):
+    def create( self, trans, email='', password='', confirm='', subscribe=False ):
+        if trans.app.config.require_login:
+            refresh_frames = [ 'masthead', 'history', 'tools' ]
+        else:
+            refresh_frames = [ 'masthead', 'history' ]
+        if not trans.app.config.allow_user_creation and not trans.user_is_admin():
+            return trans.show_error_message( 'User registration is disabled.  Please contact your Galaxy administrator for an account.' )
         email_error = password_error = confirm_error = None
         if email:
             if len( email ) == 0 or "@" not in email or "." not in email:
                 email_error = "Please enter a real email address"
-            elif len( email) > 255:
+            elif len( email ) > 255:
                 email_error = "Email address exceeds maximum allowable length"
-            elif trans.app.model.User.filter_by( email=email ).first():
+            elif trans.app.model.User.filter( and_( trans.app.model.User.table.c.email==email,
+                                                    trans.app.model.User.table.c.deleted==False ) ).first():
                 email_error = "User with that email already exists"
             elif len( password ) < 6:
                 password_error = "Please use a password of at least 6 characters"
@@ -115,6 +155,11 @@ class User( BaseController ):
                 user = trans.app.model.User( email=email )
                 user.set_password_cleartext( password )
                 user.flush()
+                trans.app.security_agent.create_private_user_role( user )
+                # We set default user permissions, before we log in and set the default history permissions
+                trans.app.security_agent.user_set_default_permissions( user )
+                # The handle_user_login() method has a call to the history_set_default_permissions() method
+                # (needed when logging in with a history), user needs to have default permissions set before logging in
                 trans.handle_user_login( user )
                 trans.log_event( "User created a new account" )
                 trans.log_event( "User logged in" )
@@ -133,13 +178,13 @@ class User( BaseController ):
                 .add_input( "checkbox","Subscribe To Mailing List","subscribe", value='subscribe' ) )
 
     @web.expose
-    def reset_password(self, trans, email=None, **kwd):
+    def reset_password( self, trans, email=None, **kwd ):
         error = ''
-        reset_user = trans.app.model.User.filter_by( email=email ).first()
+        reset_user = trans.app.model.User.filter( trans.app.model.User.table.c.email==email ).first()
         user = trans.get_user()
         if reset_user:
             if user and user.id != reset_user.id:
-                    error = "You may only reset your own password"
+                error = "You may only reset your own password"
             else:
                 chars = string.letters + string.digits
                 new_pass = ""
@@ -148,13 +193,34 @@ class User( BaseController ):
                 mail = os.popen("%s -t" % trans.app.config.sendmail_path, 'w')
                 mail.write("To: %s\nFrom: no-reply@%s\nSubject: Galaxy Password Reset\n\nYour password has been reset to \"%s\" (no quotes)." % (email, trans.request.remote_addr, new_pass) )
                 if mail.close():
-                    return trans.show_ok_message( "Failed to reset password! If this problem persist, submit a bug report.")
+                    return trans.show_error_message( 'Failed to reset password.  If this problem persists, please submit a bug report.' )
                 reset_user.set_password_cleartext( new_pass )
                 reset_user.flush()
                 trans.log_event( "User reset password: %s" % email )
-                return trans.show_ok_message( "Password has been reset and emailed to: %s." % email)
+                return trans.show_ok_message( "Password has been reset and emailed to: %s.  <a href='%s'>Click here</a> to return to the login form." % ( email, web.url_for( action='login' ) ) )
         elif email != None:
             error = "The specified user does not exist"
         return trans.show_form( 
             web.FormBuilder( web.url_for(), "Reset Password", submit_text="Submit" )
                 .add_text( "email", "Email", value=email, error=error ) )
+    
+    @web.expose
+    def set_default_permissions( self, trans, **kwd ):
+        """Sets the user's default permissions for the new histories"""
+        if trans.user:
+            if 'update_roles_button' in kwd:
+                p = util.Params( kwd )
+                permissions = {}
+                for k, v in trans.app.model.Dataset.permitted_actions.items():
+                    in_roles = p.get( k + '_in', [] )
+                    if not isinstance( in_roles, list ):
+                        in_roles = [ in_roles ]
+                    in_roles = [ trans.app.model.Role.get( x ) for x in in_roles ]
+                    action = trans.app.security_agent.get_action( v.action ).action
+                    permissions[ action ] = in_roles
+                trans.app.security_agent.user_set_default_permissions( trans.user, permissions )
+                return trans.show_ok_message( 'Default new history permissions have been changed.' )
+            return trans.fill_template( 'user/permissions.mako' )
+        else:
+            # User not logged in, history group must be only public
+            return trans.show_error_message( "You must be logged in to change your default permitted actions." )

lib/galaxy/web/controllers/workflow.py

@@ -13,12 +13,17 @@ from galaxy.util.bunch import Bunch
 from galaxy.util.topsort import topsort, topsort_levels, CycleError
 from galaxy.workflow.modules import *
 from galaxy.model.mapping import desc
+from galaxy.model.orm import *
 
 class WorkflowController( BaseController ):
     
     @web.expose
-    @web.require_login( "use Galaxy workflows" )
     def index( self, trans ):
+        return trans.fill_template( "workflow/index.mako" )
+                                   
+    @web.expose
+    @web.require_login( "use Galaxy workflows" )
+    def list( self, trans ):
         """
         Render workflow main page (management of existing workflows)
         """
@@ -33,7 +38,29 @@ class WorkflowController( BaseController ):
             .filter( model.StoredWorkflow.c.deleted == False ) \
             .order_by( desc( model.StoredWorkflow.c.update_time ) ) \
             .all()
-        return trans.fill_template( "workflow/index.mako",
+        return trans.fill_template( "workflow/list.mako",
+                                    workflows = workflows,
+                                    shared_by_others = shared_by_others )
+    
+    @web.expose
+    @web.require_login( "use Galaxy workflows" )
+    def list_for_run( self, trans ):
+        """
+        Render workflow list for analysis view (just allows running workflow
+        or switching to management view)
+        """
+        user = trans.get_user()
+        workflows = trans.sa_session.query( model.StoredWorkflow ) \
+            .filter_by( user=user, deleted=False ) \
+            .order_by( desc( model.StoredWorkflow.c.update_time ) ) \
+            .all()
+        shared_by_others = trans.sa_session \
+            .query( model.StoredWorkflowUserShareAssociation ) \
+            .filter_by( user=user ) \
+            .filter( model.StoredWorkflow.c.deleted == False ) \
+            .order_by( desc( model.StoredWorkflow.c.update_time ) ) \
+            .all()
+        return trans.fill_template( "workflow/list_for_run.mako",
                                     workflows = workflows,
                                     shared_by_others = shared_by_others )
     
@@ -44,7 +71,8 @@ class WorkflowController( BaseController ):
         # Load workflow from database
         stored = get_stored_workflow( trans, id )
         if email:
-            other = model.User.filter_by( email=email ).first()
+            other = model.User.filter( and_( model.User.table.c.email==email,
+                                             model.User.table.c.deleted==False ) ).first()
             if not other:
                 mtype = "error"
                 msg = ( "User '%s' does not exist" % email )
@@ -61,10 +89,10 @@ class WorkflowController( BaseController ):
                 share.stored_workflow = stored
                 share.user = other
                 session = trans.sa_session
-                session.save( share )
+                session.save_or_update( share )
                 session.flush()
                 trans.set_message( "Workflow '%s' shared with user '%s'" % ( stored.name, other.email ) )
-                return self.index( trans )
+                return self.list( trans )
         return trans.fill_template( "workflow/share.mako",
                                     message = msg,
                                     messagetype = mtype,
@@ -79,7 +107,7 @@ class WorkflowController( BaseController ):
             stored.name = new_name
             trans.sa_session.flush()
             trans.set_message( "Workflow renamed to '%s'." % new_name )
-            return self.index( trans )
+            return self.list( trans )
         else:
             return form( url_for( id=trans.security.encode_id(stored.id) ), "Rename workflow", submit_text="Rename" ) \
                 .add_text( "new_name", "Workflow Name", value=stored.name )
@@ -104,11 +132,11 @@ class WorkflowController( BaseController ):
         new_stored.user = user
         # Persist
         session = trans.sa_session
-        session.save( new_stored )
+        session.save_or_update( new_stored )
         session.flush()
         # Display the management page
         trans.set_message( 'Clone created with name "%s"' % new_stored.name )
-        return self.index( trans )
+        return self.list( trans )
     
     @web.expose
     @web.require_login( "create workflows" )
@@ -129,11 +157,11 @@ class WorkflowController( BaseController ):
             stored_workflow.latest_workflow = workflow
             # Persist
             session = trans.sa_session
-            session.save( stored_workflow )
+            session.save_or_update( stored_workflow )
             session.flush()
             # Display the management page
             trans.set_message( "Workflow '%s' created" % stored_workflow.name )
-            return self.index( trans )
+            return self.list( trans )
         else:
             return form( url_for(), "Create new workflow", submit_text="Create" ) \
                 .add_text( "workflow_name", "Workflow Name", value="Unnamed workflow" )
@@ -150,7 +178,7 @@ class WorkflowController( BaseController ):
         stored.flush()
         # Display the management page
         trans.set_message( "Workflow '%s' deleted" % stored.name )
-        return self.index( trans )
+        return self.list( trans )
         
     @web.expose
     @web.require_login( "edit workflows" )
@@ -430,11 +458,10 @@ class WorkflowController( BaseController ):
             stored.name = workflow_name
             workflow.stored_workflow = stored
             stored.latest_workflow = workflow
-            trans.sa_session.save( stored )
+            trans.sa_session.save_or_update( stored )
             trans.sa_session.flush()
             # Index page with message
-            trans.template_context['message'] = "Workflow '%s' created" % workflow_name
-            return self.index( trans )
+            return trans.show_message( "Workflow '%s' created from current history." % workflow_name )
             ## return trans.show_ok_message( "<p>Workflow '%s' created.</p><p><a target='_top' href='%s'>Click to load in workflow editor</a></p>"
             ##     % ( workflow_name, web.url_for( action='editor', id=trans.security.encode_id(stored.id) ) ) )       
         
@@ -507,8 +534,8 @@ class WorkflowController( BaseController ):
                                 elif isinstance( input, Conditional ):
                                     values = input_values[ input.name ]
                                     current = values["__current_case__"]
-                                    prefix = prefix + "|" + input.name
-                                    visitor( input.cases[current].inputs, values, prefix )
+                                    new_prefix = prefix + input.name + "|"
+                                    visitor( input.cases[current].inputs, values, new_prefix )
                                 else:
                                     if isinstance( input, DataToolParameter ):
                                         prefixed_name = prefix + input.name

lib/galaxy/web/form_builder.py

@@ -130,6 +130,7 @@ class SelectField(BaseField):
     >>> t.add_option( "automatic", 3 )
     >>> t.add_option( "bazooty", 4, selected=True )
     >>> print t.get_html()
+    <div class="checkUncheckAllPlaceholder" checkbox_name="bar"></div>
     <div><input type="checkbox" name="bar" value="3">automatic</div>
     <div><input type="checkbox" name="bar" value="4" checked>bazooty</div>
     """
@@ -161,6 +162,8 @@ class SelectField(BaseField):
     def get_html_checkboxes( self, prefix="" ):
         rval = []
         ctr = 0
+        if len( self.options ) > 1:
+            rval.append ( '<div class="checkUncheckAllPlaceholder" checkbox_name="%s%s"></div>' % ( prefix, self.name ) ) #placeholder for the insertion of the Select All/Unselect All buttons
         for text, value, selected in self.options:
             style = ""
             if len(self.options) > 2 and ctr % 2 == 1:

lib/galaxy/web/framework/__init__.py

@@ -4,7 +4,7 @@ Galaxy web application framework
 
 import pkg_resources
 
-import os, sys, time, random, string
+import os, sys, time, socket, random, string
 pkg_resources.require( "Cheetah" )
 from Cheetah.Template import Template
 import base
@@ -32,6 +32,17 @@ log = logging.getLogger( __name__ )
 
 url_for = base.routes.url_for
 
+UCSC_SERVERS = (
+    'hgw1.cse.ucsc.edu',
+    'hgw2.cse.ucsc.edu',
+    'hgw3.cse.ucsc.edu',
+    'hgw4.cse.ucsc.edu',
+    'hgw5.cse.ucsc.edu',
+    'hgw6.cse.ucsc.edu',
+    'hgw7.cse.ucsc.edu',
+    'hgw8.cse.ucsc.edu',
+)
+
 def expose( func ):
     """
     Decorator: mark a function as 'exposed' and thus web accessible
@@ -60,6 +71,19 @@ def require_login( verb="perform this action" ):
         return decorator
     return argcatcher
     
+def require_admin( func ):
+    def decorator( self, trans, *args, **kwargs ):
+        admin_users = trans.app.config.get( "admin_users", "" ).split( "," )
+        if not admin_users:
+            return trans.show_error_message( "You must be logged in as an administrator to access this feature, but no administrators are set in the Galaxy configuration." )
+        user = trans.get_user()
+        if not user:
+            return trans.show_error_message( "You must be logged in as an administrator to access this feature." )
+        if not user.email in admin_users:
+            return trans.show_error_message( "You must be an administrator to access this feature." )
+        return func( self, trans, *args, **kwargs )
+    return decorator
+
 NOT_SET = object()
 
 class MessageException( Exception ):
@@ -118,6 +142,8 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
         self.workflow_building_mode = False
         # Always have a valid galaxy session
         self.__ensure_valid_session( session_cookie )
+        if self.app.config.require_login:
+            self.__ensure_logged_in_user( environ )
     @property
     def sa_session( self ):
         """
@@ -223,8 +249,7 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
                     galaxy_session.user = self.__get_or_create_remote_user( remote_user_email )
                     galaxy_session_requires_flush = True
                 elif galaxy_session.user.email != remote_user_email:
-                    # Session exists but is not associated with the correct
-                    # remote user
+                    # Session exists but is not associated with the correct remote user
                     invalidate_existing_session = True
                     user_for_new_session = self.__get_or_create_remote_user( remote_user_email )
                     log.warning( "User logged in as '%s' externally, but has a cookie as '%s' invalidating session",
@@ -260,6 +285,28 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
             if prev_galaxy_session:
                 objects_to_flush.append( prev_galaxy_session )            
             sa_session.flush( objects_to_flush )
+    def __ensure_logged_in_user( self, environ ):
+        allowed_paths = (
+            url_for( controller='root', action='index' ),
+            url_for( controller='root', action='tool_menu' ),
+            url_for( controller='root', action='masthead' ),
+            url_for( controller='root', action='history' ),
+            url_for( controller='user', action='login' ),
+            url_for( controller='user', action='create' ),
+            url_for( controller='user', action='reset_password' ),
+            url_for( controller='library', action='browse' )
+        )
+        display_as = url_for( controller='root', action='display_as' )
+        if self.galaxy_session.user is None:
+            if self.app.config.ucsc_display_sites and self.request.path == display_as:
+                try:
+                    host = socket.gethostbyaddr( self.environ[ 'REMOTE_ADDR' ] )[0]
+                except( socket.error, socket.herror, socket.gaierror, socket.timeout ):
+                    host = None
+                if host in UCSC_SERVERS:
+                    return
+            if self.request.path not in allowed_paths:
+                self.response.send_redirect( url_for( controller='root', action='index' ) )
     def __create_new_session( self, prev_galaxy_session=None, user_for_new_session=None ):
         """
         Create a new GalaxySession for this request, possibly with a connection
@@ -287,7 +334,7 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
         Return the user in $HTTP_REMOTE_USER and create if necessary
         """
         # remote_user middleware ensures HTTP_REMOTE_USER exists
-        user = self.app.model.User.filter_by( email=remote_user_email ).first()
+        user = self.app.model.User.filter( self.app.model.User.table.c.email==remote_user_email ).first()
         if user is None:
             random.seed()
             user = self.app.model.User( email=remote_user_email )
@@ -295,6 +342,8 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
             user.external = True
             user.flush()
             #self.log_event( "Automatically created account '%s'", user.email )
+        elif user.deleted:
+            return self.show_error_message( "Your account is no longer valid, contact your Galaxy administrator to activate your account." )
         return user
     def __update_session_cookie( self, name='galaxysession' ):
         """
@@ -307,20 +356,25 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
         Login a new user (possibly newly created)
            - create a new session
            - associate new session with user
-           - if old session had a history and it was not associated with a user, associate it with the new session.
+           - if old session had a history and it was not associated with a user, associate it with the new session, 
+             otherwise associate the current session's history with the user
         """
         prev_galaxy_session = self.galaxy_session
         prev_galaxy_session.is_valid = False
         self.galaxy_session = self.__create_new_session( prev_galaxy_session, user )
         if prev_galaxy_session.current_history:
             history = prev_galaxy_session.current_history
-            if history.user is None:
-                self.galaxy_session.add_history( history )
-                self.galaxy_session.current_history = history
-                history.user = user   
-            self.sa_session.flush( [ prev_galaxy_session, self.galaxy_session, history ] )
+        elif self.galaxy_session.current_history:
+            history = self.galaxy_session.current_history
         else:
-            self.sa_session.flush( [ prev_galaxy_session, self.galaxy_session ] )
+            history = self.history
+        if history not in self.galaxy_session.histories:
+            self.galaxy_session.add_history( history )
+        if history.user is None:
+            history.user = user
+        self.galaxy_session.current_history = history
+        self.app.security_agent.history_set_default_permissions( history, dataset=True, bypass_manage_permission=True )
+        self.sa_session.flush( [ prev_galaxy_session, self.galaxy_session, history ] )
         # This method is not called from the Galaxy reports, so the cookie will always be galaxysession
         self.__update_session_cookie( name='galaxysession' )
     def handle_user_logout( self ):
@@ -376,6 +430,8 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
             history.user = self.galaxy_session.user
         # Track genome_build with history
         history.genome_build = util.dbnames.default_value
+        # Set the user's default history permissions
+        self.app.security_agent.history_set_default_permissions( history )
         # Save
         self.sa_session.flush( [ self.galaxy_session, history ] )
         return history
@@ -388,7 +444,13 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
         self.galaxy_session.user = user
         self.sa_session.flush( [ self.galaxy_session ] )
     user = property( get_user, set_user )
-                
+
+    def user_is_admin( self ):
+        admin_users = self.app.config.get( "admin_users", "" ).split( "," )
+        if self.user and admin_users and self.user.email in admin_users:
+            return True
+        return False
+
     def get_toolbox(self):
         """Returns the application toolbox"""
         return self.app.toolbox
@@ -434,12 +496,12 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
         Convenience method for displaying an warn message. See `show_message`.
         """
         return self.show_message( message, 'warning', refresh_frames )
-    def show_form( self, form ):
+    def show_form( self, form, header=None, template="form.mako" ):
         """
         Convenience method for displaying a simple page with a single HTML
         form.
         """    
-        return self.fill_template( "form.mako", form=form )
+        return self.fill_template( template, form=form, header=header )
     def fill_template(self, filename, **kwargs):
         """
         Fill in a template, putting any keyword arguments on the context.
@@ -476,8 +538,8 @@ class FormBuilder( object ):
         self.action = action
         self.submit_text = submit_text
         self.inputs = []
-    def add_input( self, type, name, label, value=None, error=None, help=None  ):
-        self.inputs.append( FormInput( type, label, name, value, error, help ) )
+    def add_input( self, type, name, label, value=None, error=None, help=None, use_label=True  ):
+        self.inputs.append( FormInput( type, label, name, value, error, help, use_label ) )
         return self
     def add_text( self, name, label, value=None, error=None, help=None  ):
         return self.add_input( 'text', label, name, value, error, help )
@@ -488,13 +550,14 @@ class FormInput( object ):
     """
     Simple class describing a form input element
     """
-    def __init__( self, type, name, label, value=None, error=None, help=None ):
+    def __init__( self, type, name, label, value=None, error=None, help=None, use_label=True ):
         self.type = type
         self.name = name
         self.label = label
         self.value = value
         self.error = error
         self.help = help
+        self.use_label = use_label
     
 class FormData( object ):
     """
@@ -514,3 +577,4 @@ class Bunch( dict ):
         return self[key]
     def __setattr__( self, key, value ):
         self[key] = value
+

lib/galaxy/web/framework/base.py

@@ -84,15 +84,19 @@ class WebApplication( object ):
         friendly objects, finds the appropriate method to handle the request
         and calls it.
         """
-        # Setup the transaction
-        trans = self.transaction_factory( environ )
         # Map url using routes
-        path_info = trans.request.path_info
+        path_info = environ.get( 'PATH_INFO', '' )
         map = self.mapper.match( path_info )
         if map == None:
             raise httpexceptions.HTTPNotFound( "No route for " + path_info )
-        # Save the complete mapper dict, we pop things off so they don't get passed down
-        raw_map = dict( map )
+        # Setup routes
+        rc = routes.request_config()
+        rc.mapper = self.mapper
+        rc.mapper_dict = map
+        rc.environ = environ
+        # Setup the transaction
+        trans = self.transaction_factory( environ )
+        rc.redirect = trans.response.send_redirect
         # Get the controller class
         controller_name = map.pop( 'controller', None )
         controller = self.controllers.get( controller_name, None )
@@ -111,12 +115,6 @@ class WebApplication( object ):
         # Is the method callable
         if not callable( method ):
             raise httpexceptions.HTTPNotFound( "Action not callable for " + path_info ) 
-        # Setup routes
-        rc = routes.request_config()
-        rc.mapper = self.mapper
-        rc.mapper_dict = raw_map
-        rc.environ = environ
-        rc.redirect = trans.response.send_redirect
         # Combine mapper args and query string / form args and call
         kwargs = trans.request.params.mixed()
         kwargs.update( map )
@@ -277,7 +275,7 @@ class Response( object ):
         """
         Send an HTTP redirect response to (target `url`)
         """
-        raise httpexceptions.HTTPFound( url )
+        raise httpexceptions.HTTPFound( url, headers=self.wsgi_headeritems() )
     def wsgi_headeritems( self ):
         """
         Return headers in format appropriate for WSGI `start_response`
@@ -336,4 +334,4 @@ def flatten( seq ):
             for y in flatten( x, encoding ):
                 yield y
         else:
-            yield x
+            yield x

lib/galaxy/web/framework/helpers/grids.py

@@ -0,0 +1,162 @@
+from galaxy.model import *
+from galaxy.model.orm import *
+
+from galaxy.web import url_for
+
+import sys
+
+class Grid( object ):
+    """
+    Specifieds the content and format of a grid (data table).
+    """
+    title = ""
+    exposed = True
+    model_class = None
+    columns = []
+    standard_filters = []
+    default_filter = None
+    default_sort_key = None
+    def __init__( self ):
+        pass
+    def __call__( self, trans, **kwargs ):
+        # Session
+        session = trans.sa_session
+        # Process any actions
+        status = message = None
+        ids = self.get_ids( **kwargs )
+        operation = kwargs.get( 'operation', None )
+        if operation:
+            operation = operation.lower()
+            status, message = self.handle_operation( trans, operation, ids )
+        # Build initial query
+        query = self.build_initial_query( session )
+        query = self.apply_default_filter( trans, query )
+        # Maintain sort state in generated urls
+        extra_url_args = {}
+        # Process filtering arguments
+        filter_args = {}
+        if self.default_filter:
+            filter_args.update( self.default_filter )
+        for column in self.columns:
+            if column.key:
+                if "f-" + column.key in kwargs:
+                    column_filter = kwargs.get( "f-" + column.key )
+                    if column_filter == "True":
+                        filter_args[column.key] = True
+                    elif column_filter == "False":
+                        filter_args[column.key] = False
+                    elif column_filter == "All":
+                        del filter_args[column.key]
+                    # Carry filter along to newly generated urls
+                    extra_url_args[ "f-" + column.key ] = column_filter
+        if filter_args:
+            query = query.filter_by( **filter_args )
+        # Process sort arguments
+        sort_key = sort_order = None
+        if 'sort' in kwargs:
+            sort_key = kwargs['sort']
+        elif self.default_sort_key:
+            sort_key = self.default_sort_key
+        encoded_sort_key = sort_key
+        if sort_key:
+            if sort_key.startswith( "-" ):
+                sort_key = sort_key[1:]
+                sort_order = 'desc'
+                query = query.order_by( self.model_class.c.get( sort_key ).desc() )
+            else:
+                sort_order = 'asc'
+                query = query.order_by( self.model_class.c.get( sort_key ).asc() )
+        extra_url_args['sort'] = encoded_sort_key
+        # There might be a current row
+        current_item = self.get_current_item( trans )
+        # Render
+        def url( *args, **kwargs ):
+            new_kwargs = dict( extra_url_args )
+            if len(args) > 0:
+                new_kwargs.update( args[0] )
+            new_kwargs.update( kwargs )
+            return url_for( **new_kwargs )
+        return trans.fill_template( "grid.mako",
+                                    grid=self,
+                                    query=query,
+                                    sort_key=sort_key,
+                                    encoded_sort_key=encoded_sort_key,
+                                    sort_order=sort_order,
+                                    current_item=current_item,
+                                    ids = ids,
+                                    url = url,
+                                    message_type = status,
+                                    message = message )
+    def get_ids( self, **kwargs ):
+        id = []
+        if 'id' in kwargs:
+            id = kwargs['id']
+            # Coerce ids to list
+            if not isinstance( id, list ):
+                id = id.split( "," )
+            # Ensure ids are integers
+            try:
+                id = map( int, id )
+            except:
+                error( "Invalid id" )
+        return id
+        
+    # ---- Override these ----------------------------------------------------
+    def handle_operation( self, trans, operation, ids ):
+        pass
+    def get_current_item( self, trans ):
+        return None
+    def build_initial_query( self, session ):
+        return session.query( self.model_class )
+    def apply_default_filter( self, trans, query ):
+        return query
+    
+class GridColumn( object ):
+    def __init__( self, label, key=None, method=None, format=None, link=None, attach_popup=False, visible=True, ncells=1 ):
+        self.label = label
+        self.key = key
+        self.method = method
+        self.format = format
+        self.link = link
+        self.attach_popup = attach_popup
+        self.visible = visible
+        self.ncells = ncells
+        # Currently can only sort of columns that have a database
+        # representation, not purely derived.
+        if self.key:
+            self.sortable = True
+        else:
+            self.sortable = False
+    def get_value( self, trans, grid, item ):
+        if self.method:
+            value = getattr( grid, self.method )( trans, item )
+        elif self.key:
+            value = getattr( item, self.key )
+        else:
+            value = None
+        if self.format:
+            value = self.format( value )
+        return value
+    
+class GridOperation( object ):
+    def __init__( self, label, key=None, condition=None, allow_multiple=True ):
+        self.label = label
+        self.key = key
+        self.allow_multiple = allow_multiple
+        self.condition = condition
+    def allowed( self, item ):
+        if self.condition:
+            return self.condition( item )
+        else:
+            return True
+        
+class GridColumnFilter( object ):
+    def __init__( self, label, args=None ):
+        self.label = label
+        self.args = args
+    def get_url_args( self ):
+        rval = {}
+        for k, v in self.args.items():
+            rval[ "f-" + k ] = v
+        return rval
+        

lib/galaxy/webapps/reports/config.py

@@ -27,6 +27,7 @@ class Configuration( object ):
         self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root )
         self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" )
         self.use_remote_user = string_as_bool( kwargs.get( "use_remote_user", "False" ) )
+        self.require_login = string_as_bool( kwargs.get( "require_login", "False" ) )
         self.template_path = resolve_path( kwargs.get( "template_path", "templates" ), self.root )
         self.template_cache = resolve_path( kwargs.get( "template_cache_path", "database/reports/compiled_templates" ), self.root )
         self.sendmail_path = kwargs.get('sendmail_path',"/usr/sbin/sendmail")

lib/galaxy/webapps/reports/controllers/users.py

@@ -104,7 +104,7 @@ class Users( BaseController ):
         cutoff_time = datetime.utcnow() - timedelta( days=int( not_logged_in_for_days ) )
         now = strftime( "%Y-%m-%d %H:%M:%S" )
         users = []
-        for user in galaxy.model.User.query().order_by( galaxy.model.User.table.c.email ).all():
+        for user in galaxy.model.User.filter( galaxy.model.User.table.c.deleted==False ).order_by( galaxy.model.User.table.c.email ).all():
             if user.galaxy_sessions:
                 last_galaxy_session = user.galaxy_sessions[ 0 ]
                 if last_galaxy_session.update_time < cutoff_time:

lib/galaxy/workflow/modules.py

@@ -208,8 +208,8 @@ class ToolModule( object ):
                     values = input_values[ input.name ]
                     current = values["__current_case__"]
                     label_prefix = label_prefix
-                    name_prefix = name_prefix + "|" + input.name
-                    visitor( input.cases[current].inputs, values, name_prefix, label_prefix )
+                    new_name_prefix = name_prefix + input.name + "|"
+                    visitor( input.cases[current].inputs, values, new_name_prefix, label_prefix )
                 else:
                     if isinstance( input, DataToolParameter ):
                         data_inputs.append( dict( name=name_prefix+input.name, label=label_prefix+input.label, extensions=input.extensions ) )

reports_wsgi.ini.sample

@@ -46,6 +46,9 @@ use_lint = false
 # NEVER enable this on a public site (even test or QA)
 # use_interactive = true
 
+# Force everyone to log in (disable anonymous access)
+require_login = False
+
 # path to sendmail
 sendmail_path = /usr/sbin/sendmail
 

scripts/cleanup_datasets/cleanup_datasets.py

@@ -207,6 +207,12 @@ def purge_histories( h, d, m, cutoff_time, remove_from_disk ):
                                     metadata_file.purged = True
                                     metadata_file.flush()
                                     print "%s" % metadata_file.file_name()
+                            for lda in dataset.library_associations:
+                                for metadata_file in m.filter( m.table.c.lda_id==lda.id ).all():
+                                    metadata_file.deleted = True
+                                    metadata_file.purged = True
+                                    metadata_file.flush()
+                                    print "%s" % metadata_file.file_name()
                         dataset_count += 1
                         try:
                             disk_space += file_size
@@ -275,6 +281,12 @@ def purge_datasets( d, m, cutoff_time, remove_from_disk ):
                     metadata_file.purged = True
                     metadata_file.flush()
                     print "%s" % metadata_file.file_name()
+            for lda in dataset.library_associations:
+                for metadata_file in m.filter( m.table.c.lda_id==lda.id ).all():
+                    metadata_file.deleted = True
+                    metadata_file.purged = True
+                    metadata_file.flush()
+                    print "%s" % metadata_file.file_name()
             dataset_count += 1
         try:
             disk_space += file_size
@@ -310,6 +322,12 @@ def purge_dataset( dataset, d, m ):
                     if not shared_data.deleted:
                         purgable = False
                         break
+            if purgable:
+                # This check handles the library_dataset_dataset_association approach to sharing.
+                for shared_data in dataset.library_associations:
+                    if not shared_data.deleted:
+                        purgable = False
+                        break
             if purgable:
                 dataset.purged = True
                 dataset.file_size = 0
@@ -326,6 +344,12 @@ def purge_dataset( dataset, d, m ):
                         metadata_file.purged = True
                         metadata_file.flush()
                         print "%s" % metadata_file.file_name()
+                for lda in dataset.library_associations:
+                    for metadata_file in m.filter( m.table.c.lda_id==lda.id ).all():
+                        metadata_file.deleted = True
+                        metadata_file.purged = True
+                        metadata_file.flush()
+                        print "%s" % metadata_file.file_name()
                 try:
                     # Remove associated extra files from disk if they exist
                     os.unlink( dataset.extra_files_path )

scripts/set_metadata.py

@@ -0,0 +1,41 @@
+"""
+Execute an external process to set_meta() on a provided list of pickled datasets.
+
+This should not be called directly!  Use the set_metadata.sh script in Galaxy's
+top level directly.
+
+"""
+
+import os, sys, cPickle
+assert sys.version_info[:2] >= ( 2, 4 )
+
+new_path = [ os.path.join( os.getcwd(), "lib" ) ]
+new_path.extend( sys.path[1:] ) # remove scripts/ from the path
+sys.path = new_path
+
+from galaxy import eggs
+import pkg_resources
+pkg_resources.require("simplejson")
+import simplejson
+import galaxy.model.mapping #need to load this before we unpickle, in order to setup properties assigned by the mappers
+galaxy.model.Job() #this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here
+galaxy.datatypes.metadata.DATABASE_CONNECTION_AVAILABLE = False #Let metadata know that there is no database connection, and to just assume object ids are valid
+from galaxy.util import stringify_dictionary_keys
+
+def __main__():
+    file_path = sys.argv.pop( 1 )
+    tmp_dir = sys.argv.pop( 1 )
+    galaxy.model.Dataset.file_path = file_path
+    galaxy.datatypes.metadata.MetadataTempFile.tmp_dir = tmp_dir
+    for pickled_filenames in sys.argv[1:]:
+        pickled_filename_in, pickled_filename_kwds, pickled_filename_out, pickled_filename_results_code = pickled_filenames.split( ',' )
+        try:
+            data = cPickle.load( open( pickled_filename_in ) ) #load DatasetInstance
+            kwds = stringify_dictionary_keys( simplejson.load( open( pickled_filename_kwds ) ) )#load kwds; need to ensure our keywords are not unicode
+            data.datatype.set_meta( data, **kwds )
+            data.metadata.to_JSON_dict( pickled_filename_out ) # write out results of set_meta
+            simplejson.dump( ( True, 'Metadata has been set successfully' ), open( pickled_filename_results_code, 'wb+' ) ) #setting metadata has suceeded
+        except Exception, e:
+            simplejson.dump( ( False, str( e ) ), open( pickled_filename_results_code, 'wb+' ) ) #setting metadata has failed somehow
+
+__main__()

scripts/update_database/update_database_with_security_libraries.py

@@ -0,0 +1,229 @@
+#!/usr/bin/env python
+#Dan Blankenberg
+"""
+BACKUP YOUR DATABASE BEFORE RUNNING THIS SCRIPT!
+
+This script has been tested with Postgres, SQLite, and MySQL databases.
+
+This script updates a pre-security/library database with necessary schema changes and sets default roles/permissions
+for users and their histories and datasets.  It will reset permissions on histories and datasets if they are already set.
+Due to limitations of SQLite this script is unable to add foreign keys to existing SQLite tables (foreign keys are
+ignored by SQLite anyway).
+
+REMEMBER TO BACKUP YOUR DATABASE BEFORE RUNNING THIS SCRIPT!
+"""
+
+import sys, os, ConfigParser, tempfile
+import galaxy.app, galaxy.model
+import sqlalchemy
+from galaxy.model.orm import *
+
+assert sys.version_info[:2] >= ( 2, 4 )
+
+def main():
+    def print_warning( warning_text ):
+        print "\nWarning: %s\n" % ( warning_text )
+    
+    print
+    print "The purpose of this script is to update an existing Galaxy database that does not have Library or Security settings to a version that does."
+    print
+    print "This script will do the following:"
+    print "1) Create new tables that do not exist in a pre-security/library database"
+    print "2) Alter existing tables as necessary ( add new columns, indexes, etc )"
+    print "3) Create a private role for each user"
+    print "4) Set default permissions for each user ( set as 'manage permissions' and associated with the user's private role )"
+    print "5) Set default permissions for each user's history that has not been purged ( set as 'manage permissions' and associated with the user's private role )"
+    print "6) Set permissions on all appropriate datasets ( in each user's history ) that have not been purged ( set as 'manage permissions' and associated with the user's private role )"
+    print
+    print "*** It is critically important to backup your database before you continue. ***"
+    print
+    print "If you have backed up your database and would like to run this script, enter 'yes'."
+    print
+    should_continue = raw_input("enter 'yes' to continue>")
+    if should_continue.lower() != "yes":
+        print "Script aborted by user."
+        sys.exit(0)
+    
+    # Load Configuration from file
+    ini_file = sys.argv.pop(1)
+    conf_parser = ConfigParser.ConfigParser({'here':os.getcwd()})
+    conf_parser.read(ini_file)
+    configuration = {}
+    for key, value in conf_parser.items("app:main"): configuration[key] = value
+    
+    # If we don't load the tools, the app will startup much faster
+    empty_xml = tempfile.NamedTemporaryFile()
+    empty_xml.write( "<root/>" )
+    empty_xml.flush()
+    configuration['tool_config_file'] = empty_xml.name
+    # No need to load job runners
+    configuration['enable_job_running'] = False
+    
+    print
+    print "Loading app, with database_create_tables=False, to add Columns to existing tables"
+    print
+    # Set database_create_tables to False, then load the app
+    configuration['database_create_tables'] = False
+    app = galaxy.app.UniverseApplication( global_conf = ini_file, **configuration )
+    
+    # Try to guess the database type that we have, in order to execute the proper raw SQL commands
+    if app.config.database_connection:
+        dialect = galaxy.model.mapping.guess_dialect_for_url( app.config.database_connection )
+    else:
+        # default dialect is sqlite
+        dialect = "sqlite"
+    
+    # Now we alter existing tables, unfortunately SQLAlchemy does not support this.
+    # SQLite is very lacking in its implementation of the ALTER command, we'll do what we can...
+    # We cannot check with SA to see if new columns exist, so we will try and except (trying to 
+    # access the table with our current SA Metadata which references missing columns will throw 
+    # exceptions )
+
+    # galaxy_user table - must be altered differently depending on if we are using sqlite or not
+    if dialect == "sqlite":
+        try:
+            # 'true' and 'false' doesn't work properly in SQLite --> both are always True (is sqlite actually 
+            # storing a string here, which is always true when not empty?)
+            app.model.session.execute( "ALTER TABLE 'galaxy_user' ADD COLUMN 'deleted' BOOLEAN default 0" )
+        except sqlalchemy.exceptions.OperationalError, e:
+            print_warning( "adding column 'deleted' failed: %s" % ( e ) )
+        try:
+            app.model.session.execute( "ALTER TABLE 'galaxy_user' ADD COLUMN 'purged' BOOLEAN default 0" )
+        except sqlalchemy.exceptions.OperationalError, e:
+            print_warning( "adding column 'purged' failed: %s" % ( e ) )
+    else:
+        try:
+            app.model.session.execute( "ALTER TABLE galaxy_user ADD COLUMN deleted BOOLEAN default false" )
+        except ( sqlalchemy.exceptions.ProgrammingError, sqlalchemy.exceptions.OperationalError ), e:
+            # Postgres and MySQL raise different Exceptions for this same failure.
+            print_warning( "adding column 'deleted' failed: %s" % ( e ) )
+        try:
+            app.model.session.execute( "ALTER TABLE galaxy_user ADD COLUMN purged BOOLEAN default false" )
+        except ( sqlalchemy.exceptions.ProgrammingError, sqlalchemy.exceptions.OperationalError ), e:
+            print_warning( "adding column 'purged' failed: %s" % ( e ) )
+    
+    # history_dataset_association table - these alters are the same, regardless if we are using sqlite
+    try:
+        app.model.session.execute( "ALTER TABLE history_dataset_association ADD COLUMN copied_from_library_dataset_dataset_association_id INTEGER" )
+    except ( sqlalchemy.exceptions.ProgrammingError, sqlalchemy.exceptions.OperationalError ), e:
+        print_warning( "adding column 'copied_from_library_dataset_dataset_association_id' failed: %s" % ( e ) )
+    
+    # metadata_file table
+    try:
+        app.model.session.execute( "ALTER TABLE metadata_file ADD COLUMN lda_id INTEGER" )
+    except ( sqlalchemy.exceptions.ProgrammingError, sqlalchemy.exceptions.OperationalError ), e:
+        print_warning( "adding column 'lda_id' failed: %s" % ( e ) )
+    
+    
+    # Create indexes for new columns in the galaxy_user and metadata_file tables
+    try:
+        i = sqlalchemy.Index( 'ix_galaxy_user_deleted', app.model.User.table.c.deleted )
+        i.create()
+    except Exception, e:
+        print_warning( "Adding index failed: %s" % ( e ) )
+    try:
+        i = sqlalchemy.Index( 'ix_galaxy_user_purged', app.model.User.table.c.purged )
+        i.create()
+    except Exception, e:
+        print_warning( "Adding index failed: %s" % ( e ) )
+    try:
+        i = sqlalchemy.Index( 'ix_metadata_file_lda_id', app.model.MetadataFile.table.c.lda_id )
+        i.create()
+    except Exception, e:
+        print_warning( "Adding index failed: %s" % ( e ) )
+    try:
+        i = sqlalchemy.Index( 'ix_job_state', app.model.Job.table.c.state )
+        i.create()
+    except Exception, e:
+        print_warning( "Adding index failed: %s" % ( e ) )
+
+    
+    # Shutdown the app
+    app.shutdown()
+    del app
+    print
+    print "Columns added to tables, restarting app, with database_create_tables=True"
+    
+    # Restart the app, this time with create_tables == True
+    configuration['database_create_tables'] = True
+    app = galaxy.app.UniverseApplication( global_conf = ini_file, **configuration )
+    
+    # Add foreign key constraints as necessary for new columns added above
+    print "Adding foreign key constraints"
+    if dialect != "sqlite":
+        try:
+            app.model.session.execute( "ALTER TABLE history_dataset_association ADD FOREIGN KEY (copied_from_library_dataset_dataset_association_id) REFERENCES library_dataset_dataset_association(id)" )
+        except Exception, e:
+            print_warning( "Adding foreign key constraint to table has failed for an unknown reason: %s" % ( e ) )
+        try:
+            app.model.session.execute( "ALTER TABLE metadata_file ADD FOREIGN KEY (lda_id) REFERENCES library_dataset_dataset_association(id)" )
+        except Exception, e:
+            print_warning( "Adding foreign key constraint to table has failed for an unknown reason: %s" % ( e ) )
+
+    else:
+        # SQLite ignores ( but parses on initial table creation ) foreign key constraints anyway
+        # See: http://www.sqlite.org/omitted.html (there is some way to set up behavior using triggers)
+        print_warning( "Adding foreign key constraints to table is not supported in SQLite." )
+    
+    print "creating private roles and setting defaults for existing users and their histories and datasets"
+    security_agent = app.security_agent
+    # For each user:
+    # 1. make sure they have a private role
+    # 2. set DefaultUserPermissions
+    # 3. set DefaultHistoryPermissions on existing histories 
+    # 4. set DatasetPermissions on each history's activatable_datasets
+    default_user_action = security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS.action
+    
+    for user in app.model.User.query().all():
+        print
+        print "################"
+        print "Setting up user %s." % user.email
+        private_role = security_agent.get_private_user_role( user )
+        if not private_role:
+            private_role = security_agent.create_private_user_role( user )
+            print "Created private role for %s" % user.email
+        else:
+            print_warning( "%s already has a private role, re-setting defaults anyway" % user.email )
+        print "Setting DefaultUserPermissions for user %s" % user.email
+        # Delete all of the current default permissions for the user
+        for dup in user.default_permissions:
+            dup.delete()
+            dup.flush()
+        # Add the new default permissions for the user
+        dup = app.model.DefaultUserPermissions( user, default_user_action, private_role )
+        dup.flush()
+        print "Setting DefaultHistoryPermissions for %d un-purged histories associated with %s" % ( len( user.histories ), user.email )
+        for history in user.active_histories:
+            # Delete all of the current default permissions for the history
+            for dhp in history.default_permissions:
+                dhp.delete()
+                dhp.flush()
+            # Add the new default permissions for the history
+            dhp = app.model.DefaultHistoryPermissions( history, default_user_action, private_role )
+            dhp.flush()
+            activatable_datasets = history.activatable_datasets #store this list, so we don't generate it more than once
+            print "Setting DatasetPermissions for %d un-purged datasets in history %d" % ( len( activatable_datasets ), history.id )
+            # Set the permissions on the current history's datasets that are not purged
+            for hda in activatable_datasets:
+                dataset = hda.dataset
+                if dataset.library_associations:
+                    # Don't change permissions on a dataset associated with a library
+                    continue
+                if [ assoc for assoc in dataset.history_associations if assoc.history not in user.histories ]:
+                    # Don't change permissions on a dataset associated with a history not owned by the user
+                    continue
+                # Delete all of the current permissions on the dataset
+                for dp in dataset.actions:
+                    dp.delete()
+                    dp.flush()
+                # Add the new permissions on the dataset
+                dp = app.model.DatasetPermissions( default_user_action, dataset, private_role )
+                dp.flush()
+    app.shutdown()
+    print
+    print "Update finished, please review output for warnings and errors."
+    empty_xml.close() #close tempfile, it will automatically be deleted off system
+    
+
+if __name__ == "__main__":
+    main()

scripts/update_database/update_database_with_security_libraries.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# This script must be executed from the $UNIVERSE_HOME directory
+# e.g., sh ./scripts/update_database/update_database_with_security_libraries.sh
+
+. ./scripts/get_python.sh
+. ./setup_paths.sh
+
+$GALAXY_PYTHON ./scripts/update_database/update_database_with_security_libraries.py ./universe_wsgi.ini $@

set_metadata.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+cd `dirname $0`
+python -ES ./scripts/set_metadata.py $@

setup.sh

@@ -27,6 +27,7 @@ database/compiled_templates
 database/job_working_directory
 database/import
 database/pbs
+static/genetrack/plots
 "
 
 for sample in $SAMPLES; do

static/genetrack/genetrack.css

@@ -0,0 +1,78 @@
+
+body {
+	font-family: "Trebuchet MS", Arial, tahoma, sans-serif;
+	font-size: 14px;
+	line-height: 1.6em;
+	margin: 0; 
+	padding: 0;
+	border-top: 9px solid #CCD9FF;	
+}
+
+/* Error message style */
+.error{
+	background: #FFFF66;
+}
+
+/* Error message style */
+.message{
+	background: #33FF66;
+}
+
+/* Odd data row in the table */
+.selected {
+	background-color: #FFFFCC;
+}
+
+.nav_button{
+   background-color:#EEEEEE;
+   border:1px solid;
+   color: #000000;
+}
+
+.nav_button:hover{
+   background-color:#000000;
+   border:1px solid;
+   color: #FFFFFF;
+}
+
+.grey {
+	background-color: #EFEFEF;
+}
+
+.odd {
+	background-color: #ECECEC;
+}
+
+.even {
+	background-color: #FFFFFF;
+}
+
+/* Text table style */
+.data_table {
+	border: 1px solid #CCCCCC;
+	background-color: white;
+}
+
+/* Footer is added to every page */
+#footer {
+	background: #EFEFEF; 
+	text-align:center; 
+	padding:.2em; 
+	border-top: 1px solid #CCD9FF; 
+	border-bottom: 1px solid #CCD9FF; 
+	clear: both;
+}
+
+#footer p {
+	font-size:.94em; line-height:2em; color:#cccccc; margin: 0;
+	}
+
+#tag {
+	font-size:.80em; margin: 4px; padding: 2px;
+	}
+
+
+#footer img {
+	vertical-align: middle; margin-left: 3px; padding-bottom: 2px;
+}
+

static/genetrack/genetrack.js

@@ -0,0 +1,79 @@
+var cookie_name = "genetrack_ui"
+var now = new Date();
+now.setTime(now.getTime() + 365 * 24 * 60 * 60 * 1000);
+
+// this toggles between none and block
+function toggle(name){
+	var elem = get(name)
+	if (elem) {
+		if (elem.style.display=="none"){
+			elem.style.display="block"
+			setCookie(cookie_name, name, now)
+		} else {
+			elem.style.display="none"
+			setCookie(cookie_name, '', now)
+		}
+		
+	}
+}
+
+function main(){
+	//executed upon main body load
+	var value = getCookie(cookie_name);
+	toggle(	value )		
+}
+
+// this toggles between visible and hidden
+function show(name){
+	var elem = get(name)
+    if (elem.style.visibility=="hidden"){
+        elem.style.visibility="visible";
+    } else {
+        elem.style.visibility="hidden";
+    }
+}
+
+// utility function to get the length of on object 
+function len(obj){
+	return obj.length;
+}
+
+// utility function to get an element by id
+function get(name){
+	return document.getElementById(name);
+}
+
+// pops up a window
+function pop_up(url) {
+	day = new Date();
+	id = day.getTime();
+	eval("page" + id + " = window.open(url, '" + id + "', 'toolbar=0,scrollbars=1,location=0,statusbar=1,menubar=0,resizable=1,width=500,height=300');");
+}
+
+// 
+// cookie management off the web
+// http://www.webreference.com/js/column8/property.html
+//
+function setCookie(name, value, expires, path, domain, secure) {
+  var curCookie = name + "=" + escape(value) +
+      ((expires) ? "; expires=" + expires.toGMTString() : "") +
+      ((path) ? "; path=" + path : "") +
+      ((domain) ? "; domain=" + domain : "") +
+      ((secure) ? "; secure" : "");
+  document.cookie = curCookie;
+}
+
+function getCookie(name) {
+  var dc = document.cookie;
+  var prefix = name + "=";
+  var begin = dc.indexOf("; " + prefix);
+  if (begin == -1) {
+    begin = dc.indexOf(prefix);
+    if (begin != 0) return null;
+  } else
+    begin += 2;
+  var end = document.cookie.indexOf(";", begin);
+  if (end == -1)
+    end = dc.length;
+  return unescape(dc.substring(begin + prefix.length, end));
+}

static/iui/LICENSE.txt

@@ -0,0 +1,21 @@
+Copyright (c) 2007, iUI Project Members
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+    * Neither the name of the iUI Project nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

static/iui/iui.css

@@ -0,0 +1,394 @@
+/* iui.css (c) 2007-8 by iUI Project Members, see LICENSE.txt for license */
+body {
+    margin: 0;
+    font-family: Helvetica;
+    background: #FFFFFF;
+    color: #000000;
+    overflow-x: hidden;
+    -webkit-user-select: none;
+    -webkit-text-size-adjust: none;
+}
+
+body > *:not(.toolbar) {
+    display: none;
+    position: absolute;
+    margin: 0;
+    padding: 0;
+    left: 0;
+    top: 45px;
+    width: 100%;
+    min-height: 372px;
+}
+
+body[orient="landscape"] > *:not(.toolbar) {
+    min-height: 268px;
+}
+
+body > *[selected="true"] {
+    display: block;
+}
+
+a[selected], a:active {
+    background-color: #194fdb !important;
+    background-image: url(listArrowSel.png), url(selection.png) !important;
+    background-repeat: no-repeat, repeat-x;
+    background-position: right center, left top;
+    color: #FFFFFF !important;
+}
+
+a[selected="progress"] {
+    background-image: url(loading.gif), url(selection.png) !important;
+}
+
+/************************************************************************************************/
+
+body > .toolbar {
+    box-sizing: border-box;
+    -moz-box-sizing: border-box;
+    -webkit-box-sizing: border-box;
+    border-bottom: 1px solid #2d3642;
+    border-top: 1px solid #6d84a2;
+    padding: 10px;
+    height: 45px;
+    background: url(toolbar.png) #6d84a2 repeat-x;
+}
+
+.toolbar > h1 {
+    position: absolute;
+    overflow: hidden;
+    left: 50%;
+    margin: 1px 0 0 -75px;
+    height: 45px;
+    font-size: 20px;
+    width: 150px;
+    font-weight: bold;
+    text-shadow: rgba(0, 0, 0, 0.4) 0px -1px 0;
+    text-align: center;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+    color: #FFFFFF;
+}
+
+body[orient="landscape"] > .toolbar > h1 {
+    margin-left: -125px;
+    width: 250px;
+}
+
+.button {
+    position: absolute;
+    overflow: hidden;
+    top: 8px;
+    right: 6px;
+    margin: 0;
+    border-width: 0 5px;
+    padding: 0 3px;
+    width: auto;
+    height: 30px;
+    line-height: 30px;
+    font-family: inherit;
+    font-size: 12px;
+    font-weight: bold;
+    color: #FFFFFF;
+    text-shadow: rgba(0, 0, 0, 0.6) 0px -1px 0;
+    text-overflow: ellipsis;
+    text-decoration: none;
+    white-space: nowrap;
+    background: none;
+    -webkit-border-image: url(toolButton.png) 0 5 0 5;
+}
+
+.blueButton {
+    -webkit-border-image: url(blueButton.png) 0 5 0 5;
+    border-width: 0 5px;
+}
+
+.leftButton {
+    left: 6px;
+    right: auto;
+}
+
+#backButton {
+    display: none;
+    left: 6px;
+    right: auto;
+    padding: 0;
+    max-width: 55px;
+    border-width: 0 8px 0 14px;
+    -webkit-border-image: url(backButton.png) 0 8 0 14;
+}
+
+.whiteButton,
+.grayButton {
+    display: block;
+    border-width: 0 12px;
+    padding: 10px;
+    text-align: center;
+    font-size: 20px;
+    font-weight: bold;
+    text-decoration: inherit;
+    color: inherit;
+}
+
+.whiteButton {
+    -webkit-border-image: url(whiteButton.png) 0 12 0 12;
+    text-shadow: rgba(255, 255, 255, 0.7) 0 1px 0;
+}
+
+.grayButton {
+    -webkit-border-image: url(grayButton.png) 0 12 0 12;
+    color: #FFFFFF;
+}
+
+/************************************************************************************************/
+
+body > ul > li {
+    position: relative;
+    margin: 0;
+    border-bottom: 1px solid #E0E0E0;
+    padding: 8px 0 8px 10px;
+    font-size: 20px;
+    font-weight: bold;
+    list-style: none;
+}
+
+body > ul > li.group {
+    position: relative;
+    top: -1px;
+    margin-bottom: -2px;
+    border-top: 1px solid #7d7d7d;
+    border-bottom: 1px solid #999999;
+    padding: 1px 10px;
+    background: url(listGroup.png) repeat-x;
+    font-size: 17px;
+    font-weight: bold;
+    text-shadow: rgba(0, 0, 0, 0.4) 0 1px 0;
+    color: #FFFFFF;
+}
+
+body > ul > li.group:first-child {
+    top: 0;
+    border-top: none;
+}
+
+body > ul > li > a {
+    display: block;
+    margin: -8px 0 -8px -10px;
+    padding: 8px 32px 8px 10px;
+    text-decoration: none;
+    color: inherit;
+    background: url(listArrow.png) no-repeat right center;
+}
+
+a[target="_replace"] {
+    box-sizing: border-box;
+    -webkit-box-sizing: border-box;
+    padding-top: 25px;
+    padding-bottom: 25px;
+    font-size: 18px;
+    color: cornflowerblue;
+    background-color: #FFFFFF;
+    background-image: none;
+}
+
+/************************************************************************************************/
+    
+body > .dialog {
+    top: 0;
+    width: 100%;
+    min-height: 417px;
+    z-index: 2;
+    background: rgba(0, 0, 0, 0.8);
+    padding: 0;
+    text-align: right;
+}
+
+.dialog > fieldset {
+    box-sizing: border-box;
+    -webkit-box-sizing: border-box;
+    width: 100%;
+    margin: 0;
+    border: none;
+    border-top: 1px solid #6d84a2;
+    padding: 10px 6px;
+    background: url(toolbar.png) #7388a5 repeat-x;
+}
+
+.dialog > fieldset > h1 {
+    margin: 0 10px 0 10px;
+    padding: 0;
+    font-size: 20px;
+    font-weight: bold;
+    color: #FFFFFF;
+    text-shadow: rgba(0, 0, 0, 0.4) 0px -1px 0;
+    text-align: center;
+}
+
+.dialog > fieldset > label {
+    position: absolute;
+    margin: 16px 0 0 6px;
+    font-size: 14px;
+    color: #999999;
+}
+
+input:not(input[type|=radio]):not(input[type|=checkbox]) {
+    box-sizing: border-box;
+    -webkit-box-sizing: border-box;
+    width: 100%;
+    margin: 8px 0 0 0;
+    padding: 6px 6px 6px 44px;
+    font-size: 16px;
+    font-weight: normal;
+}
+
+/************************************************************************************************/
+
+body > .panel {
+    box-sizing: border-box;
+    -moz-box-sizing: border-box;
+    -webkit-box-sizing: border-box;
+    padding: 10px;
+    background: #c8c8c8 url(pinstripes.png);
+}
+
+.panel > fieldset {
+    position: relative;
+    margin: 0 0 20px 0;
+    padding: 0;
+    background: #FFFFFF;
+    -webkit-border-radius: 10px;
+    -moz-border-radius: 10px;
+    border: 1px solid #999999;
+    text-align: right;
+    font-size: 16px;
+}
+
+.row  {
+    position: relative;
+    min-height: 42px;
+    border-bottom: 1px solid #999999;
+    -webkit-border-radius: 0;
+    text-align: right;
+}
+
+fieldset > .row:last-child {
+    border-bottom: none !important;
+}
+
+.row > input:not(input[type|=radio]):not(input[type|=checkbox]) {
+    box-sizing: border-box;
+    -moz-box-sizing: border-box;
+    -webkit-box-sizing: border-box;
+    margin: 0;
+    border: none;
+    padding: 12px 10px 0 110px;
+    height: 42px;
+    background: none;
+}
+.row > input[type|=radio], .row > input[type|=checkbox] {
+  margin: 7px 7px 0 0;
+  height: 25px;
+  width: 25px;
+}
+
+.row > label {
+    position: absolute;
+    margin: 0 0 0 14px;
+    line-height: 42px;
+    font-weight: bold;
+}
+
+.row > .error {
+    height: 25px;
+    text-align: left;
+    font-size: 14px;
+    padding: 0 0 0 110px;
+    color: red;
+}
+
+.row > span {
+    position: absolute;
+    padding: 12px 10px 0 110px;
+    margin: 0;
+}
+
+.row > .toggle {
+    position: absolute;
+    top: 6px;
+    right: 6px;
+    width: 100px;
+    height: 28px;
+}
+
+.toggle {
+    border: 1px solid #888888;
+    -webkit-border-radius: 6px;
+    background: #FFFFFF url(toggle.png) repeat-x;
+    font-size: 19px;
+    font-weight: bold;
+    line-height: 30px;
+}
+
+.toggle[toggled="true"] {
+    border: 1px solid #143fae;
+    background: #194fdb url(toggleOn.png) repeat-x;
+}
+
+.toggleOn {
+    display: none;
+    position: absolute;
+    width: 60px;
+    text-align: center;
+    left: 0;
+    top: 0;
+    color: #FFFFFF;
+    text-shadow: rgba(0, 0, 0, 0.4) 0px -1px 0;
+}
+
+.toggleOff {
+    position: absolute;
+    width: 60px;
+    text-align: center;
+    right: 0;
+    top: 0;
+    color: #666666;
+}
+
+.toggle[toggled="true"] > .toggleOn {
+    display: block;
+}
+
+.toggle[toggled="true"] > .toggleOff {
+    display: none;
+}
+
+.thumb {
+    position: absolute;
+    top: -1px;
+    left: -1px;
+    width: 40px;
+    height: 28px;    
+    border: 1px solid #888888;
+    -webkit-border-radius: 6px;
+    background: #ffffff url(thumb.png) repeat-x;
+}
+
+.toggle[toggled="true"] > .thumb {
+    left: auto;
+    right: -1px;
+}
+
+.panel > h2 {
+    margin: 0 0 8px 14px;
+    font-size: inherit;
+    font-weight: bold;
+    color: #4d4d70;
+    text-shadow: rgba(255, 255, 255, 0.75) 2px 2px 0;
+}
+
+/************************************************************************************************/
+
+#preloader {
+    display: none;
+    background-image: url(loading.gif), url(selection.png),
+        url(blueButton.png), url(listArrowSel.png), url(listGroup.png);
+}

static/iui/iui.js

@@ -0,0 +1,440 @@
+/*
+   Copyright (c) 2007-8, iUI Project Members
+   See LICENSE.txt for licensing terms
+ */
+
+
+(function() {
+
+var slideSpeed = 20;
+var slideInterval = 0;
+
+var currentPage = null;
+var currentDialog = null;
+var currentWidth = 0;
+var currentHash = location.hash;
+var hashPrefix = "#_";
+var pageHistory = [];
+var newPageCount = 0;
+var checkTimer;
+var hasOrientationEvent = false;
+
+// *************************************************************************************************
+
+window.iui =
+{
+    showPage: function(page, backwards)
+    {
+        if (page)
+        {
+            if (currentDialog)
+            {
+                currentDialog.removeAttribute("selected");
+                currentDialog = null;
+            }
+
+            if (hasClass(page, "dialog"))
+                showDialog(page);
+            else
+            {
+                var fromPage = currentPage;
+                currentPage = page;
+
+                if (fromPage)
+                    setTimeout(slidePages, 0, fromPage, page, backwards);
+                else
+                    updatePage(page, fromPage);
+            }
+        }
+    },
+
+    showPageById: function(pageId)
+    {
+        var page = $(pageId);
+        if (page)
+        {
+            var index = pageHistory.indexOf(pageId);
+            var backwards = index != -1;
+            if (backwards)
+                pageHistory.splice(index, pageHistory.length);
+
+            iui.showPage(page, backwards);
+        }
+    },
+
+    showPageByHref: function(href, args, method, replace, cb)
+    {
+        var req = new XMLHttpRequest();
+        req.onerror = function()
+        {
+            if (cb)
+                cb(false);
+        };
+        
+        req.onreadystatechange = function()
+        {
+            if (req.readyState == 4)
+            {
+                if (replace)
+                    replaceElementWithSource(replace, req.responseText);
+                else
+                {
+                    var frag = document.createElement("div");
+                    frag.innerHTML = req.responseText;
+                    iui.insertPages(frag.childNodes);
+                }
+                if (cb)
+                    setTimeout(cb, 1000, true);
+            }
+        };
+
+        if (args)
+        {
+            req.open(method || "GET", href, true);
+            req.setRequestHeader("Content-Type", "application/x-www-form-urlencoded");
+            req.setRequestHeader("Content-Length", args.length);
+            req.send(args.join("&"));
+        }
+        else
+        {
+            req.open(method || "GET", href, true);
+            req.send(null);
+        }
+    },
+    
+    insertPages: function(nodes)
+    {
+        var targetPage;
+        for (var i = 0; i < nodes.length; ++i)
+        {
+            var child = nodes[i];
+            if (child.nodeType == 1)
+            {
+                if (!child.id)
+                    child.id = "__" + (++newPageCount) + "__";
+
+                var clone = $(child.id);
+                if (clone)
+                    clone.parentNode.replaceChild(child, clone);
+                else
+                    document.body.appendChild(child);
+
+                if (child.getAttribute("selected") == "true" || !targetPage)
+                    targetPage = child;
+                
+                --i;
+            }
+        }
+
+        if (targetPage)
+            iui.showPage(targetPage);    
+    },
+
+    getSelectedPage: function()
+    {
+        for (var child = document.body.firstChild; child; child = child.nextSibling)
+        {
+            if (child.nodeType == 1 && child.getAttribute("selected") == "true")
+                return child;
+        }    
+    },
+    isNativeUrl: function(href)
+    {
+        for(var i = 0; i < iui.nativeUrlPatterns.length; i++)
+        {
+            if(href.match(iui.nativeUrlPatterns[i])) return true;
+        }
+        return false;
+    },
+    nativeUrlPatterns: [
+        new RegExp("^http:\/\/maps.google.com\/maps\?"),
+        new RegExp("^mailto:"),
+        new RegExp("^tel:"),
+        new RegExp("^http:\/\/www.youtube.com\/watch\\?v="),
+        new RegExp("^http:\/\/www.youtube.com\/v\/")
+    ]
+};
+
+// *************************************************************************************************
+
+addEventListener("load", function(event)
+{
+    var page = iui.getSelectedPage();
+    if (page)
+        iui.showPage(page);
+
+    setTimeout(preloadImages, 0);
+    setTimeout(checkOrientAndLocation, 0);
+    checkTimer = setInterval(checkOrientAndLocation, 300);
+}, false);
+
+addEventListener("unload", function(event)
+{
+	return;
+}, false);
+    
+addEventListener("click", function(event)
+{
+    var link = findParent(event.target, "a");
+    if (link)
+    {
+        function unselect() { link.removeAttribute("selected"); }
+        
+        if (link.href && link.hash && link.hash != "#")
+        {
+            link.setAttribute("selected", "true");
+            iui.showPage($(link.hash.substr(1)));
+            setTimeout(unselect, 500);
+        }
+        else if (link == $("backButton"))
+            history.back();
+        else if (link.getAttribute("type") == "submit")
+            submitForm(findParent(link, "form"));
+        else if (link.getAttribute("type") == "cancel")
+            cancelDialog(findParent(link, "form"));
+        else if (link.target == "_replace")
+        {
+            link.setAttribute("selected", "progress");
+            iui.showPageByHref(link.href, null, null, link, unselect);
+        }
+        else if (iui.isNativeUrl(link.href))
+        {
+            return;
+        }
+        else if (!link.target)
+        {
+            link.setAttribute("selected", "progress");
+            iui.showPageByHref(link.href, null, null, null, unselect);
+        }
+        else
+            return;
+        
+        event.preventDefault();        
+    }
+}, true);
+
+addEventListener("click", function(event)
+{
+    var div = findParent(event.target, "div");
+    if (div && hasClass(div, "toggle"))
+    {
+        div.setAttribute("toggled", div.getAttribute("toggled") != "true");
+        event.preventDefault();        
+    }
+}, true);
+
+function orientChangeHandler()
+{
+  var orientation=window.orientation;
+  switch(orientation)
+  {
+    case 0:
+        setOrientation("portrait");
+        break;  
+        
+    case 90:
+    case -90: 
+        setOrientation("landscape");
+        break;
+  }
+}
+
+if (typeof window.onorientationchange == "object")
+{
+    window.onorientationchange=orientChangeHandler;
+    hasOrientationEvent = true;
+    setTimeout(orientChangeHandler, 0);
+}
+
+function checkOrientAndLocation()
+{
+    if (!hasOrientationEvent)
+    {
+      if (window.innerWidth != currentWidth)
+      {   
+          currentWidth = window.innerWidth;
+          var orient = currentWidth == 320 ? "portrait" : "landscape";
+          setOrientation(orient);
+      }
+    }
+
+    if (location.hash != currentHash)
+    {
+        var pageId = location.hash.substr(hashPrefix.length);
+        iui.showPageById(pageId);
+    }
+}
+
+function setOrientation(orient)
+{
+    document.body.setAttribute("orient", orient);
+    setTimeout(scrollTo, 100, 0, 1);
+}
+
+function showDialog(page)
+{
+    currentDialog = page;
+    page.setAttribute("selected", "true");
+    
+    if (hasClass(page, "dialog") && !page.target)
+        showForm(page);
+}
+
+function showForm(form)
+{
+    form.onsubmit = function(event)
+    {
+        event.preventDefault();
+        submitForm(form);
+    };
+    
+    form.onclick = function(event)
+    {
+        if (event.target == form && hasClass(form, "dialog"))
+            cancelDialog(form);
+    };
+}
+
+function cancelDialog(form)
+{
+    form.removeAttribute("selected");
+}
+
+function updatePage(page, fromPage)
+{
+    if (!page.id)
+        page.id = "__" + (++newPageCount) + "__";
+
+    location.href = currentHash = hashPrefix + page.id;
+    pageHistory.push(page.id);
+
+    var pageTitle = $("pageTitle");
+    if (page.title)
+        pageTitle.innerHTML = page.title;
+
+    if (page.localName.toLowerCase() == "form" && !page.target)
+        showForm(page);
+        
+    var backButton = $("backButton");
+    if (backButton)
+    {
+        var prevPage = $(pageHistory[pageHistory.length-2]);
+        if (prevPage && !page.getAttribute("hideBackButton"))
+        {
+            backButton.style.display = "inline";
+            backButton.innerHTML = prevPage.title ? prevPage.title : "Back";
+        }
+        else
+            backButton.style.display = "none";
+    }    
+}
+
+function slidePages(fromPage, toPage, backwards)
+{        
+    var axis = (backwards ? fromPage : toPage).getAttribute("axis");
+    if (axis == "y")
+        (backwards ? fromPage : toPage).style.top = "100%";
+    else
+        toPage.style.left = "100%";
+
+    toPage.setAttribute("selected", "true");
+    scrollTo(0, 1);
+    clearInterval(checkTimer);
+    
+    var percent = 100;
+    slide();
+    var timer = setInterval(slide, slideInterval);
+
+    function slide()
+    {
+        percent -= slideSpeed;
+        if (percent <= 0)
+        {
+            percent = 0;
+            if (!hasClass(toPage, "dialog"))
+                fromPage.removeAttribute("selected");
+            clearInterval(timer);
+            checkTimer = setInterval(checkOrientAndLocation, 300);
+            setTimeout(updatePage, 0, toPage, fromPage);
+        }
+    
+        if (axis == "y")
+        {
+            backwards
+                ? fromPage.style.top = (100-percent) + "%"
+                : toPage.style.top = percent + "%";
+        }
+        else
+        {
+            fromPage.style.left = (backwards ? (100-percent) : (percent-100)) + "%"; 
+            toPage.style.left = (backwards ? -percent : percent) + "%"; 
+        }
+    }
+}
+
+function preloadImages()
+{
+    var preloader = document.createElement("div");
+    preloader.id = "preloader";
+    document.body.appendChild(preloader);
+}
+
+function submitForm(form)
+{
+    iui.showPageByHref(form.action || "POST", encodeForm(form), form.method);
+}
+
+function encodeForm(form)
+{
+    function encode(inputs)
+    {
+        for (var i = 0; i < inputs.length; ++i)
+        {
+            if (inputs[i].name)
+                args.push(inputs[i].name + "=" + escape(inputs[i].value));
+        }
+    }
+
+    var args = [];
+    encode(form.getElementsByTagName("input"));
+    encode(form.getElementsByTagName("textarea"));
+    encode(form.getElementsByTagName("select"));
+    return args;    
+}
+
+function findParent(node, localName)
+{
+    while (node && (node.nodeType != 1 || node.localName.toLowerCase() != localName))
+        node = node.parentNode;
+    return node;
+}
+
+function hasClass(self, name)
+{
+    var re = new RegExp("(^|\\s)"+name+"($|\\s)");
+    return re.exec(self.getAttribute("class")) != null;
+}
+
+function replaceElementWithSource(replace, source)
+{
+    var page = replace.parentNode;
+    var parent = replace;
+    while (page.parentNode != document.body)
+    {
+        page = page.parentNode;
+        parent = parent.parentNode;
+    }
+
+    var frag = document.createElement(parent.localName);
+    frag.innerHTML = source;
+
+    page.removeChild(parent);
+
+    while (frag.firstChild)
+        page.appendChild(frag.firstChild);
+}
+
+function $(id) { return document.getElementById(id); }
+function ddd() { console.log.apply(console, arguments); }
+
+})();