Merge from stable

.hgignore

@@ -60,6 +60,7 @@ shed_tool_data_table_conf.xml
 job_conf.xml
 data_manager_conf.xml
 shed_data_manager_conf.xml
+object_store_conf.xml
 config/*
 static/welcome.html.*
 static/welcome.html

.hgtags

@@ -4,3 +4,5 @@ a4113cc1cb5eaa68091c9a73375f00555b66dd11 release_2013.01.13
 2cc8d10988e03257dc7b97f8bb332c7df745d1dd security_2013.04.08
 524f246ca85395082719ae7a6ff72260d7ad5612 release_2013.06.03
 1ae95b3aa98d1ccf15b243ac3ce6a895eb7efc53 release_2013.08.12
+26f58e05aa1068761660681583821e21e6cbf7ab release_2013.11.04
+5e605ed6069fe4c5ca9875e95e91b2713499e8ca release_2014.02.10

buildbot_setup.sh

@@ -4,28 +4,6 @@ cd `dirname $0`
 
 : ${HOSTTYPE:=`uname -m`}
 
-# link to HYPHY is arch-dependent
-case "$OSTYPE" in
-    linux-gnu)
-        kernel=`uname -r | cut -f1,2 -d.`
-        HYPHY="/galaxy/software/linux$kernel-$HOSTTYPE/hyphy"
-        ;;
-    darwin*)
-        this_minor=`uname -r | awk -F. '{print ($1-4)}'`
-        machine=`machine`
-        for minor in `jot - 3 $this_minor 1`; do
-            HYPHY="/galaxy/software/macosx10.$minor-$machine/hyphy"
-            [ -d "$HYPHY" ] && break
-        done
-        [ ! -d "$HYPHY" ] && unset HYPHY
-        ;;
-    solaris2.10)
-        # For the psu-production builder which is Solaris, but jobs run on a
-        # Linux cluster
-        HYPHY="/galaxy/software/linux2.6-x86_64/hyphy"
-        ;;
-esac
-
 LINKS="
 /galaxy/data/location/add_scores.loc
 /galaxy/data/location/all_fasta.loc
@@ -121,12 +99,6 @@ case $type in
 		    ln -sf $link tool-data
 		done
 		
-		if [ -d "$HYPHY" ]; then
-		    echo "Linking $HYPHY"
-		    rm -f tool-data/HYPHY
-		    ln -sf $HYPHY tool-data/HYPHY
-		fi
-		
 		if [ -d "$JARS" ]; then
 		    echo "Linking $JARS"
 		    rm -f tool-data/shared/jars

config/plugins/visualizations/additional_template_paths.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- these relative paths can contain common templates importable by all visualization plugins -->
+<paths>
+    <path>common/templates</path>
+</paths>

config/plugins/visualizations/charts/config/charts.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE visualization SYSTEM "../../visualization.dtd">
+<visualization name="Charts">
+    <data_sources>
+        <data_source>
+            <model_class>HistoryDatasetAssociation</model_class>
+            <test type="isinstance" test_attr="datatype" result_type="datatype">tabular.Tabular</test>
+            <to_param param_attr="id">dataset_id</to_param>
+        </data_source>
+    </data_sources>
+    <params>
+        <param type="dataset" var_name_in_template="hda" required="true">dataset_id</param>
+    </params>
+    <template>charts.mako</template>
+</visualization>

config/plugins/visualizations/charts/sample.txt

@@ -0,0 +1,9 @@
+43	55	63	99	28	85	20	31	47	14
+1	63	94	83	94	74	7	73	51	7
+8	34	2	94	87	21	72	51	3	84
+5	50	40	30	59	43	19	77	30	80
+22	55	69	43	77	72	12	54	53	56
+36	34	81	28	59	64	54	29	57	69
+57	90	93	32	45	98	40	16	52	83
+6	17	39	86	10	11	38	69	19	50
+67	51	47	22	58	27	65	49	92	96

config/plugins/visualizations/charts/static/app.js

@@ -0,0 +1,92 @@
+// dependencies
+define(['library/portlet', 'library/ui', 'library/utils',
+        'views/charts', 'views/viewport', 'views/chart', 'views/group',
+        'models/config', 'models/datasets', 'models/chart', 'models/charts', 'models/group', 'models/types'],
+        function(   Portlet, Ui, Utils,
+                    ChartsView, ViewportView, ChartView, GroupView,
+                    Config, Datasets, Chart, Charts, Group, Types
+                ) {
+
+// widget
+return Backbone.View.extend(
+{
+    // initialize
+    initialize: function(options)
+    {
+        // link options
+        this.options = options;
+    
+        // link galaxy
+        this.modal = parent.Galaxy.modal;
+        
+        // create configuration model
+        this.config = new Config();
+        
+        // create chart models
+        this.types = new Types();
+        this.chart = new Chart();
+        this.charts = new Charts();
+        this.group = new Group();
+        
+        // create dataset handler
+        this.datasets = new Datasets(this);
+        
+        // create views
+        this.charts_view = new ChartsView(this);
+        this.group_view = new GroupView(this);
+        this.chart_view = new ChartView(this);
+        this.viewport_view = new ViewportView(this);
+        
+        // append view port to charts viewer
+        this.charts_view.append(this.viewport_view.$el);
+            
+        // create portlet
+        if (!this.options.config.widget) {
+            this.portlet = new Portlet({icon : 'fa-bar-chart-o', label : 'Charts'});
+        } else {
+            this.portlet = $('<div></div>');
+        }
+        
+        // append views
+        this.portlet.append(this.charts_view.$el);
+        this.portlet.append(this.group_view.$el);
+        this.portlet.append(this.chart_view.$el);
+        
+        // set element
+        if (!this.options.config.widget) {
+            this.setElement(this.portlet.$el);
+        } else {
+            this.setElement(this.portlet);
+        }
+        
+        // hide views
+        this.group_view.$el.hide();
+        this.charts_view.$el.hide();
+        
+        // events
+        var self = this;
+        this.config.on('change:current_view', function() {
+            self._showCurrent();
+        });
+    },
+    
+    // current view
+    _showCurrent: function() {
+        
+    },
+    
+    // execute command
+    execute: function(options) {
+    },
+    
+    // unload
+    onunload: function() {
+    },
+    
+    // log
+    log: function(location, message) {
+        console.log(location + ' ' + message);
+    }
+});
+
+});

config/plugins/visualizations/charts/static/charts/bardiagram.js

@@ -0,0 +1,33 @@
+// dependencies
+define([], function() {
+
+// widget
+return Backbone.View.extend(
+{
+    // initialize
+    initialize: function(app, options) {
+        this.app        = app;
+        this.options    = options;
+    },
+            
+    // render
+    refresh : function(data)
+    {
+        // add graph to screen
+        var self = this;
+        nv.addGraph(function() {
+            self.d3_chart = nv.models.multiBarChart();
+                
+            self.d3_chart.xAxis.tickFormat(d3.format('.2f'))
+            self.d3_chart.yAxis.tickFormat(d3.format('.1f'))
+            
+            d3.select(self.options.svg_id)
+                .datum(data)
+                .call(self.d3_chart);
+ 
+            nv.utils.windowResize(self.d3_chart.update);
+        });
+    }
+});
+
+});

config/plugins/visualizations/charts/static/charts/horizontal.js

@@ -0,0 +1,33 @@
+// dependencies
+define(['library/utils'], function(Utils) {
+
+// widget
+return Backbone.View.extend(
+{
+    // initialize
+    initialize: function(app, options) {
+        this.app        = app;
+        this.options    = options;
+        this.chart      = options.chart;
+    },
+            
+    // render
+    refresh : function(data)
+    {
+        // add graph to screen
+        var self = this;
+        nv.addGraph(function() {
+            self.d3_chart = nv.models.multiBarHorizontalChart();
+            
+            self.d3_chart.xAxis.tickFormat(function() { return ''; });
+            
+            d3.select(self.options.svg_id)
+                .datum(data)
+                .call(self.d3_chart);
+ 
+            nv.utils.windowResize(self.d3_chart.update);
+        });
+    }
+});
+
+});

config/plugins/visualizations/charts/static/charts/line.js

@@ -0,0 +1,39 @@
+// dependencies
+define(['library/utils'], function(Utils) {
+
+// widget
+return Backbone.View.extend(
+{
+    // initialize
+    initialize: function(app, options) {
+        this.app        = app;
+        this.options    = options;
+        this.chart      = options.chart;
+    },
+            
+    // render
+    refresh : function(data)
+    {
+        // add graph to screen
+        var self = this;
+        nv.addGraph(function() {
+            self.chart_3d = nv.models.lineChart();
+
+            self.chart_3d.xAxis
+                .tickFormat(d3.format(',f'));
+
+            self.chart_3d.yAxis
+                .tickFormat(d3.format(',.2f'));
+            
+            d3.select(self.options.svg_id)
+                .datum(data)
+                .call(self.chart_3d);
+
+            nv.utils.windowResize(self.chart_3d.update);
+            
+            return self.chart_3d;
+        });
+    }
+});
+
+});

config/plugins/visualizations/charts/static/charts/linewithfocus.js

@@ -0,0 +1,37 @@
+// dependencies
+define(['library/utils'], function(Utils) {
+
+// widget
+return Backbone.View.extend(
+{
+    // initialize
+    initialize: function(app, options) {
+        this.app        = app;
+        this.options    = options;
+        this.chart      = options.chart;
+    },
+            
+    // render
+    refresh : function(data)
+    {
+        // add graph to screen
+        var self = this;
+        nv.addGraph(function() {
+            self.chart_3d = nv.models.lineWithFocusChart();
+
+            self.chart_3d.xAxis
+                .tickFormat(d3.format(',f'));
+
+            self.chart_3d.yAxis
+                .tickFormat(d3.format(',.2f'));
+            
+            d3.select(self.options.svg_id)
+                .datum(data)
+                .call(self.chart_3d);
+
+            nv.utils.windowResize(self.chart_3d.update);
+        });
+    }
+});
+
+});

config/plugins/visualizations/charts/static/charts/piechart.js

@@ -0,0 +1,51 @@
+// dependencies
+define(['library/utils'], function(Utils) {
+
+// widget
+return Backbone.View.extend(
+{
+    // initialize
+    initialize: function(app, options) {
+        this.app        = app;
+        this.options    = options;
+        this.chart      = options.chart;
+    },
+            
+    // render
+    refresh : function(data)
+    {
+        // loop through data groups
+        for (var key in data) {
+            // get group
+            var group = data[key];
+            
+            // format chart data
+            var pie_data = [];
+            for (var key in group.values) {
+                var value = group.values[key];
+                pie_data.push ({
+                    key : value.x,
+                    y   : value.y
+                });
+            }
+            
+            // add graph to screen
+            var self = this;
+            nv.addGraph(function() {
+                self.chart_3d = nv.models.pieChart()
+                    .donut(true)
+                    .showLegend(false);
+                
+                d3.select(self.options.svg_id)
+                    .datum(pie_data)
+                    .call(self.chart_3d);
+
+                nv.utils.windowResize(self.chart_3d.update);
+                
+                return self.chart_3d;
+            });
+        }
+    }
+});
+
+});

config/plugins/visualizations/charts/static/charts/scatterplot.js

@@ -0,0 +1,37 @@
+// dependencies
+define(['library/utils'], function(Utils) {
+
+// widget
+return Backbone.View.extend(
+{
+    // initialize
+    initialize: function(app, options) {
+        this.app        = app;
+        this.options    = options;
+        this.chart      = options.chart;
+    },
+            
+    // render
+    refresh : function(data)
+    {
+        // add graph to screen
+        var self = this;
+        nv.addGraph(function() {
+            self.d3_chart = nv.models.scatterChart()
+                .showDistX(true)
+                .showDistY(true)
+                .color(d3.scale.category10().range());
+                
+            self.d3_chart.xAxis.tickFormat(d3.format('.02f'))
+            self.d3_chart.yAxis.tickFormat(d3.format('.02f'))
+            
+            d3.select(self.options.svg_id)
+                .datum(data)
+                .call(self.d3_chart);
+ 
+            nv.utils.windowResize(self.d3_chart.update);
+        });
+    }
+});
+
+});

config/plugins/visualizations/charts/static/charts/stackedarea.js

@@ -0,0 +1,59 @@
+// dependencies
+define(['library/utils'], function(Utils) {
+
+// widget
+return Backbone.View.extend(
+{
+    // initialize
+    initialize: function(app, options) {
+        this.app        = app;
+        this.options    = options;
+        this.chart      = options.chart;
+    },
+            
+    // render
+    refresh : function(data)
+    {
+        // add graph to screen
+        var self = this;
+        nv.addGraph(function() {
+            // check data
+            var valid = true;
+            var length = 0;
+            for (var key in data) {
+                // evalute length
+                if (length == 0) {
+                    length = data[key].values.length;
+                } else {
+                    if (length != data[key].values.length) {
+                        valid = false;
+                        break;
+                    }
+                }
+            }
+            if (!valid) {
+                return;
+            }
+            
+            // make plot
+            self.d3_chart = nv.models.stackedAreaChart()
+                .x(function(d) {
+                    return d.x
+                })
+                .y(function(d) {
+                    return d.y
+                })
+                .clipEdge(true);
+            
+            self.d3_chart.xAxis.tickFormat(function() { return ''; });
+            
+            d3.select(self.options.svg_id)
+                .datum(data)
+                .call(self.d3_chart);
+ 
+            nv.utils.windowResize(self.d3_chart.update);
+        });
+    }
+});
+
+});

config/plugins/visualizations/charts/static/library/portlet.js

@@ -0,0 +1,171 @@
+// dependencies
+define(['library/utils'], function(Utils) {
+
+// return
+return Backbone.View.extend(
+{
+    // visibility
+    visible: false,
+    
+    // defaults options
+    optionsDefault: {
+        label       : '',
+        icon        : 'fa-tasks',
+        buttons     : null,
+        body        : null,
+        height      : null,
+        operations  : null,
+        placement   : 'bottom',
+        overflow    : 'auto'
+    },
+    
+    // content
+    $content : null,
+    
+    // initialize
+    initialize : function(options) {
+        // configure options
+        this.options = Utils.merge(options, this.optionsDefault);
+        
+        // create new element
+        this.setElement(this.template(this.options));
+        
+        // link content
+        this.$content = this.$el.find('#content');
+        
+        // set content height
+        if (this.options.height) {
+            this.$el.find('#body').css('height', this.options.height);
+            this.$el.find('#content').css('overflow', this.options.overflow);
+        }
+        
+        // append buttons
+        this.$buttons = $(this.el).find('#buttons');
+        if (this.options.buttons) {
+            // link functions
+            var self = this;
+            $.each(this.options.buttons, function(name, item) {
+                item.$el.prop('id', name);
+                self.$buttons.append(item.$el);
+            });
+        } else {
+            this.$buttons.remove();
+        }
+        
+        // append operations
+        this.$operations = $(this.el).find('#operations');
+        if (this.options.operations) {
+            // link functions
+            var self = this;
+            $.each(this.options.operations, function(name, item) {
+                item.$el.prop('id', name);
+                self.$operations.append(item.$el);
+            });
+        }
+        
+        // add body
+        if(this.options.body) {
+            this.append(this.options.body);
+        }
+    },
+    
+    // append
+    append: function($el) {
+        this.$content.append(Utils.wrap($el));
+    },
+    
+    // content
+    content: function() {
+        return this.$content;
+    },
+    
+    // hide modal
+    show: function(){
+        // fade in
+        this.$el.fadeIn('fast');
+        
+        // set flag
+        this.visible = true;
+    },
+    
+    // hide modal
+    hide: function(){
+        // fade out
+        this.$el.fadeOut('fast');
+        
+        // set flag
+        this.visible = false;
+    },
+
+    // enable buttons
+    enableButton: function(id) {
+        this.$buttons.find('#' + id).prop('disabled', false);
+    },
+
+    // disable buttons
+    disableButton: function(id) {
+        this.$buttons.find('#' + id).prop('disabled', true);
+    },
+    
+    // enable operation
+    hideOperation: function(id) {
+        this.$operations.find('#' + id).hide();
+    },
+
+    // disable operation
+    showOperation: function(id) {
+        this.$operations.find('#' + id).show();
+    },
+    
+    // set operation
+    setOperation: function(id, callback) {
+        var $el = this.$operations.find('#' + id);
+        $el.off('click');
+        $el.on('click', callback);
+    },
+    
+    // label
+    label: function(new_label) {
+        var $el = this.$el.find('#label');
+        if (new_label) {
+            $el.html(new_label);
+        }
+        return $el.html();
+    },
+    
+    // fill regular modal template
+    template: function(options) {
+        var tmpl =  '<div class="toolForm">';
+        
+        if (options.label || options.icon) {
+            tmpl +=     '<div id="title" class="toolFormTitle" style="overflow:hidden;">' +
+                            '<div id="operations" style="float: right;"></div>' +
+                            '<div style="overflow: hidden">';
+                            
+            if (options.icon)
+                tmpl +=         '<i style="padding-top: 3px; float: left; font-size: 1.2em" class="icon fa ' + options.icon + '">&nbsp;</i>';
+        
+            tmpl +=             '<div id="label" style="padding-top: 2px; float: left;">' + options.label + '</div>';
+            
+            tmpl +=         '</div>' +
+                        '</div>';
+        }
+        tmpl +=         '<div id="body" class="toolFormBody">';
+        
+        if (options.placement == 'top') {
+            tmpl +=         '<div id="buttons" class="buttons" style="height: 50px; padding: 10px;"></div>';
+        }
+        
+        tmpl +=             '<div id="content" class="content" style="height: inherit; padding: 10px;"></div>';
+        
+        if (options.placement == 'bottom') {
+            tmpl +=         '<div id="buttons" class="buttons" style="height: 50px; padding: 10px;"></div>';
+        }
+        
+        tmpl +=         '</div>' +
+                    '</div>';
+        return tmpl;
+    }
+});
+
+});

config/plugins/visualizations/charts/static/library/table.js

@@ -0,0 +1,188 @@
+// dependencies
+define(['library/utils'], function(Utils) {
+
+// return
+return Backbone.View.extend(
+{
+    // current row
+    row: null,
+    
+    // count rows
+    row_count: 0,
+    
+    // defaults options
+    optionsDefault: {
+        content     : 'No content available.',
+        onchange    : null,
+        ondblclick  : null,
+        onconfirm   : null
+    },
+    
+    // events
+    events : {
+        'click'     : 'onclick',
+        'dblclick'  : 'ondblclick'
+    },
+    
+    // first
+    first: true,
+    
+    // initialize
+    initialize : function(options) {
+        // configure options
+        this.options = Utils.merge(options, this.optionsDefault);
+        
+        // create new element
+        this.setElement(this.template(options));
+                
+        // initialize row
+        this.row = $('<tr></tr>');
+    },
+    
+    // add header cell
+    addHeader: function($el) {
+        var wrapper = $('<th></th>');
+        wrapper.append($el);
+        this.row.append(wrapper);
+    },
+    
+    // header
+    appendHeader: function() {
+        // append header row
+        $(this.el).find('thead').append(this.row);
+
+        // row
+        this.row = $('<tr></tr>');
+    },
+    
+    // add row cell
+    add: function($el) {
+        var wrapper = $('<td></td>');
+        wrapper.append($el);
+        this.row.append(wrapper);
+    },
+    
+    // append
+    append: function(id) {
+        this.commit(id);
+    },
+    
+    // prepend
+    prepend: function(id) {
+        this.commit(id, true);
+    },
+    
+    // commit
+    commit: function(id, prepend) {
+        // add
+        this.row.attr('id', id);
+       
+        // add row
+        if (prepend) {
+            $(this.el).find('tbody').prepend(this.row);
+        } else {
+            $(this.el).find('tbody').append(this.row);
+        }
+        
+        // row
+        this.row = $('<tr></tr>');
+        
+        // row count
+        this.row_count++;
+        this.refresh();
+    },
+    
+    // remove
+    remove: function(id) {
+        $(this.el).find('#' + id).remove();
+        this.row_count--;
+        this.refresh();
+    },
+
+    // remove
+    removeAll: function() {
+        $(this.el).find('tbody').html('');
+        this.row_count = 0;
+        this.refresh();
+    },
+        
+    // value
+    value: function(new_value) {
+        // get current id/value
+        this.before = this.$el.find('.current').attr('id');
+        
+        // check if new_value is defined
+        if (new_value !== undefined) {
+            this.$el.find('tr').removeClass('current');
+            if (new_value) {
+                this.$el.find('#' + new_value).addClass('current');
+            }
+        }
+        
+        // get current id/value
+        var after = this.$el.find('.current').attr('id');
+        if(after === undefined) {
+            return null;
+        } else {
+            // fire onchange
+            if (after != this.before && this.options.onchange) {
+                this.options.onchange(new_value);
+            }
+            
+            // return current value
+            return after;
+        }
+    },
+    
+    // confirm new value
+    confirm: function(new_value) {
+        this.value(new_value);
+    },
+    
+    // onclick
+    onclick: function(e) {
+        // get values
+        var old_value = this.value();
+        var new_value = $(e.target).closest('tr').attr('id');
+        
+        // check equality
+        if (new_value && old_value != new_value) {
+            if (this.options.onconfirm) {
+                this.options.onconfirm(new_value);
+            } else {
+                this.confirm(new_value);
+            }
+        }
+    },
+
+    // onclick
+    ondblclick: function(e) {
+        var value = this.value();
+        if (value && this.options.ondblclick) {
+            this.options.ondblclick(value);
+        }
+    },
+        
+    // refresh
+    refresh: function() {
+        if (this.row_count == 0) {
+            this.$el.find('tmessage').show();
+        } else {
+            this.$el.find('tmessage').hide();
+        }
+    },
+        
+    // load html template
+    template: function(options)
+    {
+        return  '<div>' +
+                    '<table class="grid">' +
+                        '<thead></thead>' +
+                        '<tbody style="cursor: pointer;"></tbody>' +
+                    '</table>' +
+                    '<tmessage>' + options.content + '</tmessage>' +
+                '<div>';
+    }
+});
+
+});

config/plugins/visualizations/charts/static/library/ui.js

@@ -0,0 +1,472 @@
+// dependencies
+define(['library/utils', 'library/ui.select'], function(Utils, Select) {
+
+// plugin
+var Label = Backbone.View.extend(
+{
+    // options
+    optionsDefault: {
+        label : ''
+    },
+    
+    // initialize
+    initialize : function(options) {
+        // get options
+        this.options = Utils.merge(options, this.optionsDefault);
+            
+        // create new element
+        this.setElement(this.template(this.options));
+    },
+    
+    // template
+    template: function(options) {
+        return '<label><b>' + options.label + '</b></label>';
+    },
+    
+    // value
+    value: function() {
+        return options.label;
+    }
+});
+
+// plugin
+var Button = Backbone.View.extend(
+{
+    // options
+    optionsDefault: {
+        id    : null,
+        label : '',
+        float : 'right',
+        cls   : 'btn-default',
+        type  : 'submit',
+        icon  : ''
+    },
+    
+    // initialize
+    initialize : function(options) {
+        // get options
+        this.options = Utils.merge(options, this.optionsDefault);
+            
+        // create new element
+        this.setElement(this._template(this.options));
+        
+        // add event
+        $(this.el).on('click', options.onclick);
+    },
+    
+    // element
+    _template: function(options) {
+        var str =   '<button id="' + options.id + '" type="' + options.type + '" style="margin-right: 5px; float: ' + options.float + ';" type="button" class="btn ' + options.cls + '">';
+        if (options.icon) {
+            str +=      '<i class="icon fa ' + options.icon + '"></i>&nbsp;' ;
+        }
+        str +=          options.label +
+                    '</button>';
+        return str;
+    }
+});
+
+// plugin
+var Anchor = Backbone.View.extend(
+{
+    // options
+    optionsDefault: {
+        label : ''
+    },
+    
+    // initialize
+    initialize : function(options) {
+        // get options
+        this.options = Utils.merge(options, this.optionsDefault);
+            
+        // create new element
+        this.setElement(this._template(this.options));
+        
+        // add event
+        $(this.el).on('click', options.onclick);
+    },
+    
+    // element
+    _template: function(options) {
+        return '<div><a href="javascript:void(0)">' + options.label + '</a></div>';
+    }
+});
+
+// plugin
+var Message = Backbone.View.extend(
+{
+    // options
+    optionsDefault: {
+        message : '',
+        status : 'info',
+        persistent : false
+    },
+    
+    // initialize
+    initialize : function(options) {
+        // configure options
+        this.options = Utils.merge(options, this.optionsDefault);
+        
+        // create new element
+        this.setElement('<div></div>');
+    },
+    
+    // update
+    update : function(options) {
+        // get options
+        this.options = Utils.merge(options, this.optionsDefault);
+        
+        // show message
+        if (options.message != '') {
+            this.$el.html(this.template(this.options));
+            this.$el.fadeIn();
+            
+            // check if message is persistent
+            if (!options.persistent) {
+                // set timer
+                var self = this;
+                window.setTimeout(function() {
+                    if (self.$el.is(':visible')) {
+                        self.$el.fadeOut();
+                    } else {
+                        self.$el.hide();
+                    }
+                }, 3000);
+            }
+        } else {
+            this.$el.fadeOut();
+        }
+    },
+    
+    // element
+    template: function(options) {
+        return '<div class="alert alert-' + options.status + '" style="padding: 2px 2px 2px 10px;">' + options.message + '</div>';
+    }
+});
+
+// plugin
+var Searchbox = Backbone.View.extend(
+{
+    // options
+    optionsDefault: {
+        onclick : null,
+        searchword : ''
+    },
+    
+    // initialize
+    initialize : function(options) {
+        // configure options
+        this.options = Utils.merge(options, this.optionsDefault);
+        
+        // create new element
+        this.setElement(this.template(this.options));
+        
+        // add click event
+        var self = this;
+        if (this.options.onclick) {
+            this.$el.on('submit', function(e) {
+                var search_field = self.$el.find('#search');
+                self.options.onclick(search_field.val());
+            });
+        }
+    },
+    
+    // element
+    template: function(options) {
+        return  '<div class="search">' +
+                    '<form onsubmit="return false;">' +
+                        '<input id="search" class="form-control input-sm" type="text" name="search" placeholder="Search..." value="' + options.searchword + '">' +
+                        '<button type="submit" class="btn search-btn">' +
+                            '<i class="fa fa-search"></i>' +
+                        '</button>' +
+                    '</form>' +
+                '</div>';
+    }
+});
+
+// plugin
+var Title = Backbone.View.extend(
+{
+    // options
+    optionsDefault: {
+        label   : 'Unlabeled',
+        body    : null
+    },
+    
+    // initialize
+    initialize : function(options) {
+        // configure options
+        this.options = Utils.merge(options, this.optionsDefault);
+        
+        // create new element
+        this.setElement(this.template(this.options));
+        
+        if (this.options.body) {
+            this.$el.find('.body').append(this.options.body);
+        }
+    },
+    
+    // element
+    template: function(options) {
+        return  '<div id="title" class="title">' +
+                    options.label + ':' +
+                '</div>';
+    }
+});
+
+// tab
+var ButtonMenu = Backbone.View.extend(
+{
+    // main options
+    optionsDefault:
+    {
+        id              : '',
+        title           : '',
+        target          : '',
+        href            : '',
+        onunload        : null,
+        onclick         : null,
+        visible         : true,
+        icon            : null,
+        tag             : ''
+    },
+    
+    // optional sub menu
+    $menu: null,
+    
+    // initialize
+    initialize: function (options)
+    {
+        // get options
+        this.options = Utils.merge(options, this.optionsDefault);
+        
+        // add template for tab
+        this.setElement($(this._template(this.options)));
+        
+        // find root
+        var $root = $(this.el).find('.root');
+        
+        // link head
+        var self = this;
+        $root.on('click', function(e)
+        {
+            // prevent default
+            e.preventDefault();
+            
+            // add click event
+            if(self.options.onclick) {
+                self.options.onclick();
+            }
+        });
+        
+        // visiblity
+        if (!this.options.visible)
+            this.hide();
+    },
+    
+    // show
+    show: function()
+    {
+        $(this.el).show();
+    },
+        
+    // hide
+    hide: function()
+    {
+        $(this.el).hide();
+    },
+    
+    // add menu item
+    addMenu: function (options)
+    {
+        // menu option defaults
+        var menuOptions = {
+            title       : '',
+            target      : '',
+            href        : '',
+            onclick     : null,
+            divider     : false,
+            icon        : null
+        }
+    
+        // get options
+        menuOptions = Utils.merge(options, menuOptions);
+        
+        // check if submenu element is available
+        if (!this.$menu)
+        {
+            // insert submenu element into root
+            $(this.el).append(this._templateMenu());
+            
+            // update element link
+            this.$menu = $(this.el).find('.menu');
+        }
+        
+        // create
+        var $item = $(this._templateMenuItem(menuOptions));
+        
+        // add events
+        $item.on('click', function(e)
+        {
+            // prevent default
+            e.preventDefault();
+            
+            // add click event
+            if(menuOptions.onclick) {
+                menuOptions.onclick();
+            }
+        });
+        
+        // append menu
+        this.$menu.append($item);
+        
+        // append divider
+        if (menuOptions.divider)
+            this.$menu.append($(this._templateDivider()));
+    },
+    
+    // fill template header
+    _templateMenuItem: function (options)
+    {
+        var tmpl =  '<li>' +
+                        '<a href="' + options.href + '" target="' + options.target + '">';
+                
+        if (options.icon)
+            tmpl +=         '<i class="fa ' + options.icon + '"></i>';
+        
+        tmpl +=             ' ' + options.title +
+                        '</a>' +
+                    '</li>';
+        return tmpl;
+    },
+    
+    // fill template header
+    _templateMenu: function ()
+    {
+        return '<ul class="menu dropdown-menu pull-right" role="menu"></ul>';
+    },
+    
+    _templateDivider: function()
+    {
+        return '<li class="divider"></li>';
+    },
+    
+    // fill template
+    _template: function (options)
+    {
+        // start template
+        var tmpl =  '<div id="' + options.id + '" class="button-menu btn-group">' +
+                        '<button type="button" class="root btn btn-default dropdown-toggle" data-toggle="dropdown">';
+        
+        if (options.icon)
+            tmpl +=         '<i class="fa ' + options.icon + '"></i>';
+         
+                        '</button>' +
+                    '</div>';
+        
+        // return template
+        return tmpl;
+    }
+});
+
+// plugin
+var ButtonIcon = Backbone.View.extend(
+{
+    // options
+    optionsDefault: {
+        id      : null,
+        label   : '',
+        float   : 'right',
+        cls     : 'icon-btn',
+        type    : 'submit',
+        icon    : '',
+        tooltip : ''
+    },
+    
+    // initialize
+    initialize : function(options) {
+        // get options
+        this.options = Utils.merge(options, this.optionsDefault);
+            
+        // create new element
+        this.setElement(this.template(this.options));
+        
+        // add event
+        $(this.el).on('click', options.onclick);
+        
+        // add tooltip
+        $(this.el).tooltip({title: options.tooltip, placement: 'bottom'});
+    },
+    
+    // element
+    template: function(options) {
+        var str =   '<a id="' + options.id + '" style="margin-right: 5px; float: ' + options.float + ';" class="' + options.cls + '">';
+        if (options.icon) {
+            str +=      '<i class="icon fa ' + options.icon + '"></i>';
+        }
+        str +=          options.label +
+                    '</a>';
+        return str;
+    }
+});
+
+
+// plugin
+var Input = Backbone.View.extend(
+{
+    // options
+    optionsDefault: {
+        value           : '',
+        type            : 'text',
+        placeholder     : '',
+        disabled        : false,
+        visible         : true
+    },
+    
+    // initialize
+    initialize : function(options) {
+        // configure options
+        this.options = Utils.merge(options, this.optionsDefault);
+            
+        // create new element
+        this.setElement(this.template(this.options));
+        
+        // disable input field
+        if (this.options.disabled) {
+            this.$el.prop('disabled', true);
+        }
+        
+        // hide input field
+        if (!this.options.visible) {
+            this.$el.hide();
+        }
+    },
+    
+    // value
+    value : function (new_val) {
+        if (new_val !== undefined) {
+            this.$el.val(new_val);
+        }
+        return this.$el.val();
+    },
+    
+    // element
+    template: function(options) {
+        return '<input id="' + options.id + '" type="' + options.type + '" value="' + options.value + '" placeholder="' + options.placeholder + '" class="form-control">';
+    }
+});
+
+// return
+return {
+    Label   : Label,
+    Button  : Button,
+    ButtonIcon : ButtonIcon,
+    Input : Input,
+    Anchor  : Anchor,
+    Message : Message,
+    Searchbox : Searchbox,
+    Title : Title,
+    Select : Select,
+    ButtonMenu : ButtonMenu
+}
+});

config/plugins/visualizations/charts/static/library/ui.select.js

@@ -0,0 +1,165 @@
+// dependencies
+define(['library/utils'], function(Utils) {
+
+// plugin
+return Backbone.View.extend(
+{
+    // options
+    optionsDefault: {
+        id      : '',
+        cls     : '',
+        empty   : 'No data available'
+    },
+    
+    // initialize
+    initialize : function(options) {
+        // configure options
+        this.options = Utils.merge(options, this.optionsDefault);
+        
+        // create new element
+        this.setElement(this._template(this.options));
+        
+        // add change event
+        var self = this;
+        if (this.options.onchange) {
+            this.$el.on('change', function() { self.options.onchange(self.value()); });
+        }
+        
+        // refresh
+        this._refresh();
+    },
+    
+    // value
+    value : function (new_value) {
+        // get current id/value
+        var before = this.$el.val();
+        
+        // check if new_value is defined
+        if (new_value !== undefined) {
+            this.$el.val(new_value);
+        }
+        
+        // get current id/value
+        var after = this.$el.val();
+        if(after === undefined) {
+            return null;
+        } else {
+            // fire onchange
+            if ((after != before && this.options.onchange) || this.$el.find('option').length == 1) {
+                this.options.onchange(after);
+            }
+            
+            // return current value
+            return after;
+        }
+    },
+    
+    // label
+    label : function () {
+        return this.$el.find('option:selected').text();
+    },
+    
+    // disabled
+    disabled: function() {
+        return this.$el.is(':disabled');
+    },
+
+    // enable
+    enable: function() {
+        this.$el.prop('disabled', false);
+    },
+        
+    // disable
+    disable: function() {
+        this.$el.prop('disabled', true);
+    },
+    
+    // add
+    add: function(options) {
+        // add options
+        $(this.el).append(this._templateOption(options));
+        
+        // refresh
+        this._refresh();
+    },
+    
+    // remove
+    remove: function(value) {
+        // remove option
+        $(this.el).find('option[value=' + value + ']').remove();
+        $(this.el).trigger('change');
+        
+        // refresh
+        this._refresh();
+    },
+    
+    // render
+    update: function(options) {
+        // selected
+        var selected = this.$el.val();
+
+        // remove all options
+        $(this.el).find('option').remove();
+
+        // add new options
+        for (var key in options.data) {
+            $(this.el).append(this._templateOption(options.data[key]));
+        }
+        
+        // add selected value
+        if (this._exists(selected))
+            $(this.el).val(selected);
+        
+        // refresh
+        this._refresh();
+    },
+    
+    // refresh
+    _refresh: function() {
+        // remove placeholder
+        $(this.el).find('option[value=null]').remove();
+        
+        // count remaining entries
+        var remaining = $(this.el).find('option').length;
+        if (remaining == 0) {
+            // append placeholder
+            $(this.el).append(this._templateOption({value : 'null', label : this.options.empty}));
+            this.disable();
+        } else {
+            this.enable();
+        }
+    },
+    
+    // exists
+    _exists: function(value) {
+        // check if selected value exists
+        return 0 != $(this.el).find('option[value=' + value + ']').length;
+    },
+    
+    // option
+    _templateOption: function(options) {
+        return '<option value="' + options.value + '">' + options.label + '</option>';
+    },
+    
+    // element
+    _template: function(options) {
+        var tmpl =  '<select id="' + options.id + '" class="select ' + options.cls + ' ' + options.id + '">';
+        for (key in options.data) {
+            // options
+            var item = options.data[key];
+            
+            // identify selected value
+            var tag = '';
+            if (item.value == options.selected || item.value == '') {
+                tag = 'selected';
+            }
+            
+            // add template string
+            tmpl +=     '<option value="' + item.value + '" ' + tag + '>' + item.label + '</option>';
+        }
+        tmpl +=     '</select>';
+        return tmpl;
+    }
+});
+
+});

config/plugins/visualizations/charts/static/library/utils.js

@@ -0,0 +1,130 @@
+// dependencies
+define([], function() {
+
+// get
+function get (url, success, error) {
+    request('GET', url, {}, success, error);
+};
+
+// generic function to send json to url
+function request (method, url, data, success, error) {
+
+    // encode data into url
+    if (method == 'GET' || method == 'DELETE') {
+        url = url + "?" + $.param(data);
+    }
+    
+    // prepare request
+    var xhr = new XMLHttpRequest();
+    xhr.open(method, url, true);
+    xhr.setRequestHeader('Accept', 'application/json');
+    xhr.setRequestHeader('Cache-Control', 'no-cache');
+    xhr.setRequestHeader('X-Requested-With', 'XMLHttpRequest');
+    xhr.setRequestHeader('Content-Type', 'application/json');
+    xhr.onloadend = function() {
+        var status = xhr.status;
+        if (status == 200) {
+            try {
+                response = jQuery.parseJSON(xhr.responseText);
+            } catch (e) {
+                response = xhr.responseText;
+            }
+            success && success(response);
+        } else {
+            error && error(status);
+        }
+    };
+    
+    // make request
+    if (method == 'GET' || method == 'DELETE') {
+        xhr.send();
+    } else {
+        xhr.send(JSON.stringify(data));
+    }
+};
+
+// merge
+function merge (options, optionsDefault) {
+    if (options)
+        return _.defaults(options, optionsDefault);
+    else
+        return optionsDefault;
+};
+
+// get css value
+function cssGetAttribute (classnames, name) {
+    if (classnames.length == 0) {
+       console.log('cssGetAttribute() : Requires a list of classnames');
+    }
+
+    // place dummy element
+    var el = $('<div class="' + classnames[0] + '"></div>');
+    var traverse_el = el;
+    for (var i in classnames) {
+        traverse_el.append('<div class="' + classnames[i] + '"></div>');
+        traverse_el = traverse_el.find('.' + classnames[i]);
+    }
+      
+    // required append
+    el.appendTo(':eq(0)');
+    
+    // get value
+    var value = el.css(name);
+    
+    // remove element
+    el.remove();
+        
+    // return css value
+    return value;
+};
+    
+// load css
+function cssLoadFile (url) {
+    // check if css is already available
+    if (!$('link[href^="' + url + '"]').length)
+        $('<link href="' + galaxy_config.root + url + '" rel="stylesheet">').appendTo('head');
+};
+
+// unique ide
+function uuid(){
+    return (new Date().getTime()).toString(36);
+};
+
+// wrap
+function wrap($el) {
+    var wrapper = $('<p></p>');
+    wrapper.append($el);
+    return wrapper;
+};
+
+// time
+function time() {
+    // get date object
+    var d = new Date();
+    
+    // format items
+    var hours = (d.getHours() < 10 ? "0" : "") + d.getHours();
+    var minutes = (d.getMinutes() < 10 ? "0" : "") + d.getMinutes()
+    
+    // format final stamp
+    var datetime = d.getDate() + "/"
+                + (d.getMonth() + 1)  + "/"
+                + d.getFullYear() + ", "
+                + hours + ":"
+                + minutes;
+    return datetime;
+};
+
+// return
+return {
+    cssLoadFile : cssLoadFile,
+    cssGetAttribute : cssGetAttribute,
+    get : get,
+    merge : merge,
+    request: request,
+    uuid: uuid,
+    wrap : wrap,
+    time : time
+};
+
+});

config/plugins/visualizations/charts/static/models/chart.js

@@ -0,0 +1,43 @@
+// dependencies
+define(['models/groups'], function(Groups) {
+
+
+// model
+return Backbone.Model.extend(
+{
+    // defaults
+    defaults : {
+        id      : null,
+        title   : '',
+        type    : '',
+        date    : null
+    },
+    
+    // initialize
+    initialize: function(options)
+    {
+        this.groups = new Groups();
+    },
+    
+    // reset
+    reset: function()
+    {
+        this.clear({silent: true}).set(this.defaults);
+        this.groups.reset();
+        this.trigger('reset', this);
+    },
+    
+    // copy
+    copy: function(new_chart) {
+        // copy chart/groups
+        var current = this;
+        current.set(new_chart.attributes);
+        current.groups.reset();
+        new_chart.groups.each(function(group) {
+            current.groups.add(group.clone());
+        });
+        current.trigger('change', current);
+    }
+});
+
+});

config/plugins/visualizations/charts/static/models/charts.js

@@ -0,0 +1,10 @@
+// dependencies
+define(['models/chart'], function(Chart) {
+
+// collection
+return Backbone.Collection.extend(
+{
+    model: Chart
+});
+
+});

config/plugins/visualizations/charts/static/models/config.js

@@ -0,0 +1,16 @@
+// dependencies
+define([], function() {
+
+
+// model
+return Backbone.Model.extend(
+{
+    // options
+    defaults : {
+        query_limit     : 20,
+        query_pace      : 1000,
+        query_max       : 5
+    }
+});
+
+});

config/plugins/visualizations/charts/static/models/datasets.js

@@ -0,0 +1,125 @@
+// dependencies
+define(['library/utils'], function(Utils) {
+
+// widget
+return Backbone.Model.extend(
+{
+    // options
+    optionsDefault : {
+        limit : 500,
+        pace  : 1000,
+        max   : 2
+    },
+
+    // list
+    list: {},
+    
+    // initialize
+    initialize: function(app, options)
+    {
+        // link app
+        this.app = app;
+        
+        // configure options
+        this.options = Utils.merge(options, this.optionsDefault);
+    },
+
+    // get
+    get: function(options, callback) {
+        // check if dataset already exists
+        var dataset = this.list[options.id];
+        if (dataset) {
+            if (dataset.values) {
+                if (dataset.values[options.column]) {
+                    if (callback) {
+                        callback(dataset);
+                    }
+                    return dataset;
+                }
+            }
+        } else {
+            this.list[options.id] = dataset = this.app.options.dataset;
+        }
+        
+        // initiate fetch cycle
+        if (options.column !== undefined) {
+            this._fetch(options, callback);
+        } else {
+            if (callback) {
+                callback(dataset);
+            }
+            return dataset;
+        }
+        
+        // return
+        return dataset;
+    },
+    
+    // size
+    length: function() {
+        return this.list.length;
+    },
+    
+    // request data
+    _fetch : function(options, callback) {
+        // get dataset
+        var dataset = this.list[options.id];
+        
+        // initialize values for datasets
+        if (!dataset.values) {
+            dataset.values = {};
+        }
+        if (!dataset.values[options.column]) {
+            dataset.values[options.column] = [];
+        }
+        
+        // initialize page number
+        if (options.page === undefined) {
+            options.page = 0;
+        } else {
+            options.page++;
+        }
+        
+        // check max page
+        if (options.page > this.options.max) {
+            return;
+        }
+        
+        // config limits
+        var limit   = this.options.limit;
+        var offset  = this.options.limit * options.page;
+        
+        // make request
+        var self = this;
+        Utils.request('GET', config.root + 'api/datasets/' + options.id, {
+                data_type   : 'raw_data',
+                provider    : 'dataset-column',
+                limit       : limit,
+                offset      : offset,
+                indeces     : options.column
+            }, function(response) {
+                // verify if all data has been fetched
+                if (response.data.length > 0) {
+                    // add values to dataset
+                    for (var key in response.data){
+                        var value = response.data[key][0];
+                        if (value !== undefined) {
+                            dataset.values[options.column].push(value);
+                        } else {
+                            return dataset;
+                        }
+                    }
+                    
+                    // update charts
+                    if (callback) {
+                        callback(dataset);
+                    }
+                    
+                    // next call
+                    setTimeout(function() { self._fetch(options, callback); }, self.options.pace);
+                }
+            });
+    }
+});
+
+});

config/plugins/visualizations/charts/static/models/group.js

@@ -0,0 +1,15 @@
+// dependencies
+define([], function() {
+
+// model
+return Backbone.Model.extend(
+{
+    // reset
+    reset: function()
+    {
+        this.clear({silent: true}).set(this.defaults);
+        this.trigger('reset', this);
+    }
+});
+
+});

config/plugins/visualizations/charts/static/models/groups.js

@@ -0,0 +1,10 @@
+// dependencies
+define(['models/group'], function(Group) {
+
+// collection
+return Backbone.Collection.extend(
+{
+    model: Group
+});
+
+});

config/plugins/visualizations/charts/static/models/types.js

@@ -0,0 +1,71 @@
+// dependencies
+define([], function() {
+
+// widget
+return Backbone.Model.extend(
+{
+    // types
+    defaults: {
+        'bardiagram' : {
+            title : 'Bar diagram',
+            data  : {
+                y : {
+                    title   : 'Values for y-axis'
+                }
+            }
+        },
+        'horizontal' : {
+            title : 'Bar diagram (horizontal)',
+            data  : {
+                y : {
+                    title   : 'Values for y-axis'
+                }
+            }
+        },
+        'line' : {
+            title : 'Line chart',
+            data  : {
+                y : {
+                    title   : 'Values for y-axis'
+                }
+            }
+        },
+        'linewithfocus' : {
+            title : 'Line chart (with focus)',
+            data  : {
+                y : {
+                    title   : 'Values for y-axis'
+                }
+            }
+        },
+        'piechart' : {
+            title : 'Pie chart',
+            data  : {
+                y : {
+                    title   : 'Values for y-axis'
+                }
+            }
+        },
+        'scatterplot' : {
+            title : 'Scatter plot',
+            data  : {
+                x : {
+                    title   : 'Values for x-axis'
+                },
+                y : {
+                    title   : 'Values for y-axis'
+                }
+            }
+        },
+        'stackedarea' : {
+            title : 'Stacked area',
+            data  : {
+                y : {
+                    title   : 'Values for y-axis'
+                }
+            }
+        }
+    }
+});
+
+});

config/plugins/visualizations/charts/static/views/chart.js

@@ -0,0 +1,201 @@
+// dependencies
+define(['library/portlet', 'library/table', 'library/ui', 'library/utils', 'models/chart', 'views/groups'],
+    function(Portlet, Table, Ui, Utils, Chart, GroupsView) {
+
+// widget
+return Backbone.View.extend(
+{
+    // defaults options
+    optionsDefault: {
+        header  : true,
+        content : 'No content available.'
+    },
+
+    // initialize
+    initialize: function(app, options)
+    {
+        // link application
+        this.app = app;
+        
+        // get current chart object
+        this.chart = this.app.chart;
+        
+        // configure options
+        this.options = Utils.merge(options, this.optionsDefault);
+        
+        // main elements
+        this.message = new Ui.Message();
+        this.title = new Ui.Input({placeholder: 'Chart title'});
+        this.dataset = new Ui.Input({value : app.options.dataset.id, disabled: true, visible: false});
+        
+        // configure dataset
+        this.groups_view = new GroupsView(this.app);
+        
+        // table
+        var self = this;
+        this.table = new Table({
+            header : false,
+            onconfirm : function(type) {
+                if (self.chart.groups.length > 0) {
+                    // show modal
+                    self.app.modal.show({
+                        title   : 'Switching the chart type?',
+                        body    : 'You configured data sources. If you switch chart types your configurations will be removed.',
+                        buttons : {
+                            'Cancel'    : function() {
+                                // hide modal
+                                self.app.modal.hide();
+                            },
+                            'Continue'  : function() {
+                                // hide modal
+                                self.app.modal.hide();
+                                
+                                // confirm
+                                self.table.confirm(type);
+                            }
+                        }
+                    });
+                } else {
+                    // confirm
+                    self.table.confirm(type);
+                }
+            },
+            onchange : function(type) {
+                // update chart type
+                self.chart.set({type: type});
+                    
+                // reset groups
+                self.chart.groups.reset();
+            },
+            content: 'No chart types available'
+        });
+        
+        // add types
+        var types_n = 0;
+        var types = app.types.attributes;
+        for (var id in types){
+            var chart_type = types[id];
+            this.table.add (++types_n + '.');
+            this.table.add (chart_type.title);
+            this.table.append(id);
+        }
+                        
+        // add table to portlet
+        var self = this;
+        this.portlet = new Portlet({
+            icon : 'fa-edit',
+            label : 'Create a new chart:',
+            operations : {
+                'save'      : new Ui.ButtonIcon({
+                                icon    : 'fa-save',
+                                tooltip : 'Save Chart',
+                                onclick : function() {
+                                    self._saveChart();
+                                }
+                            }),
+                'back'      : new Ui.ButtonIcon({
+                                icon    : 'fa-caret-left',
+                                tooltip : 'Return',
+                                onclick : function() {
+                                    self.$el.hide();
+                                    self.app.charts_view.$el.show();
+                                }
+                            })
+            }
+        });
+        this.portlet.append(this.message.$el);
+        this.portlet.append(this.dataset.$el);
+        this.portlet.append((new Ui.Label({ label : 'Provide a chart title:'})).$el);
+        this.portlet.append(this.title.$el);
+        this.portlet.append((new Ui.Label({ label : 'Select a chart type:'})).$el);
+        this.portlet.append(this.table.$el);
+        this.portlet.append(this.groups_view.$el);
+
+        // elements
+        this.setElement(this.portlet.$el);
+        
+        // hide back button on startup
+        this.portlet.hideOperation('back');
+        
+        // model events
+        var self = this;
+        this.chart.on('change:title', function(chart) {
+            self.title.value(chart.get('title'));
+        });
+        this.chart.on('change:type', function(chart) {
+            self.table.value(chart.get('type'));
+        });
+        this.chart.on('reset', function(chart) {
+            self._resetChart();
+        });
+        
+        // collection events
+        this.app.charts.on('add', function(chart) {
+            self.portlet.showOperation('back');
+        });
+        this.app.charts.on('remove', function(chart) {
+            if (self.app.charts.length == 1) {
+                self.portlet.hideOperation('back');
+            }
+        });
+        this.app.charts.on('reset', function(chart) {
+            self.portlet.hideOperation('back');
+        });
+        
+        // reset
+        this._resetChart();
+    },
+
+    // reset
+    _resetChart: function() {
+        this.chart.set('id', Utils.uuid());
+        this.chart.set('dataset_id', this.app.options.dataset.id);
+        this.chart.set('type', 'bardiagram');
+        this.chart.set('title', 'Chart title');
+    },
+    
+    // create chart
+    _saveChart: function() {
+        // update chart data
+        this.chart.set({
+            type        : this.table.value(),
+            title       : this.title.value(),
+            dataset_id  : this.dataset.value(),
+            date        : Utils.time()
+        });
+        
+        // validate
+        if (!this.chart.get('title')) {
+            this.message.update({message : 'Please enter a title for your chart.'});
+            return;
+        }
+        
+        if (!this.chart.get('type')) {
+            this.message.update({message : 'Please select a chart type.'});
+            return;
+        }
+
+        if (this.chart.groups.length == 0) {
+            this.message.update({message : 'Please configure at least one data source.'});
+            return;
+        }
+        
+        // create/get chart
+        var current = this.app.charts.get(this.chart.id);
+        if (!current) {
+            current = this.chart.clone();
+            this.app.charts.add(current);
+        }
+                
+        // update chart model
+        current.copy(this.chart);
+        
+        // hide
+        this.$el.hide();
+        
+        // update main
+        this.app.charts_view.$el.show();
+    }
+});
+
+});

config/plugins/visualizations/charts/static/views/charts.js

@@ -0,0 +1,133 @@
+// dependencies
+define(['library/portlet', 'library/table', 'library/ui', 'library/utils'], function(Portlet, Table, Ui, Utils) {
+
+// widget
+return Backbone.View.extend(
+{
+    // initialize
+    initialize: function(app, options)
+    {
+        // link app
+        this.app = app;
+        
+        // table
+        this.table = new Table({
+            content     : 'Add charts to this table.',
+            ondblclick  : function(chart_id) {
+                // get chart
+                var chart = self.app.charts.get(chart_id);
+                self.app.chart.copy(chart);
+                               
+                // show edit
+                self.$el.hide();
+                
+                // update model and show create
+                self.app.chart_view.$el.show();
+            },
+            onchange : function(chart_id) {
+                self.app.viewport_view.show(chart_id);
+            }
+        });
+        
+        // add table to portlet
+        var self = this;
+        this.portlet = new Portlet({
+            icon : 'fa-list',
+            label : 'List of created charts:',
+            height : 100,
+            operations : {
+                'new'   : new Ui.ButtonIcon({
+                            icon : 'fa-plus',
+                            tooltip: 'Create',
+                            onclick: function() {
+                                self.$el.hide();
+                                self.app.chart.reset();
+                                self.app.chart_view.$el.show();
+                            }
+                        }),
+                'delete' : new Ui.ButtonIcon({
+                    icon : 'fa-minus',
+                    tooltip: 'Delete',
+                    onclick: function() {
+                            
+                            // check if element has been selected
+                            var chart_id = self.table.value();
+                            if (!chart_id) {
+                                return;
+                            }
+                    
+                            // title
+                            var chart = self.app.charts.get(chart_id);
+                    
+                            // show modal
+                            self.app.modal.show({
+                            title   : 'Are you sure?',
+                            body    : 'The selected chart "' + chart.get('title') + '" will be irreversibly deleted.',
+                            buttons : {
+                                'Cancel'    : function() {self.app.modal.hide();},
+                                'Delete'    : function() {
+                                    // hide modal
+                                    self.app.modal.hide();
+                                    
+                                    // remove chart
+                                    self.app.charts.remove(chart_id);
+                                }
+                            }
+                        });
+                    }
+                }),
+            }
+        });
+        this.portlet.append(this.table.$el);
+        
+        // append to main
+        if (!this.app.options.config.widget) {
+            this.$el.append(this.portlet.$el);
+        }
+        
+        // events
+        var self = this;
+        this.app.charts.on('add', function(chart) {
+            // add
+            self._addChart(chart);
+        });
+        this.app.charts.on('remove', function(chart) {
+            // remove
+            self._removeChart(chart);
+        });
+        this.app.charts.on('change', function(chart) {
+            // replace
+            self._removeChart(chart);
+            self._addChart(chart);
+        });
+    },
+    
+    // append
+    append : function($el) {
+        this.$el.append(Utils.wrap(''));
+        this.$el.append($el);
+    },
+    
+    // add
+    _addChart: function(chart) {
+        // add title to table
+        this.table.add(chart.get('title'));
+        
+        // get chart type
+        var type = this.app.types.get(chart.get('type'));
+        this.table.add(type.title);
+        
+        // add additional columns
+        this.table.add('Last change: ' + chart.get('date'));
+        this.table.prepend(chart.get('id'));
+        this.table.value(chart.get('id'));
+    },
+    
+    // remove
+    _removeChart: function(chart) {
+        // remove from to table
+        this.table.remove(chart.id);
+    }
+});
+
+});

config/plugins/visualizations/charts/static/views/group.js

@@ -0,0 +1,219 @@
+// dependencies
+define(['library/portlet', 'library/table', 'library/ui', 'library/utils'],
+        function(Portlet, Table, Ui, Utils) {
+
+// chart config
+return Backbone.View.extend(
+{
+    // columns
+    columns: [],
+    
+    // initialize
+    initialize: function(app, options) {
+        // link app
+        this.app = app;
+    
+        // get current chart object
+        this.chart = this.app.chart;
+        this.group = this.app.group;
+        
+        // ui elements
+        this.message = new Ui.Message();
+        this.label = new Ui.Input({placeholder: 'Group label'});
+        this.table = new Table({content: 'No data column.'});
+        
+        // add table to portlet
+        var self = this;
+        this.portlet = new Portlet({
+            icon : 'fa-edit',
+            label : 'Define group properties:',
+            operations : {
+                'save'      : new Ui.ButtonIcon({
+                                icon    : 'fa-save',
+                                tooltip : 'Save',
+                                onclick : function() {
+                                    // save/add group
+                                    self._saveGroup();
+                                }
+                            }),
+                'back'      : new Ui.ButtonIcon({
+                                icon    : 'fa-caret-left',
+                                tooltip : 'Return',
+                                onclick : function() {
+                                    self.$el.hide();
+                                    self.app.chart_view.$el.show();
+                                }
+                            })
+            }
+        });
+        this.portlet.append(this.message.$el);
+        this.portlet.append(this.label.$el);
+        this.portlet.append(this.table.$el);
+        
+        // add element
+        this.setElement(this.portlet.$el);
+        
+        // change
+        var self = this;
+        this.chart.on('change:dataset_id', function() {
+            self._refreshDataset();
+        });
+        this.chart.on('change:type', function() {
+            self._refreshType();
+        });
+        this.group.on('change:label', function() {
+            self._refreshLabel();
+        });
+        this.group.on('change', function() {
+            self._refreshGroup();
+        });
+        this.group.on('reset', function() {
+            self._resetGroup();
+        });
+    },
+    
+    // show
+    show: function() {
+        this.$el.show();
+    },
+    
+    // reset
+    _resetGroup: function() {
+        this.group.set('id', Utils.uuid());
+        this.group.set('label', 'Group label');
+    },
+    
+    // update dataset
+    _refreshDataset: function() {
+        // identify datasets
+        var dataset_id = this.chart.get('dataset_id');
+        
+        // check if dataset is available
+        if (!dataset_id) {
+            return;
+        }
+        
+        // get dataset
+        var dataset = this.app.datasets.get({id : dataset_id});
+        
+        // check
+        if (!dataset) {
+            this.app.log('Config::render()', 'Failed to retrieve dataset.');
+            return;
+        }
+        
+        // configure columns
+        this.columns = [];
+        var meta = dataset.metadata_column_types;
+        for (var key in meta){
+            this.columns.push({
+                'label' : key + ' [' + meta[key] + ']',
+                'value' : key
+            });
+        }
+        
+        // update select fields
+        for (var key in this.list) {
+            this.list[key].update(this.columns);
+        }
+    },
+    
+    // update
+    _refreshType: function() {
+        // configure chart type
+        var self = this;
+        var chart_type = this.chart.get('type');
+        if (chart_type) {
+            var chart_settings = this.app.types.get(chart_type);
+        
+            // table
+            this.table.removeAll();
+            this.list = {};
+            for (var id in chart_settings.data) {
+                // create select field
+                var data_def = chart_settings.data[id];
+                var select = new Ui.Select({
+                    id   : 'select_' + id,
+                    gid  : id,
+                    data : this.columns,
+                    onchange : function(value) {
+                        self.group.set(this.gid, value);
+                    }
+                });
+                
+                // add row to table
+                this.table.add(data_def.title);
+                this.table.add(select.$el);
+                this.table.append(id);
+                
+                // add select field to list
+                this.list[id] = select;
+            }
+        }
+    },
+    
+    // update
+    _refreshGroup: function() {
+        // update select fields
+        for (var id in this.list) {
+            var col = this.group.get(id);
+            if (col === undefined) {
+                col = 0;
+            }
+            this.list[id].value(col);
+        }
+    },
+    
+    // update label
+    _refreshLabel: function() {
+        var label_text = this.group.get('label');
+        if (label_text === undefined) {
+            label_text = '';
+        }
+        this.label.value(label_text);
+    },
+    
+    // create group
+    _saveGroup: function() {
+        // get current chart
+        var chart = this.chart;
+        
+        // update group object
+        var group = this.group;
+        for (var key in this.list) {
+            group.set(key, this.list[key].value());
+        }
+        
+        // add label
+        group.set({
+            dataset_id  : this.chart.get('dataset_id'),
+            label       : this.label.value(),
+            date        : Utils.time()
+        });
+        
+        // validate
+        if (!group.get('label')) {
+            this.message.update({message : 'Please enter a label for your group.'});
+            return;
+        }
+        
+        // get groups of current chart
+        var groups = this.chart.groups;
+        
+        // create/update group
+        var group_update = groups.get(group.id);
+        if (group_update) {
+            group_update.set(group.attributes);
+        } else {
+            groups.add(group.clone());
+        }
+        
+        // hide
+        this.$el.hide();
+        
+        // update main
+        this.app.chart_view.$el.show();
+    }
+});
+
+});

config/plugins/visualizations/charts/static/views/groups.js

@@ -0,0 +1,154 @@
+// dependencies
+define(['library/portlet', 'library/table', 'library/ui', 'library/utils'], function(Portlet, Table, Ui, Utils) {
+
+// chart config
+return Backbone.View.extend(
+{
+    // initialize
+    initialize: function(app, options) {
+        // link app
+        this.app = app;
+    
+        // get current chart object
+        this.chart = this.app.chart;
+        
+        // table
+        this.table = new Table({
+            content     : 'Add data columns to this table.',
+            ondblclick  : function(group_id) {
+                // get group
+                var group = self.app.chart.groups.get(group_id);
+                
+                // show edit
+                self.app.chart_view.$el.hide();
+                self.app.group_view.show();
+                self.app.group_view.group.set(group.attributes);
+            }
+        });
+        
+        // add table to portlet
+        var self = this;
+        this.portlet = new Portlet({
+            icon : '',
+            label : 'Select data columns:',
+            height : 100,
+            operations : {
+                'new'   : new Ui.ButtonIcon({
+                            icon : 'fa-plus',
+                            tooltip: 'Create',
+                            onclick: function() {
+                                self.app.group.reset();
+                                self.app.chart_view.$el.hide();
+                                self.app.group_view.show();
+                            }
+                        }),
+                'edit'  : new Ui.ButtonIcon({
+                            icon : 'fa-pencil',
+                            tooltip: 'Edit',
+                            onclick: function() {
+                                // check if element has been selected
+                                var group_id = self.table.value();
+                                if (!group_id) {
+                                    return;
+                                }
+                                
+                                // get group
+                                var group = self.app.chart.groups.get(group_id);
+                                
+                                // show edit
+                                self.app.chart_view.$el.hide();
+                                self.app.group_view.show();
+                                self.app.group_view.group.set(group.attributes);
+                            }
+                        }),
+                'delete' : new Ui.ButtonIcon({
+                            icon : 'fa-minus',
+                            tooltip: 'Delete',
+                            onclick: function() {
+                                // check if element has been selected
+                                var id = self.table.value();
+                                if (!id) {
+                                    return;
+                                }
+                                // remove group from chart
+                                self.chart.groups.remove(id);
+                            }
+                        })
+            }
+        });
+        this.portlet.append(this.table.$el);
+        
+        // add element
+        this.setElement(this.portlet.$el);
+        
+        // change
+        var self = this;
+        this.chart.on('change', function() {
+            self._refresh();
+        });
+        this.chart.on('reset', function() {
+            self._removeGroupAll();
+        });
+        this.chart.on('change:type', function() {
+            self.app.group_view.trigger('change');
+        });
+        this.chart.groups.on('add', function(group) {
+            self._addGroup(group);
+        });
+        this.chart.groups.on('remove', function(group) {
+            self._removeGroup(group);
+        });
+        this.chart.groups.on('reset', function(group) {
+            self._removeGroupAll();
+        });
+        this.chart.groups.on('change', function(group) {
+            self._removeGroup(group);
+            self._addGroup(group);
+        });
+    },
+    
+    // refresh
+    _refresh: function() {
+        this._removeGroupAll();
+        var self = this;
+        var groups = this.chart.groups;
+        if (groups) {
+            groups.each(function(group) { self._addGroup(group); });
+        }
+    },
+    
+    // add
+    _addGroup: function(group) {
+        // make custom info string
+        var info = '[';
+        var chart_type = this.chart.get('type');
+        if (chart_type) {
+            var chart_settings = this.app.types.get(chart_type);
+            for (var key in chart_settings.data) {
+                info += key + '=' + group.get(key) + ', '
+            }
+        }
+        info = info.substring(0, info.length - 2) + ']';
+        
+        // add to table
+        this.table.add(group.get('label'));
+        this.table.add(info);
+        this.table.add('Last changed: ' + group.get('date'));
+        this.table.prepend(group.id);
+        this.table.value(group.id);
+    },
+    
+    // remove
+    _removeGroup: function(group) {
+        // remove from to table
+        this.table.remove(group.id);
+    },
+    
+    // data config reset
+    _removeGroupAll: function() {
+        // reset options table
+        this.table.removeAll();
+    }
+});
+
+});

config/plugins/visualizations/charts/static/views/viewport.js

@@ -0,0 +1,231 @@
+// dependencies
+define(['library/portlet', 'library/ui', 'library/utils'],
+        function(Portlet, Ui, Utils) {
+
+// widget
+return Backbone.View.extend(
+{
+    // list
+    list: {},
+    
+    // options
+    optionsDefault :
+    {
+        height : 300
+    },
+    
+    // initialize
+    initialize: function(app, options)
+    {
+        // link app
+        this.app = app;
+        
+        // link options
+        this.options = Utils.merge(options, this.optionsDefault);
+        
+        // add table to portlet
+        this.portlet = new Portlet({
+            label       : '',
+            icon        : 'fa-signal',
+            height      : this.options.height,
+            overflow    : 'hidden',
+            operations  : {
+                'edit'  : new Ui.ButtonIcon({
+                    icon    : 'fa-gear',
+                    tooltip : 'Configure'
+                })
+            }
+        });
+        
+        // set this element
+        this.setElement(this.portlet.$el);
+        
+        // events
+        var self = this;
+        
+        // add
+        this.app.charts.on('add', function(chart) {
+            self._addChart(chart);
+        });
+        
+        // remove
+        this.app.charts.on('remove', function(chart) {
+            self._removeChart(chart.id);
+        });
+        
+        // replace
+        this.app.charts.on('change', function(chart) {
+            self._removeChart(chart.id);
+            self._addChart(chart);
+        });
+    },
+    
+    // show
+    show: function(chart_id) {
+        // hide all
+        this.$el.find('svg').hide();
+        
+        // identify selected item from list
+        var item = this.list[chart_id];
+        if (item) {
+            // show selected chart
+            this.$el.find(item.svg_id).show();
+        
+            // get chart
+            var chart = self.app.charts.get(chart_id);
+                
+            // update portlet
+            this.portlet.label(chart.get('title'));
+            this.portlet.setOperation('edit', function() {
+                // get chart
+                self.app.chart.copy(chart);
+                
+                // show edit
+                self.app.charts_view.$el.hide();
+                self.app.chart_view.$el.show();
+            });
+        
+            // this trigger d3 update events
+            $(window).trigger('resize');
+        }
+    },
+    
+    // add
+    _addChart: function(chart) {
+        // link this
+        var self = this;
+        
+        // backup chart details
+        var chart_id = chart.id;
+    
+        // make sure that svg does not exist already
+        this._removeChart(chart_id);
+            
+        // create id
+        var svg_id = '#svg_' + chart_id;
+        
+        // create element
+        var chart_el = $(this._template({id: svg_id, height : this.options.height}));
+        
+        // add to portlet
+        this.portlet.append(chart_el);
+        
+        // backup id
+        this.list[chart_id] = {
+            svg_id : svg_id
+        }
+        
+        // identify chart type
+        var chart_type = chart.get('type');
+        var chart_settings = this.app.types.get(chart_type);
+            
+        // create chart view
+        var self = this;
+        require(['charts/' + chart_type], function(ChartView) {
+            // create chart
+            var view = new ChartView(self.app, {svg_id : svg_id, chart : chart});
+            
+            // reset chart data
+            var chart_data = [];
+            
+            // request data
+            var chart_index = 0;
+            chart.groups.each(function(group) {
+
+                // add group data
+                chart_data.push({
+                    key     : (chart_index + 1) + ':' + group.get('label'),
+                    values  : []
+                });
+                    
+                // get data
+                for (var key in chart_settings.data) {
+                    // configure request
+                    var data_options = {
+                        view        : view,
+                        data        : chart_data,
+                        index       : chart_index,
+                        dataset_id  : group.get('dataset_id'),
+                        column      : group.get(key),
+                        column_key  : key
+                    }
+                    
+                    // make request
+                    self._data(data_options);
+                }
+        
+                // count
+                chart_index++;
+            });
+            
+            // show
+            self.show(chart_id);
+        });
+    },
+    
+    // data
+    _data: function(options) {
+        // gather objects
+        var view        = options.view;
+        var data        = options.data;
+        
+        // gather parameters
+        var index       = options.index;
+        var column      = options.column;
+        var dataset_id  = options.dataset_id;
+        var column_key  = options.column_key;
+        var group_data  = data[options.index];
+        
+        // send request
+        self.app.datasets.get({id : dataset_id, column: column}, function(dataset) {
+            // select column
+            var values = dataset.values[column];
+            var n_values = values.length;
+            
+            // read column values
+            var column_values = [];
+            for (var i = 0; i < n_values; i++) {
+                var value = values[i];
+                if(!isNaN(value)) {
+                    column_values.push(value);
+                } else {
+                    column_values.push(0)
+                }
+            }
+             
+            // write column values
+            for (var i = 0; i < n_values; i++) {
+                // make sure dictionary exists
+                if (group_data.values[i] === undefined) {
+                    group_data.values[i] = {
+                        x : i,
+                        y : 0
+                    }
+                }
+                    
+                // write data
+                group_data.values[i][column_key] = column_values[i];
+            }
+            
+            // refresh view
+            view.refresh(data);
+        });
+
+    },
+    
+    // remove
+    _removeChart: function(id) {
+        var item = this.list[id];
+        if (item) {
+            d3.select(item.svg_id).remove();
+            $(item.svg_id).remove();
+        }
+    },
+    
+    // template
+    _template: function(options) {
+        return '<svg id="' + options.id.substr(1) + '"style="height: ' + options.height + 'px;"></svg>';
+    }
+});
+
+});

config/plugins/visualizations/charts/templates/charts.mako

@@ -0,0 +1,78 @@
+<%
+    scripts = [	'jquery/jquery.js',
+    			'bootstrap.js',
+    			'require.js',
+    			'underscore.js',
+    			'backbone/backbone.js',
+                'd3.js']
+%>
+
+
+<!DOCTYPE HTML>
+<html>
+    <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+        <title>${hda.name} | ${visualization_name}</title>
+
+		%for v in scripts:
+	        <script type="text/javascript" charset="utf-8" src="/static/scripts/libs/${v}" ></script>
+	    %endfor
+
+        ## css
+        <link type="text/css" rel="Stylesheet" media="screen" href="/static/style/base.css">
+        
+        ## install nv.d3 module
+        <script type="text/javascript" charset="utf-8" src="/plugins/visualizations/charts/static/plugins/nv.d3.js" ></script>
+        <link type="text/css" rel="Stylesheet" media="screen" href="/plugins/visualizations/charts/static/plugins/nv.d3.css">
+
+    </head>
+
+    <body>
+        <script type="text/javascript">
+
+            // get configuration
+            var config = {
+                root : '/'
+            };
+            
+            // console protection
+            window.console = window.console || {
+                log     : function(){},
+                debug   : function(){},
+                info    : function(){},
+                warn    : function(){},
+                error   : function(){},
+                assert  : function(){}
+            };
+
+            // configure require
+            require.config({
+                baseUrl: config.root + "plugins/visualizations/charts/static/",
+                shim: {
+                    "libs/underscore": { exports: "_" },
+                    "libs/backbone/backbone": { exports: "Backbone" }
+                }
+            });
+
+            // application
+            var app = null;
+            $(function() {   
+                // request application script
+                require(['app'], function(App) {
+                    // load options
+                    var options = {
+                        config  : ${h.to_json_string( config )},
+                        dataset : ${h.to_json_string( trans.security.encode_dict_ids( hda.to_dict() ) )}
+                    }
+
+                    // create application
+                    app = new App(options);
+                    
+                    // add to body
+                    $('body').append(app.$el);
+                });
+            });
+
+        </script>
+    </body>
+</html>

config/plugins/visualizations/circster/config/circster.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE visualization SYSTEM "../../visualization.dtd">
-<visualization name="circster">
+<visualization name="Circster" disabled="true">
     <data_sources>
         <data_source>
             <model_class>HistoryDatasetAssociation</model_class>
@@ -24,5 +24,5 @@
     <!-- template_root and template are currently ignored for the 'built-in' visualizations -->
     <template_root>webapps/galaxy/visualization</template_root>
     <template>circster.mako</template>
-    <render_location>_top</render_location>
+    <render_target>_top</render_target>
 </visualization>

config/plugins/visualizations/common/templates/README.txt

@@ -0,0 +1,8 @@
+Common templates for visualization plugins
+==========================================
+
+Placing Mako templates in this directory will allow them to be properly
+inherited and imported in plugin directories. E.g. if you have a template file
+in this directory named 'config_utils.mako', you can import it in your plugin
+templates using:
+    <%namespace name="config_utils" file="config_utils.mako" />

config/plugins/visualizations/common/templates/config_utils.mako

@@ -0,0 +1,46 @@
+<%def name="add_config_defaults( defaults )">
+## overwrite default_config_dict with config (if any) then assign to config
+<%
+    for key, default in defaults.items():
+        if key not in config or config[ key ] is None:
+            config[ key ] = default
+%>
+</%def>
+
+<%def name="config_form( config_dict )">
+## render form for everything in possible config
+</%def>
+
+<%def name="link_to_change_config( link_contents, new_settings, target='' )">
+<%
+    # assumes there's a config var
+    url_for_args = {
+        'controller'            : 'visualization',
+        'action'                : 'render',
+        'visualization_name'    : visualization_name,
+        'title'                 : title
+    }
+    url_for_args.update( config )
+    url_for_args.update( new_settings )
+    if visualization_id:
+        url_for_args[ 'id' ] = visualization_id
+%>
+    <a href="${h.url_for( **url_for_args )}" target="${target}">${link_contents}</a>
+</%def>
+
+<%def name="save_button( text='Save' )">
+<%
+    # still a GET
+    url_for_args = {
+        'controller'    : 'visualization',
+        'action'        : 'saved',
+        'type'          : visualization_name,
+        'title'         : title,
+        'config'        : h.to_json_string( config )
+    }
+    # save to existing visualization
+    if visualization_id:
+        url_for_args[ 'id' ] = visualization_id
+%>
+    <form action="${h.url_for( **url_for_args )}" method="post"><input type="submit" value="${text}" /></form>
+</%def>

config/plugins/visualizations/phyloviz/config/phyloviz.xml

@@ -1,11 +1,12 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE visualization SYSTEM "../../visualization.dtd">
-<visualization name="phyloviz">
+<visualization name="Phyloviz">
     <data_sources>
         <data_source>
             <model_class>HistoryDatasetAssociation</model_class>
             <test type="isinstance" test_attr="datatype" result_type="datatype">data.Newick</test>
             <test type="isinstance" test_attr="datatype" result_type="datatype">data.Nexus</test>
+            <test type="isinstance" test_attr="datatype" result_type="datatype">xml.Phyloxml</test>
             <to_param param_attr="id">dataset_id</to_param>
         </data_source>
     </data_sources>
@@ -16,5 +17,5 @@
     <!-- template_root and template are currently ignored for the 'built-in' visualizations -->
     <template_root>webapps/galaxy/visualization</template_root>
     <template>phyloviz.mako</template>
-    <render_location>_top</render_location>
+    <render_target>_top</render_target>
 </visualization>

config/plugins/visualizations/scatterplot/Gruntfile.js

@@ -6,9 +6,10 @@ module.exports = function(grunt) {
         pkg: grunt.file.readJSON( 'package.json' ),
 
         handlebars: {
+            // compile all hb templates into a single file in the build dir
             compile: {
                 options: {
-                    namespace: 'Templates',
+                    namespace: 'scatterplot',
                     processName : function( filepath ){
                         return filepath.match( /\w*\.handlebars/ )[0].replace( '.handlebars', '' );
                     }
@@ -20,6 +21,7 @@ module.exports = function(grunt) {
         },
 
         concat: {
+            // concat the template file and any js files in the src dir into a single file in the build dir
             options: {
                 separator: ';\n'
             },
@@ -31,16 +33,19 @@ module.exports = function(grunt) {
         },
 
         uglify: {
+            // uglify the concat single file directly into the static dir
             options: {
+                //mangle      : false,
+                //beautify    : true
             },
             dist: {
                 src : 'build/scatterplot-concat.js',
-                // uglify directly into static dir
-                dest: 'static/scatterplot.js'
+                dest: 'static/scatterplot-edit.js'
             }
         },
 
         watch: {
+            // watch for changes in the src dir
             files: [ 'src/**.js', 'src/handlebars/*.handlebars' ],
             tasks: [ 'default' ]
         }
@@ -52,5 +57,6 @@ module.exports = function(grunt) {
     grunt.loadNpmTasks( 'grunt-contrib-watch' );
 
     grunt.registerTask( 'default', [ 'handlebars', 'concat', 'uglify' ]);
-    grunt.registerTask( 'watch',   [ 'handlebars', 'concat', 'uglify', 'watch' ]);
+    // you can run grunt watch directly:
+    //  grunt watch
 };

config/plugins/visualizations/scatterplot/config/scatterplot.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE visualization SYSTEM "../../visualization.dtd">
-<visualization name="scatterplot">
+<visualization name="Scatterplot">
     <data_sources>
         <data_source>
             <model_class>HistoryDatasetAssociation</model_class>

config/plugins/visualizations/scatterplot/package.json

@@ -19,6 +19,6 @@
     "grunt-contrib-handlebars": "~0.5.10",
     "grunt-contrib-concat": "~0.3.0",
     "grunt-contrib-uglify": "~0.2.2",
-    "grunt-contrib-watch": "~0.5.1"
+    "grunt-contrib-watch": "~0.5.3"
   }
 }

config/plugins/visualizations/scatterplot/src/handlebars/chartControl.handlebars

@@ -1,56 +0,0 @@
-<p class="help-text">
-        Use the following controls to how the chart is displayed.
-        The slide controls can be moved by the mouse or, if the 'handle' is in focus, your keyboard's arrow keys.
-        Move the focus between controls by using the tab or shift+tab keys on your keyboard.
-        Use the 'Draw' button to render (or re-render) the chart with the current settings.
-    </p>
-
-    <div id="datapointSize" class="form-input numeric-slider-input">
-        <label for="datapointSize">Size of data point: </label>
-        <div class="slider-output">{{datapointSize}}</div>
-        <div class="slider"></div>
-        <p class="form-help help-text-small">
-            Size of the graphic representation of each data point
-        </p>
-    </div>
-
-    <div id="animDuration" class="form-input checkbox-input">
-        <label for="animate-chart">Animate chart transitions?: </label>
-        <input type="checkbox" id="animate-chart"
-            class="checkbox control"{{#if animDuration}} checked="true"{{/if}} />
-        <p class="form-help help-text-small">
-            Uncheck this to disable the animations used on the chart
-        </p>
-    </div>
-
-    <div id="width" class="form-input numeric-slider-input">
-        <label for="width">Chart width: </label>
-        <div class="slider-output">{{width}}</div>
-        <div class="slider"></div>
-        <p class="form-help help-text-small">
-            (not including chart margins and axes)
-        </p>
-    </div>
-
-    <div id="height" class="form-input numeric-slider-input">
-        <label for="height">Chart height: </label>
-        <div class="slider-output">{{height}}</div>
-        <div class="slider"></div>
-        <p class="form-help help-text-small">
-            (not including chart margins and axes)
-        </p>
-    </div>
-
-    <div id="X-axis-label"class="text-input form-input">
-        <label for="X-axis-label">Re-label the X axis: </label>
-        <input type="text" name="X-axis-label" id="X-axis-label" value="{{xLabel}}" />
-        <p class="form-help help-text-small"></p>
-    </div>
-
-    <div id="Y-axis-label" class="text-input form-input">
-        <label for="Y-axis-label">Re-label the Y axis: </label>
-        <input type="text" name="Y-axis-label" id="Y-axis-label" value="{{yLabel}}" />
-        <p class="form-help help-text-small"></p>
-    </div>
-
-    <input id="render-button" type="button" value="Draw" />

config/plugins/visualizations/scatterplot/src/handlebars/chartDisplay.handlebars

@@ -1 +0,0 @@
-<svg width="{{width}}" height="{{height}}"></svg>

config/plugins/visualizations/scatterplot/src/handlebars/chartcontrol.handlebars

@@ -0,0 +1,47 @@
+<p class="help-text">
+    Use the following controls to how the chart is displayed.
+    The slide controls can be moved by the mouse or, if the 'handle' is in focus, your keyboard's arrow keys.
+    Move the focus between controls by using the tab or shift+tab keys on your keyboard.
+    Use the 'Draw' button to render (or re-render) the chart with the current settings.
+</p>
+
+<div data-config-key="datapointSize" class="form-input numeric-slider-input">
+    <label for="datapointSize">Size of data point: </label>
+    <div class="slider-output">{{datapointSize}}</div>
+    <div class="slider"></div>
+    <p class="form-help help-text-small">
+        Size of the graphic representation of each data point
+    </p>
+</div>
+
+<div data-config-key="width" class="form-input numeric-slider-input">
+    <label for="width">Chart width: </label>
+    <div class="slider-output">{{width}}</div>
+    <div class="slider"></div>
+    <p class="form-help help-text-small">
+        (not including chart margins and axes)
+    </p>
+</div>
+
+<div data-config-key="height" class="form-input numeric-slider-input">
+    <label for="height">Chart height: </label>
+    <div class="slider-output">{{height}}</div>
+    <div class="slider"></div>
+    <p class="form-help help-text-small">
+        (not including chart margins and axes)
+    </p>
+</div>
+
+<div data-config-key="X-axis-label"class="text-input form-input">
+    <label for="X-axis-label">Re-label the X axis: </label>
+    <input type="text" name="X-axis-label" id="X-axis-label" value="{{x.label}}" />
+    <p class="form-help help-text-small"></p>
+</div>
+
+<div data-config-key="Y-axis-label" class="text-input form-input">
+    <label for="Y-axis-label">Re-label the Y axis: </label>
+    <input type="text" name="Y-axis-label" id="Y-axis-label" value="{{y.label}}" />
+    <p class="form-help help-text-small"></p>
+</div>
+
+<button class="render-button btn btn-primary active">Draw</button>

config/plugins/visualizations/scatterplot/src/handlebars/dataControl.handlebars

@@ -1,56 +0,0 @@
-<p class="help-text">
-        Use the following controls to change the data used by the chart.
-        Use the 'Draw' button to render (or re-render) the chart with the current settings.
-    </p>
-
-    {{! column selector containers }}
-    <div class="column-select">
-        <label for="X-select">Data column for X: </label>
-        <select name="X" id="X-select">
-        {{#each numericColumns}}
-            <option value="{{index}}">{{name}}</option>
-        {{/each}}
-        </select>
-    </div>
-    <div class="column-select">
-        <label for="Y-select">Data column for Y: </label>
-        <select name="Y" id="Y-select">
-        {{#each numericColumns}}
-            <option value="{{index}}">{{name}}</option>
-        {{/each}}
-        </select>
-    </div>
-
-    {{! optional id column }}
-    <div id="include-id">
-        <label for="include-id-checkbox">Include a third column as data point IDs?</label>
-        <input type="checkbox" name="include-id" id="include-id-checkbox" />
-        <p class="help-text-small">
-            These will be displayed (along with the x and y values) when you hover over
-            a data point.
-        </p>
-    </div>
-    <div class="column-select" style="display: none">
-        <label for="ID-select">Data column for IDs: </label>
-        <select name="ID" id="ID-select">
-        {{#each allColumns}}
-            <option value="{{index}}">{{name}}</option>
-        {{/each}}
-        </select>
-    </div>
-
-    {{! if we're using generic column selection names ('column 1') - allow the user to use the first line }}
-    <div id="first-line-header" style="display: none;">
-        <p>Possible headers: {{ possibleHeaders }}
-        </p>
-        <label for="first-line-header-checkbox">Use the above as column headers?</label>
-        <input type="checkbox" name="include-id" id="first-line-header-checkbox"
-            {{#if usePossibleHeaders }}checked="true"{{/if}}/>
-        <p class="help-text-small">
-            It looks like Galaxy couldn't get proper column headers for this data.
-            Would you like to use the column headers above as column names to select columns?
-        </p>
-    </div>
-
-    <input id="render-button" type="button" value="Draw" />
-    <div class="clear"></div>

config/plugins/visualizations/scatterplot/src/handlebars/datacontrol.handlebars

@@ -0,0 +1,55 @@
+<p class="help-text">
+    Use the following controls to change the data used by the chart.
+    Use the 'Draw' button to render (or re-render) the chart with the current settings.
+</p>
+
+{{! column selector containers }}
+<div class="column-select">
+    <label>Data column for X: </label>
+    <select name="xColumn">
+    {{#each numericColumns}}
+        <option value="{{index}}">{{name}}</option>
+    {{/each}}
+    </select>
+</div>
+<div class="column-select">
+    <label>Data column for Y: </label>
+    <select name="yColumn">
+    {{#each numericColumns}}
+        <option value="{{index}}">{{name}}</option>
+    {{/each}}
+    </select>
+</div>
+
+{{! optional id column }}
+<div id="include-id">
+    <label for="include-id-checkbox">Include a third column as data point IDs?</label>
+    <input type="checkbox" name="include-id" id="include-id-checkbox" />
+    <p class="help-text-small">
+        These will be displayed (along with the x and y values) when you hover over
+        a data point.
+    </p>
+</div>
+<div class="column-select" style="display: none">
+    <label for="ID-select">Data column for IDs: </label>
+    <select name="idColumn">
+    {{#each allColumns}}
+        <option value="{{index}}">{{name}}</option>
+    {{/each}}
+    </select>
+</div>
+
+{{! if we're using generic column selection names ('column 1') - allow the user to use the first line }}
+<div id="first-line-header" style="display: none;">
+    <p>Possible headers: {{ possibleHeaders }}
+    </p>
+    <label for="first-line-header-checkbox">Use the above as column headers?</label>
+    <input type="checkbox" name="include-id" id="first-line-header-checkbox"
+        {{#if usePossibleHeaders }}checked="true"{{/if}}/>
+    <p class="help-text-small">
+        It looks like Galaxy couldn't get proper column headers for this data.
+        Would you like to use the column headers above as column names to select columns?
+    </p>
+</div>
+
+<button class="render-button btn btn-primary active">Draw</button>

config/plugins/visualizations/scatterplot/src/handlebars/editor.handlebars

@@ -0,0 +1,42 @@
+<div class="scatterplot-editor tabbable tabs-left">
+    {{! tab buttons/headers using Bootstrap }}
+    <ul class="nav nav-tabs">
+        {{! start with the data controls as the displayed tab }}
+        <li class="active">
+            <a title="Use this tab to change which data are used"
+               href="#data-control" data-toggle="tab">Data Controls</a>
+        </li>
+        <li>
+            <a title="Use this tab to change how the chart is drawn"
+               href="#chart-control" data-toggle="tab" >Chart Controls</a>
+        </li>
+        {{! chart starts as disabled since there's no info yet }}
+        <li class="disabled">
+            <a title="This tab will display the chart"
+               href="#chart-display" data-toggle="tab">Chart</a>
+        </li>
+        {{! ... }}
+        <li class="file-controls">
+<!--            <button class="copy-btn btn btn-default"
+                    title="Save this as a new visualization">Save to new</button>-->
+            <button class="save-btn btn btn-default">Save</button>
+        </li>
+    </ul>
+
+    {{! data form, chart config form, chart all get their own tab }}
+    <div class="tab-content">
+        {{! ---------------------------- tab for data settings form }}
+        <div id="data-control" class="scatterplot-config-control tab-pane active">
+            {{! rendered separately }}
+        </div>
+    
+        {{! ---------------------------- tab for chart graphics control form }}
+        <div id="chart-control" class="scatterplot-config-control tab-pane">
+            {{! rendered separately }}
+        </div>
+
+        {{! ---------------------------- tab for actual chart }}
+        <div id="chart-display" class="scatterplot-display tab-pane"></div>
+
+    </div>{{! end .tab-content }}
+</div>{{! end .chart-control }}

config/plugins/visualizations/scatterplot/src/handlebars/scatterplotControlForm.handlebars

@@ -1,46 +0,0 @@
-{{! main layout }}
-
-<div class="scatterplot-container chart-container tabbable tabs-left">
-    {{! tab buttons/headers using Bootstrap }}
-    <ul class="nav nav-tabs">
-        {{! start with the data controls as the displayed tab }}
-        <li class="active"><a href="#data-control" data-toggle="tab" class="tooltip"
-            title="Use this tab to change which data are used">Data Controls</a></li>
-        <li><a href="#chart-control" data-toggle="tab" class="tooltip"
-            title="Use this tab to change how the chart is drawn">Chart Controls</a></li>
-        <li><a href="#stats-display" data-toggle="tab" class="tooltip"
-            title="This tab will display overall statistics for your data">Statistics</a></li>
-        <li><a href="#chart-display" data-toggle="tab" class="tooltip"
-            title="This tab will display the chart">Chart</a>
-            {{! loading indicator - initially hidden }}
-            <div id="loading-indicator" style="display: none;">
-                <img class="loading-img" src="{{loadingIndicatorImagePath}}" />
-                <span class="loading-message">{{message}}</span>
-            </div>
-        </li>
-    </ul>
-
-    {{! data form, chart config form, stats, and chart all get their own tab }}
-    <div class="tab-content">
-        {{! ---------------------------- tab for data settings form }}
-        <div id="data-control" class="tab-pane active">
-            {{! rendered separately }}
-        </div>
-    
-        {{! ---------------------------- tab for chart graphics control form }}
-        <div id="chart-control" class="tab-pane">
-            {{! rendered separately }}
-        </div>
-
-        {{! ---------------------------- tab for data statistics }}
-        <div id="stats-display" class="tab-pane">
-            {{! rendered separately }}
-        </div>
-
-        {{! ---------------------------- tab for actual chart }}
-        <div id="chart-display" class="tab-pane">
-            {{! chart rendered separately }}
-        </div>
-
-    </div>{{! end .tab-content }}
-</div>{{! end .chart-control }}

config/plugins/visualizations/scatterplot/src/handlebars/statsDisplay.handlebars

@@ -1,8 +0,0 @@
-<p class="help-text">By column:</p>
-    <table id="chart-stats-table">
-        <thead><th></th><th>X</th><th>Y</th></thead>
-        {{#each stats}}
-        <tr><td>{{name}}</td><td>{{xval}}</td><td>{{yval}}</td></tr>
-        </tr>
-        {{/each}}
-    </table>

config/plugins/visualizations/scatterplot/src/scatterplot-config-editor.js

@@ -0,0 +1,276 @@
+/* =============================================================================
+todo:
+    import button(display), func(model) - when user doesn't match
+    Move margins into wid/hi calcs (so final svg dims are w/h)
+    Better separation of AJAX in scatterplot.js (maybe pass in function?)
+    Labels should auto fill in chart control when dataset has column_names
+    Allow column selection/config using the peek output as a base for UI
+    Allow setting perPage in config
+    Allow option to auto set width/height based on screen real estate avail.
+    Handle large number of pages better (Known genes hg19)
+    Use d3.nest to allow grouping, pagination/filtration by group (e.g. chromCol)
+    Semantic HTML (figure, caption)
+    Save as SVG/png
+    Does it work w/ Galaxy.Frame?
+    Embedding
+    Small multiples
+    Drag & Drop other splots onto current (redraw with new axis and differentiate the datasets)
+    Remove 'chart' names
+    Somehow link out from info box?
+    
+    Subclass on specific datatypes? (vcf, cuffdiff, etc.)
+    What can be common/useful to other visualizations?
+
+============================================================================= */
+/**
+ *  Scatterplot config control UI as a backbone view
+ *      handles:
+ *          configuring which data will be used
+ *          configuring the plot display
+ */
+var ScatterplotConfigEditor = Backbone.View.extend( LoggableMixin ).extend({
+    //TODO: !should be a view on a visualization model
+    //logger      : console,
+    className   : 'scatterplot-control-form',
+    
+    /** initialize requires a configuration Object containing a dataset Object */
+    initialize : function( attributes ){
+        if( !this.model ){
+            this.model = new Visualization({ type: 'scatterplot' });
+        }
+        this.log( this + '.initialize, attributes:', attributes );
+
+        if( !attributes || !attributes.dataset ){
+            throw new Error( "ScatterplotConfigEditor requires a dataset" );
+        }
+        this.dataset = attributes.dataset;
+        this.log( 'dataset:', this.dataset );
+
+        this.display = new ScatterplotDisplay({
+            dataset : attributes.dataset,
+            model   : this.model
+        });
+    },
+
+    // ------------------------------------------------------------------------- CONTROLS RENDERING
+    render : function(){
+        //console.log( this + '.render' );
+        // render the tab controls, areas and loading indicator
+        this.$el.empty().append( ScatterplotConfigEditor.templates.mainLayout({}));
+        if( this.model.id ){
+            this.$el.find( '.copy-btn' ).show();
+            this.$el.find( '.save-btn' ).text( 'Update saved' );
+        }
+        this.$el.find( '[title]' ).tooltip();
+
+        // render the tab content
+        this._render_dataControl();
+        this._render_chartControls();
+        this._render_chartDisplay();
+
+        // set up behaviours
+
+        // auto render if given both x, y column choices
+        var config = this.model.get( 'config' );
+        if( this.model.id && _.isFinite( config.xColumn ) && _.isFinite( config.yColumn ) ){
+            this.renderChart();
+        }
+        return this;
+    },
+
+    /** controls for which columns are used to plot datapoints (and ids/additional info to attach if desired) */
+    _render_dataControl : function( $where ){
+        //TODO: better handling of missing column names, column types
+        $where = $where || this.$el;
+        var editor  = this,
+            dataset = this.dataset,
+            column_names = dataset.metadata_column_names || [],
+            config  = this.model.get( 'config' );
+        //console.log( 'metadata_column_types:', this.dataset.metadata_column_types );
+        //console.log( 'metadata_column_names:', this.dataset.metadata_column_names );
+
+//TODO: to peek based control
+        var numericColumns = [],
+            allColumns = _.map( dataset.metadata_column_types, function( type, i ){
+                // save column data for select rendering, adding metadata name if available in dataset
+                var column = { index: i, type: type, name: ( column_names[ i ] || ( 'column ' + ( i + 1 )) ) };
+                // also add column to numerics if numeric type
+                if( ( column.type === 'int' ) || ( column.type === 'float' ) ){
+                    numericColumns.push( column );
+                }
+                return column;
+            });
+        if( numericColumns.length < 2 ){
+            numericColumns = allColumns;
+        }
+        //console.log( 'allColumns:', allColumns );
+        //console.log( 'numericColumns:', numericColumns );
+
+        // render the html
+        var $dataControl = $where.find( '.tab-pane#data-control' );
+        $dataControl.html( ScatterplotConfigEditor.templates.dataControl({
+            allColumns      : allColumns,
+            numericColumns  : numericColumns
+        }));
+
+//TODO: column selection boilerplate
+        // preset to column selectors if they were passed in the config in the query string; set up events
+        var newConfig = {
+            xColumn  : ( _.isFinite( config.xColumn ) )? ( config.xColumn ): ( numericColumns[0].index ),
+            yColumn  : ( _.isFinite( config.yColumn ) )? ( config.yColumn ): ( numericColumns[1].index ),
+            idColumn : allColumns[0].index
+        };
+        // use an idColumn from the config or attempt to get one different from the numeric
+        if( _.isFinite( config.idColumn ) ){
+            newConfig.idColumn = config.idColumn;
+        } else {
+            if( allColumns.length > 2 ){
+                var uniqueCol = _.find( allColumns, function( column, i ){
+                    return i !== newConfig.xColumn && i !== newConfig.yColumn;
+                });
+                newConfig.idColumn = uniqueCol.index;
+            }
+        }
+        config = this.model.set( 'config', newConfig, { silent: true }).get( 'config' );
+
+        $dataControl.find( '[name="xColumn"]' ).val( config.xColumn ).on( 'change', function(){
+            editor.model.set( 'config', { xColumn: Number( $( this ).val() ) });
+        });
+        $dataControl.find( '[name="yColumn"]' ).val( config.yColumn ).on( 'change', function(){
+            editor.model.set( 'config', { yColumn: Number( $( this ).val() ) });
+        });
+        $dataControl.find( 'select[name="idColumn"]' ).val( config.idColumn ).on( 'change', function(){
+            editor.model.set( 'config', { idColumn: Number( $( this ).val() ) });
+        });
+        if( config.idColumn !== undefined ){
+            $dataControl.find( '#include-id-checkbox' ).prop( 'checked', true ).trigger( 'change' );
+        }
+
+        $dataControl.find( '[title]' ).tooltip();
+        return $dataControl;
+    },
+    
+    /** tab content to control how the chart is rendered (data glyph size, chart size, etc.) */
+    _render_chartControls : function( $where ){
+        $where = $where || this.$el;
+        var editor = this,
+            config = this.model.get( 'config' ),
+            $chartControls = $where.find( '#chart-control' );
+
+        // ---- skeleton/form for controls
+        $chartControls.html( ScatterplotConfigEditor.templates.chartControl( config ) );
+        //console.debug( '$chartControl:', $chartControls );
+
+        // ---- slider controls
+        // limits for controls (by control/chartConfig id)
+        //TODO: as class attribute
+        var controlRanges = {
+                'datapointSize' : { min: 2, max: 10, step: 1 },
+                'width'         : { min: 200, max: 800, step: 20 },
+                'height'        : { min: 200, max: 800, step: 20 }
+            };
+
+        function onSliderChange(){
+            // set the model config when changed and update the slider output text
+            var $this = $( this ),
+                //note: returns a number nicely enough
+                newVal = $this.slider( 'value' );
+            // parent of slide event target has html5 attr data-config-key
+            editor.model.set( 'config', _.object([[ $this.parent().data( 'config-key' ), newVal ]]) );
+            $this.siblings( '.slider-output' ).text( newVal );
+        }
+
+        //console.debug( 'numeric sliders:', $chartControls.find( '.numeric-slider-input' ) );
+        $chartControls.find( '.numeric-slider-input' ).each( function(){
+            // set up the slider with control ranges, change event; set output text to initial value
+            var $this = $( this ),
+                configKey = $this.attr( 'data-config-key' ),
+                sliderSettings = _.extend( controlRanges[ configKey ], {
+                    value   : config[ configKey ],
+                    change  : onSliderChange,
+                    slide   : onSliderChange
+                });
+            //console.debug( configKey + ' slider settings:', sliderSettings );
+            $this.find( '.slider' ).slider( sliderSettings );
+            $this.children( '.slider-output' ).text( config[ configKey ] );
+        });
+
+        // ---- axes labels
+        var columnNames = this.dataset.metadata_column_names || [];
+        var xLabel = config.xLabel || columnNames[ config.xColumn ] || 'X';
+        var yLabel = config.yLabel || columnNames[ config.yColumn ] || 'Y';
+        // set label inputs to current x, y metadata_column_names (if any)
+        $chartControls.find( 'input[name="X-axis-label"]' ).val( xLabel )
+            .on( 'change', function(){
+                editor.model.set( 'config', { xLabel: $( this ).val() });
+            });
+        $chartControls.find( 'input[name="Y-axis-label"]' ).val( yLabel )
+            .on( 'change', function(){
+                editor.model.set( 'config', { yLabel: $( this ).val() });
+            });
+
+        //console.debug( '$chartControls:', $chartControls );
+        $chartControls.find( '[title]' ).tooltip();
+        return $chartControls;
+    },
+
+    /** render the tab content where the chart is displayed (but not the chart itself) */
+    _render_chartDisplay : function( $where ){
+        $where = $where || this.$el;
+        var $chartDisplay = $where.find( '.tab-pane#chart-display' );
+        this.display.setElement( $chartDisplay );
+        this.display.render();
+
+        $chartDisplay.find( '[title]' ).tooltip();
+        return $chartDisplay;
+    },
+
+    // ------------------------------------------------------------------------- EVENTS
+    events : {
+        'change #include-id-checkbox'          : 'toggleThirdColumnSelector',
+        'click #data-control .render-button'   : 'renderChart',
+        'click #chart-control .render-button'  : 'renderChart',
+        'click .save-btn'                      : 'saveVisualization',
+        //'click .copy-btn'                       : function(e){ this.model.save(); }
+    },
+
+    saveVisualization : function(){
+        var editor = this;
+        this.model.save()
+            .fail( function( xhr, status, message ){
+                console.error( xhr, status, message );
+                editor.trigger( 'save:error', view );
+                alert( 'Error loading data:\n' + xhr.responseText );
+            })
+            .then( function(){
+                editor.render();
+            });
+    },
+
+    toggleThirdColumnSelector : function(){
+        // show/hide the id selector on the data settings panel
+        this.$el.find( 'select[name="idColumn"]' ).parent().toggle();
+    },
+
+    // ------------------------------------------------------------------------- CHART/STATS RENDERING
+    renderChart : function(){
+        //console.log( this + '.renderChart' );
+        // fetch the data, (re-)render the chart
+        this.$el.find( '.nav li.disabled' ).removeClass( 'disabled' );
+        this.$el.find( 'ul.nav' ).find( 'a[href="#chart-display"]' ).tab( 'show' );
+        this.display.fetchData();
+        //console.debug( this.display.$el );
+    },
+
+    toString : function(){
+        return 'ScatterplotConfigEditor(' + (( this.dataset )?( this.dataset.id ):( '' )) + ')';
+    }
+});
+
+ScatterplotConfigEditor.templates = {
+    mainLayout      : scatterplot.editor,
+    dataControl     : scatterplot.datacontrol,
+    chartControl    : scatterplot.chartcontrol
+};
+
+//==============================================================================

config/plugins/visualizations/scatterplot/src/scatterplot-display.js

@@ -0,0 +1,285 @@
+// =============================================================================
+/**
+ *  Scatterplot display control UI as a backbone view
+ *      handles:
+ *          fetching the data (if needed)
+ *          computing and displaying data stats
+ *          controls for pagination of data (if needed)
+ */
+var ScatterplotDisplay = Backbone.View.extend({
+
+    initialize : function( attributes ){
+        this.data = null,
+        this.dataset = attributes.dataset;
+        this.calcNumPages();
+    },
+
+    calcNumPages : function(){
+        var config = this.model.get( 'config' );
+        this.lineCount = this.dataset.metadata_data_lines,
+        this.numPages = ( this.lineCount )?( Math.ceil( this.lineCount / config.pagination.perPage ) ):( undefined );
+        if( !this.lineCount || this.numPages === undefined ){
+            console.warn( 'no data total found' );
+        }
+    },
+
+    fetchData : function(){
+//TODO: doesn't work bc it's rendered in render()...
+        this.showLoadingIndicator( 'getting data' );
+        //console.debug( 'currPage', this.config.pagination.currPage );
+        var view = this,
+            config = this.model.get( 'config' ),
+//TODO: very tied to datasets - should be generalized eventually
+            xhr = jQuery.getJSON( '/api/datasets/' + this.dataset.id, {
+                data_type   : 'raw_data',
+                provider    : 'dataset-column',
+                limit       : config.pagination.perPage,
+                offset      : ( config.pagination.currPage * config.pagination.perPage )
+            });
+        xhr.done( function( data ){
+            view.data = data.data;
+            view.trigger( 'data:fetched', view );
+            view.renderData();
+        });
+        xhr.fail( function( xhr, status, message ){
+            console.error( xhr, status, message );
+            view.trigger( 'data:error', view );
+            alert( 'Error loading data:\n' + xhr.responseText );
+        });
+        return xhr;
+    },
+
+    showLoadingIndicator : function(){
+        // display the loading indicator over the tab panels if hidden, update message (if passed)
+        this.$el.find( '.scatterplot-data-info' ).html([
+            '<div class="loading-indicator">',
+                '<span class="fa fa-spinner fa-spin"></span>',
+                '<span class="loading-indicator-message">loading...</span>',
+            '</div>'
+        ].join( '' ));
+    },
+
+    template : function(){
+        var html = [
+            '<div class="controls clear">',
+                '<div class="left">',
+                '</div>',
+                '<div class="right">',
+                    '<p class="scatterplot-data-info"></p>',
+                    '<button class="stats-toggle-btn">Stats</button>',
+                    '<button class="rerender-btn">Redraw</button>',
+                '</div>',
+            '</div>',
+            '<svg/>', //TODO: id
+            '<div class="stats-display"></div>'
+        ].join( '' );
+        return html;
+    },
+
+    render : function(){
+        this.$el.addClass( 'scatterplot-display' ).html( this.template() );
+        if( this.data ){
+            this.renderData();
+        }
+        return this;
+    },
+
+    renderData : function(){
+        this.renderLeftControls();
+        this.renderRightControls();
+        this.renderPlot( this.data );
+        this.getStats();
+    },
+
+    renderLeftControls : function(){
+        if( this.lineCount ){
+            this.$el.find( '.controls .left' ).empty().append( this.renderPagination() );
+        } else {
+            this.$el.find( '.controls .left' ).empty().append( this.renderPrevNext() );
+        }
+        return this;
+    },
+
+    renderRightControls : function(){
+        var view = this;
+        this.setLineInfo( this.data );
+        // clear prev. handlers due to closure around data
+        this.$el.find( '.stats-toggle-btn' )
+            .off().click( function(){
+                view.toggleStats();
+            });
+        this.$el.find( '.rerender-btn' )
+            .off().click( function(){
+                view.resetZoom();
+                view.renderPlot( this.data );
+            });
+    },
+
+    /** render and show the d3 plot into the svg node of the view */
+    renderPlot : function(){
+        var view = this,
+            $svg = this.$el.find( 'svg' );
+        // turn off stats, clear previous svg, and make it visible
+        this.toggleStats( false );
+        $svg.off().empty().show()
+            // set up listeners for events from plot
+            .on( 'zoom.scatterplot', function( ev, zoom ){
+                //TODO: possibly throttle this
+                //console.debug( 'zoom.scatterplot', zoom.scale, zoom.translate );
+                view.model.set( 'config', zoom );
+            });
+        //TODO: may not be necessary to off/on this more than the initial on
+        // call the sep. d3 function to generate the plot
+        scatterplot( $svg.get( 0 ), this.model.get( 'config' ), this.data );
+    },
+
+    setLineInfo : function( data, contents ){
+        if( data ){
+            var config = this.model.get( 'config' ),
+                totalLines = this.lineCount || 'an unknown total',
+                lineStart  = config.pagination.currPage * config.pagination.perPage,
+                lineEnd    = lineStart + data.length;
+            this.$el.find( '.controls p.scatterplot-data-info' )
+               .text([ lineStart + 1, 'to', lineEnd, 'of', totalLines ].join( ' ' ));
+        } else {
+            this.$el.find( '.controls p.scatterplot-data-info' ).html( contents || '' );
+        }
+
+        return this;
+    },
+
+    resetZoom : function( scale, translate ){
+        scale = ( scale !== undefined )?( scale ):( 1 );
+        translate = ( translate !== undefined )?( translate ):( [ 0, 0 ] );
+        this.model.set( 'config', { scale: scale, translate: translate } );
+        return this;
+    },
+
+    // ------------------------------------------------------------------------ data pagination
+//TODO: to pagination control
+    goToPage : function( page ){
+        var pagination = this.model.get( 'config' ).pagination;
+        //console.debug( 'goToPage', page, pagination, this.numPages );
+        if( page <= 0 ){ page = 0; }
+        if( this.numPages && page >= this.numPages ){ page = this.numPages - 1; }
+        if( page === pagination.currPage ){ return this; }
+
+        //console.debug( '\t going to page ' + page )
+        pagination.currPage = page;
+        this.model.set( 'config', { pagination: pagination });
+        this.resetZoom();
+        this.fetchData();
+        return this;
+    },
+
+    nextPage : function(){
+        var currPage = this.model.get( 'config' ).pagination.currPage;
+        return this.goToPage( currPage + 1 );
+    },
+
+    prevPage : function(){
+        var currPage = this.model.get( 'config' ).pagination.currPage;
+        return this.goToPage( currPage - 1 );
+    },
+
+    /** render previous and next pagination buttons */
+    renderPrevNext : function(){
+        var config = this.model.get( 'config' );
+        // if there's no data or there's less than one page of data - return null
+        if( !this.data ){ return null; }
+        if( config.pagination.currPage === 0 && this.data.length < config.pagination.perPage ){ return null; }
+
+        var view = this,
+            $prev = $( '<li><a href="javascript:void(0);">Prev</a></li>' )
+                .click( function(){ view.prevPage(); }),
+            $next = $( '<li><a href="javascript:void(0);">Next</a></li>' )
+                .click( function(){ view.nextPage(); });
+
+        // disable if it either end
+        if( config.pagination.currPage === 0 ){
+            $prev.addClass( 'disabled' );
+        }
+        if( this.numPages && config.pagination.currPage === ( this.numPages - 1 ) ){
+            $next.addClass( 'disabled' );
+        }
+        return $( '<ul/>' ).addClass( 'pagination data-prev-next' ).append([ $prev, $next ]);
+    },
+
+    /** render page links for each possible page (if we can) */
+    renderPagination : function(){
+        var config = this.model.get( 'config' );
+        // if there's no data, no page count, or there's less than one page of data - return null
+        if( !this.data ){ return null; }
+        if( !this.numPages ){ return null; }
+        if( config.pagination.currPage === 0 && this.data.length < config.pagination.perPage ){ return null; }
+
+        var view = this,
+            $pagesList = $( '<ul/>' ).addClass( 'pagination data-pages' );
+            pageNumClick = function( ev ){
+                view.goToPage( $( this ).data( 'page' ) );
+            };
+
+        for( var i=0; i<this.numPages; i+=1 ){
+            // add html5 data tag 'page' for later click event handler use
+            var $pageLi = $([ '<li><a href="javascript:void(0);">', i + 1, '</a></li>' ].join( '' ))
+                    .attr( 'data-page', i ).click( pageNumClick );
+            // highlight the current page
+            if( i === config.pagination.currPage ){
+                $pageLi.addClass( 'active' );
+            }
+            $pagesList.append( $pageLi );
+        }
+        return $pagesList;
+    },
+
+    // ------------------------------------------------------------------------ statistics display
+    /** create a webworker to calc stats for data given */
+    getStats : function(){
+        if( !this.data ){ return; }
+        var view = this,
+            config = this.model.get( 'config' ),
+            meanWorker = new Worker( '/plugins/visualizations/scatterplot/static/worker-stats.js' );
+        meanWorker.postMessage({
+            data    : this.data,
+            keys    : [ config.xColumn, config.yColumn ]
+        });
+        meanWorker.onerror = function( event ){
+            meanWorker.terminate();
+        };
+        meanWorker.onmessage = function( event ){
+            view.renderStats( event.data );
+        };
+    },
+
+    renderStats : function( stats, error ){
+        //console.debug( 'renderStats:', stats, error );
+        //console.debug( JSON.stringify( stats, null, '  ' ) );
+        var config = this.model.get( 'config' ),
+            $statsTable = this.$el.find( '.stats-display' ),
+            xLabel = config.x.label, yLabel = config.y.label,
+            $table = $( '<table/>' ).addClass( 'table' )
+                .append([ '<thead><th></th><th>', xLabel, '</th><th>', yLabel, '</th></thead>' ].join( '' ))
+                .append( _.map( stats, function( stat, key ){
+                    return $([ '<tr><td>', key, '</td><td>', stat[0], '</td><td>', stat[1], '</td></tr>' ].join( '' ));
+                }));
+        $statsTable.empty().append( $table );
+    },
+
+    toggleStats : function( showStats ){
+        var $statsDisplay = this.$el.find( '.stats-display' );
+        showStats = ( showStats === undefined )?( $statsDisplay.is( ':hidden' ) ):( showStats );
+        if( showStats ){
+            this.$el.find( 'svg' ).hide();
+            $statsDisplay.show();
+            this.$el.find( '.controls .stats-toggle-btn' ).text( 'Plot' );
+        } else {
+            $statsDisplay.hide();
+            this.$el.find( 'svg' ).show();
+            this.$el.find( '.controls .stats-toggle-btn' ).text( 'Stats' );
+        }
+    },
+
+    toString : function(){
+        return 'ScatterplotView()';
+    }
+});

config/plugins/visualizations/scatterplot/src/scatterplot.js

@@ -1,488 +1,283 @@
-/* =============================================================================
-todo:
-    outside this:
-        BUG: setting width, height in plot controls doesn't re-interpolate data locations!!
-        BUG?: get metadata_column_names (from datatype if necessary)
-        BUG: single vis in popupmenu should have tooltip with that name NOT 'Visualizations'
-    
-    wire label setters, anim setter
-    
-    TwoVarScatterplot:
-        ??: maybe better to do this with a canvas...
-        save as visualization
-        to seperate file?
-        remove underscore dependencies
-        add interface to change values (seperate)?
-        download svg -> base64 encode
-        incorporate glyphs, glyph state renderers
-        
-    ScatterplotSettingsForm:
-        some css bug that lowers the width of settings form when plot-controls tab is open
-            causes chart to shift
-        what can be abstracted/reused for other graphs?
-        avoid direct manipulation of this.plot
-        allow option to put plot into seperate tab of interface (for small multiples)
-    
-        provide callback in view to load data incrementally - for large sets
-            paginate
-                handle rerender
-                use endpoint (here and on the server (fileptr))
-            fetch (new?) data
-                handle rerender
-            use d3.TSV?
-            render warning on long data (> maxDataPoints)
-                adjust endpoint
-        
-        selectable list of preset column comparisons (rnaseq etc.)
-            how to know what sort of Tabular the data is?
-        smarter about headers
-        validate columns selection (here or server)
-
-        set stats column names by selected columns
-        move chart into tabbed area...
-        
-    Scatterplot.mako:
-        multiple plots on one page (small multiples)
-        ?? ensure svg styles thru d3 or css?
-            d3: configable (easily)
-            css: standard - better maintenance
-            ? override at config
-
-============================================================================= */
+// =============================================================================
 /**
  *  Two Variable scatterplot visualization using d3
  *      Uses semi transparent circles to show density of data in x, y grid
  *      usage :
- *          var plot = new TwoVarScatterplot({ containerSelector : 'div#my-plot', ... })
- *          plot.render( xColumnData, yColumnData );
- *
- *  depends on: d3, underscore
+ *          var plot = new scatterplot( $( 'svg' ).get(0), config, data )
  */
-function TwoVarScatterplot( config ){
-    var TICK_LINE_AND_PADDING = 10,
-        GUESS_AT_SVG_CHAR_WIDTH = 7,
-        GUESS_AT_SVG_CHAR_HEIGHT = 10,
-        PADDING = 8,
-        X_LABEL_TOO_LONG_AT = 5;
-        
-    // set up logging
-    //this.debugging = true;
-    this.log = function(){
-        if( this.debugging && console && console.debug ){
-            var args = Array.prototype.slice.call( arguments );
-            args.unshift( this.toString() );
-            console.debug.apply( console, args );
-        }
-    };
-    this.log( 'new TwoVarScatterplot:', config );
-    
-    // ........................................................ set up chart config
-    // config will default to these values when not passed in
-    //NOTE: called on new
-    this.defaults = {
-        id : 'TwoVarScatterplot',
-        containerSelector : 'body',
-        //TODO??: needed?
-        maxDataPoints : 30000,
-        datapointSize : 4,
-        animDuration : 500,
-        //TODO: variable effect (not always exactly # of ticks set to)
-        xNumTicks : 10,
-        yNumTicks : 10,
-        xAxisLabelBumpY : 40,
-        yAxisLabelBumpX : -40,
-        width   : 400,
-        height  : 400,
-        //TODO: anyway to make this a sub-obj?
-        marginTop : 50,
-        marginRight : 50,
-        marginBottom : 50,
-        marginLeft : 50,
-        
-        xMin : null,
-        xMax : null,
-        yMin : null,
-        yMax : null,
-        
-        xLabel : "X",
-        yLabel : "Y"
-    };
-    this.config = _.extend( {}, this.defaults, config );
-    this.log( 'intial config:', this.config );
-    
-    this.updateConfig = function( newConfig, rerender ){
-        // setter for chart config
-        //TODO: validate here
-        _.extend( this.config, newConfig );
-        this.log( this + '.updateConfig:', this.config );
-        //TODO: implement rerender flag
-    };
-    
-    // ........................................................ helpers
-    this.toString = function(){
-        return this.config.id;
-    };
-    // conv. methods for svg transforms
-    this.translateStr = function( x, y ){
-        return 'translate(' + x + ',' + y + ')';
-    };
-    this.rotateStr = function( d, x, y ){
-        return 'rotate(' + d + ',' + x + ',' + y + ')';
+function scatterplot( renderTo, config, data ){
+    //console.log( 'scatterplot', config );
+
+    var translateStr = function( x, y ){
+            return 'translate(' + x + ',' + y + ')';
+        },
+        rotateStr = function( d, x, y ){
+            return 'rotate(' + d + ',' + x + ',' + y + ')';
+        },
+        getX = function( d, i ){
+            //console.debug( d[ config.xColumn ] );
+            return d[ config.xColumn ];
+        },
+        getY = function( d, i ){
+            //console.debug( d[ config.yColumn ] );
+            return d[ config.yColumn ];
+        };
+
+    // .................................................................... scales
+    var stats = {
+            x    : { extent: d3.extent( data, getX ) },
+            y    : { extent: d3.extent( data, getY ) }
+        };
+
+    //TODO: set pan/zoom limits
+    //  from http://stackoverflow.com/questions/10422738/limiting-domain-when-zooming-or-panning-in-d3-js
+    //self.x.domain([Math.max(self.x.domain()[0], self.options.xmin), Math.min(self.x.domain()[1], self.options.xmax)]);
+    //self.y.domain([Math.max(self.y.domain()[0], self.options.ymin), Math.min(self.y.domain()[1], self.options.ymax)]);
+    var interpolaterFns = {
+        x : d3.scale.linear()
+            .domain( stats.x.extent )
+            .range([ 0, config.width ]),
+        y : d3.scale.linear()
+            .domain( stats.y.extent )
+            .range([ config.height, 0 ])
     };
 
-    // ........................................................ initial element creation
-    this.adjustChartDimensions = function( top, right, bottom, left ){
-        //this.log( this + '.adjustChartDimensions', arguments );
-        top = top || 0;
-        right = right || 0;
-        bottom = bottom || 0;
-        left = left || 0;
-        this.svg
-            .attr( "width",  this.config.width  + ( this.config.marginRight  + right )
-                                                + ( this.config.marginLeft   + left ) )
-            .attr( "height", this.config.height + ( this.config.marginTop    + top )
-                                                + ( this.config.marginBottom + bottom ) )
-            // initial is hidden - show it
-            .style( 'display', 'block' );
-            
-        // move content group away from margins
-        //TODO: allow top, right axis
-        this.content = this.svg.select( "g.content" )
-            .attr( "transform", this.translateStr( this.config.marginLeft + left, this.config.marginTop + top ) );
-    };
-    
-    // ........................................................ data and scales
-    this.preprocessData = function( data, min, max ){
-        //this.log( this + '.preprocessData', arguments );
-        //TODO: filter by min, max if set
-        
-        // set a cap on the data, limit to first n points
-        return ( data.length > this.config.maxDataPoints )? ( data.slice( 0, this.config.maxDataPoints ) ): ( data );
-    };
-    
-    this.findMinMaxes = function( xCol, yCol, meta ){
-        //this.log( this + '.findMinMaxes', arguments );
-        // configuration takes priority, otherwise meta (from the server) if passed, last-resort: compute it here
-        this.xMin = this.config.xMin || ( meta )?( meta[0].min ):( d3.min( xCol ) );
-        this.xMax = this.config.xMax || ( meta )?( meta[0].max ):( d3.max( xCol ) );
-        this.yMin = this.config.yMin || ( meta )?( meta[1].min ):( d3.min( yCol ) );
-        this.yMax = this.config.yMax || ( meta )?( meta[1].max ):( d3.max( yCol ) );
-    };
-    
-    this.setUpScales = function(){
-        //this.log( this + '.setUpScales', arguments );
-        // Interpolation for x, y based on data domains
-        this.xScale = d3.scale.linear()
-                .domain([ this.xMin, this.xMax ])
-                .range([ 0, this.config.width ]),
-        this.yScale = d3.scale.linear()
-                .domain([ this.yMin, this.yMax ])
-                .range([ this.config.height, 0 ]);
-    };
-    
-    // ........................................................ axis and ticks
-    this.setUpXAxis = function(){
-        //this.log( this + '.setUpXAxis', arguments );
-        // origin: bottom, left
-        //TODO: incoporate top, right
-        this.xAxisFn = d3.svg.axis()
-            .scale( this.xScale )
-            .ticks( this.config.xNumTicks )
-            .orient( 'bottom' );
-        this.xAxis// = content.select( 'g#x-axis' )
-            .attr( 'transform', this.translateStr( 0, this.config.height ) )
-            .call( this.xAxisFn );
-        //this.log( 'xAxis:', this.xAxis );
-        
-        //TODO: adjust ticks when tick labels are long - move odds down and extend tick line
-        // (for now) hide them
-        var xLongestTickLabel = d3.max( _.map( [ this.xMin, this.xMax ],
-                                     function( number ){ return ( String( number ) ).length; } ) );
-        //this.log( 'xLongestTickLabel:', xLongestTickLabel );
-        if( xLongestTickLabel >= X_LABEL_TOO_LONG_AT ){
-            this.xAxis.selectAll( 'g' ).filter( ':nth-child(odd)' ).style( 'display', 'none' );
-        }
-        
-        this.log( 'this.config.xLabel:', this.config.xLabel );
-        this.xAxisLabel// = xAxis.select( 'text#x-axis-label' )
-            .attr( 'x', this.config.width / 2 )
-            .attr( 'y', this.config.xAxisLabelBumpY )
-            .attr( 'text-anchor', 'middle' )
-            .text( this.config.xLabel );
-        this.log( 'xAxisLabel:', this.xAxisLabel );
+    // .................................................................... main components
+    var zoom = d3.behavior.zoom()
+        .x( interpolaterFns.x )
+        .y( interpolaterFns.y )
+        .scaleExtent([ 1, 30 ])
+        .scale( config.scale || 1 )
+        .translate( config.translate || [ 0, 0 ] );
+
+    //console.debug( renderTo );
+    var svg = d3.select( renderTo )
+        .attr( "class", "scatterplot" )
+        //.attr( "width",  config.width  + ( config.margin.right + config.margin.left ) )
+        .attr( "width",  '100%' )
+        .attr( "height", config.height + ( config.margin.top + config.margin.bottom ) );
+
+    var content = svg.append( "g" )
+        .attr( "class", "content" )
+        .attr( "transform", translateStr( config.margin.left, config.margin.top ) )
+        .call( zoom );
+
+    // a BIG gotcha - zoom (or any mouse/touch event in SVG?) requires the pointer to be over an object
+    //  create a transparent rect to be that object here
+    content.append( 'rect' )
+        .attr( "class", "zoom-rect" )
+        .attr( "width", config.width ).attr( "height", config.height )
+        .style( "fill", "transparent" );
+
+    //console.log( 'svg:', svg, 'content:', content );
+
+    // .................................................................... axes
+    var axis = { x : {}, y : {} };
+    //console.log( 'x.ticks:', config.x.ticks );
+    //console.log( 'y.ticks:', config.y.ticks );
+    axis.x.fn = d3.svg.axis()
+        .orient( 'bottom' )
+        .scale( interpolaterFns.x )
+        .ticks( config.x.ticks )
+        // this will convert thousands -> k, millions -> M, etc.
+        .tickFormat( d3.format( 's' ) );
+
+    axis.y.fn = d3.svg.axis()
+        .orient( 'left' )
+        .scale( interpolaterFns.y )
+        .ticks( config.y.ticks )
+        .tickFormat( d3.format( 's' ) );
+
+    axis.x.g = content.append( 'g' )
+        .attr( 'class', 'x axis' )
+        .attr( 'transform', translateStr( 0, config.height ) )
+        .call( axis.x.fn );
+    //console.log( 'axis.x.g:', axis.x.g );
+
+    axis.y.g = content.append( 'g' )
+        .attr( 'class', 'y axis' )
+        .call( axis.y.fn );
+    //console.log( 'axis.y.g:', axis.y.g );
+
+    // ................................ axis labels
+    var padding = 6;
+    // x-axis label
+    axis.x.label = svg.append( 'text' )
+        .attr( 'class', 'axis-label' )
+        .text( config.x.label )
+        // align to the top-middle
+        .attr( 'text-anchor', 'middle' )
+        .attr( 'dominant-baseline', 'text-after-edge' )
+        .attr( 'x', ( config.width / 2 ) + config.margin.left )
+        // place 4 pixels below the axis bounds
+        .attr( 'y', ( config.height + config.margin.bottom + config.margin.top ) - padding );
+    //console.log( 'axis.x.label:', axis.x.label );
+
+//TODO: anchor to left of x margin/graph
+    // y-axis label
+    // place 4 pixels left of the axis.y.g left edge
+    axis.y.label = svg.append( 'text' )
+        .attr( 'class', 'axis-label' )
+        .text( config.y.label )
+        // align to bottom-middle
+        .attr( 'text-anchor', 'middle' )
+        .attr( 'dominant-baseline', 'text-before-edge' )
+        .attr( 'x', padding )
+        .attr( 'y', config.height / 2 )
+        // rotate around the alignment point
+        .attr( 'transform', rotateStr( -90, padding, config.height / 2 ) );
+    //console.log( 'axis.y.label:', axis.y.label );
+
+    axis.redraw = function _redrawAxis(){
+        svg.select( ".x.axis" ).call( axis.x.fn );
+        svg.select( ".y.axis" ).call( axis.y.fn );
     };
 
-    this.setUpYAxis = function(){
-        //this.log( this + '.setUpYAxis', arguments );
-        this.yAxisFn = d3.svg.axis()
-            .scale( this.yScale )
-            .ticks( this.config.yNumTicks )
-            .orient( 'left' );
-        this.yAxis// = content.select( 'g#y-axis' )
-            .call( this.yAxisFn );
-        //this.log( 'yAxis:', this.yAxis );
-    
-        // a too complicated section for increasing the left margin when tick labels are long
-        // get the tick labels for the y axis
-        var yTickLabels = this.yAxis.selectAll( 'text' ).filter( function( e, i ){ return i !== 0; } );
-        this.log( 'yTickLabels:', yTickLabels );
-        
-        // get the longest label length (or 0 if no labels)
-        this.yLongestLabel = d3.max(
-            //NOTE: d3 returns an nested array - use the plain array inside ([0])
-            yTickLabels[0].map( function( e, i ){
-                return ( d3.select( e ).text() ).length;
-            })
-        ) || 0;
-        //this.log( 'yLongestLabel:', this.yLongestLabel );
-        //TODO: lose the guessing if possible
-        var neededY = TICK_LINE_AND_PADDING + ( this.yLongestLabel * GUESS_AT_SVG_CHAR_WIDTH )
-                    + PADDING + GUESS_AT_SVG_CHAR_HEIGHT;
-        //this.log( 'neededY:', neededY );
-            
-        // increase width for yLongerStr, increase margin for y
-        //TODO??: (or transform each number: 2k)
-        this.config.yAxisLabelBumpX = -( neededY - GUESS_AT_SVG_CHAR_HEIGHT );
-        if( this.config.marginLeft < neededY ){
-            var adjusting = ( neededY ) - this.config.marginLeft;
-            adjusting = ( adjusting < 0 )?( 0 ):( adjusting );
-            //this.log( 'adjusting:', adjusting );
-            
-            // update dimensions, translations
-            this.adjustChartDimensions( 0, 0, 0, adjusting );
-        }
-        //this.log( 'this.config.yAxisLableBumpx, this.config.marginLeft:',
-        //          this.config.yAxisLabelBumpX, this.config.marginLeft );
-        
-        this.yAxisLabel// = yAxis.select( 'text#y-axis-label' )
-            .attr( 'x', this.config.yAxisLabelBumpX )
-            .attr( 'y', this.config.height / 2 )
-            .attr( 'text-anchor', 'middle' )
-            .attr( 'transform', this.rotateStr( -90, this.config.yAxisLabelBumpX, this.config.height / 2 ) )
-            .text( this.config.yLabel );
-        //this.log( 'yAxisLabel:', this.yAxisLabel );
-    };
-    
-    // ........................................................ grid lines
-    this.renderGrid = function(){
-        //this.log( this + '.renderGrid', arguments );
-        // VERTICAL
-        // select existing
-        this.vGridLines = this.content.selectAll( 'line.v-grid-line' )
-            .data( this.xScale.ticks( this.xAxisFn.ticks()[0] ) );
-            
-        // append any extra lines needed (more ticks)
-        this.vGridLines.enter().append( 'svg:line' )
+    // .................................................................... grid
+    function renderGrid(){
+        var grid = { v : {}, h: {} };
+        // vertical
+        grid.v.lines = content.selectAll( 'line.v-grid-line' )
+            // data are the axis ticks; enter, update, exit
+            .data( interpolaterFns.x.ticks( axis.x.fn.ticks()[0] ) );
+        // enter: append any extra lines needed (more ticks)
+        grid.v.lines.enter()
+            .append( 'svg:line' )
             .classed( 'grid-line v-grid-line', true );
-            
-        // update the attributes of existing and appended
-        this.vGridLines
-            .attr( 'x1', this.xScale )
+        // update: set coords
+        grid.v.lines
+            .attr( 'x1', interpolaterFns.x )
+            .attr( 'x2', interpolaterFns.x )
             .attr( 'y1', 0 )
-            .attr( 'x2', this.xScale )
-            .attr( 'y2', this.config.height );
-            
-        // remove unneeded (less ticks)
-        this.vGridLines.exit().remove();
-        //this.log( 'vGridLines:', this.vGridLines );
+            .attr( 'y2', config.height );
+        // exit: just remove them
+        grid.v.lines.exit().remove();
+        //console.log( 'grid.v.lines:', grid.v.lines );
 
-        // HORIZONTAL
-        this.hGridLines = this.content.selectAll( 'line.h-grid-line' )
-            .data( this.yScale.ticks( this.yAxisFn.ticks()[0] ) );
-            
-        this.hGridLines.enter().append( 'svg:line' )
+        // horizontal
+        grid.h.lines = content.selectAll( 'line.h-grid-line' )
+            .data( interpolaterFns.y.ticks( axis.y.fn.ticks()[0] ) );
+        grid.h.lines.enter()
+            .append( 'svg:line' )
             .classed( 'grid-line h-grid-line', true );
-        
-        this.hGridLines
+        grid.h.lines
             .attr( 'x1', 0 )
-            .attr( 'y1', this.yScale )
-            .attr( 'x2', this.config.width )
-            .attr( 'y2', this.yScale );
-            
-        this.hGridLines.exit().remove();
-        //this.log( 'hGridLines:', this.hGridLines );
-    };
-    
-    // ........................................................ data points
-    this.renderDatapoints = function( xCol, yCol, ids ){
-        this.log( this + '.renderDatapoints', arguments );
-        var count = 0,
-            plot = this,
-            xPosFn = function( d, i ){
-                //if( d ){ this.log( 'x.data:', newXCol[ i ], 'plotted:', plot.xScale( newXCol[ i ] ) ); }
-                return plot.xScale( xCol[ i ] );
-            },
-            yPosFn = function( d, i ){
-                //if( d ){ this.log( 'y.data:', newYCol[ i ], 'plotted:', plot.yScale( newYCol[ i ] ) ); }
-                return plot.yScale( yCol[ i ] );
-            };
+            .attr( 'x2', config.width )
+            .attr( 'y1', interpolaterFns.y )
+            .attr( 'y2', interpolaterFns.y );
+        grid.h.lines.exit().remove();
+        //console.log( 'grid.h.lines:', grid.h.lines );
+        return grid;
+    }
+    var grid = renderGrid();
 
-        //this.datapoints = this.addDatapoints( xCol, yCol, ids, ".glyph" );
-        var datapoints = this.content.selectAll( '.glyph' ).data( xCol );
+    //// .................................................................... datapoints
+    var datapoints = content.selectAll( '.glyph' ).data( data )
+        // enter - NEW data to be added as glyphs
+        .enter().append( 'svg:circle' )
+            .classed( "glyph", true )
+            .attr( "cx", function( d, i ){ return interpolaterFns.x( getX( d, i ) ); })
+            .attr( "cy", function( d, i ){ return interpolaterFns.y( getY( d, i ) ); })
+            .attr( "r",  0 );
 
-        // enter - NEW data to be added as glyphs: give them a 'entry' position and style
-        count = 0;
-        datapoints.enter()
-            .append( 'svg:circle' )
-                .each( function(){ count += 1; } )
-                .classed( "glyph", true )
-                .attr( "cx", 0 )
-                .attr( "cy", this.config.height )
-                // start all bubbles small...
-                .attr( "r", 0 );
-        this.log( count, ' new glyphs created' );
-        
-        // for all EXISTING glyphs and those that need to be added: transition anim to final state
-        count = 0;
-        datapoints
-            // ...animate to final position
-            .transition().duration( this.config.animDuration )
-                .each( function(){ count += 1; } )
-                .attr( "cx", xPosFn )
-                .attr( "cy", yPosFn )
-                .attr( "r", plot.config.datapointSize );
-        this.log( count, ' existing glyphs transitioned' );
-        
-        // events
-        // glyphs that need to be removed: transition to from normal state to 'exit' state, remove from DOM
-        datapoints.exit()
-            .each( function(){ count += 1; } )
-            .transition().duration( this.config.animDuration )
-                .attr( "cy", this.config.height )
-                .attr( "r", 0 )
-            .remove();
-        this.log( count, ' glyphs removed' );
+    // for all EXISTING glyphs and those that need to be added: transition anim to final state
+    datapoints.transition().duration( config.animDuration )
+        .attr( "r", config.datapointSize );
+    //console.log( 'datapoints:', datapoints );
 
-        this._addDatapointEventhandlers( datapoints, xCol, yCol, ids );
-    };
+    function _redrawDatapointsClipped(){
+        return datapoints
+            //TODO: interpolates twice
+            .attr( "cx", function( d, i ){ return interpolaterFns.x( getX( d, i ) ); })
+            .attr( "cy", function( d, i ){ return interpolaterFns.y( getY( d, i ) ); })
+            .style( 'display', 'block' )
+            // filter out points now outside the graph content area and hide them
+            .filter( function( d, i ){
+                var cx = d3.select( this ).attr( "cx" ),
+                    cy = d3.select( this ).attr( "cy" );
+                if( cx < 0 || cx > config.width  ){ return true; }
+                if( cy < 0 || cy > config.height ){ return true; }
+                return false;
+            }).style( 'display', 'none' );
+    }
+    _redrawDatapointsClipped();
 
-    this._addDatapointEventhandlers = function( datapoints, xCol, yCol, ids ){
-        var plot = this;
-        datapoints
-            //TODO: remove magic numbers
-            .on( 'mouseover', function( d, i ){
-                var datapoint = d3.select( this );
-                datapoint
-                    .style( 'fill', 'red' )
-                    .style( 'fill-opacity', 1 );
-                
-                // create horiz, vert lines to axis
-                plot.content.append( 'line' )
-                    .attr( 'stroke', 'red' )
-                    .attr( 'stroke-width', 1 )
-                    // start not at center, but at the edge of the circle - to prevent mouseover thrashing
-                    .attr( 'x1', datapoint.attr( 'cx' ) - plot.config.datapointSize )
-                    .attr( 'y1', datapoint.attr( 'cy' ) )
-                    .attr( 'x2', 0 )
-                    .attr( 'y2', datapoint.attr( 'cy' ) )
-                    .classed( 'hoverline', true );
+    // .................................................................... behaviors
+    function zoomed( scale, translateX, translateY ){
+        //console.debug( 'zoom', this, zoom.scale(), zoom.translate() );
 
-                // if the vertical hoverline
-                if( datapoint.attr( 'cy' ) < plot.config.height ){
-                    plot.content.append( 'line' )
-                        .attr( 'stroke', 'red' )
-                        .attr( 'stroke-width', 1 )
-                        .attr( 'x1', datapoint.attr( 'cx' ) )
-                        .attr( 'y1', datapoint.attr( 'cy' ) + plot.config.datapointSize )
-                        .attr( 'x2', datapoint.attr( 'cx' ) )
-                        .attr( 'y2', plot.config.height )
-                        .classed( 'hoverline', true );
-                }
+        // re-render axis, grid, and datapoints
+        $( '.chart-info-box' ).remove();
+        axis.redraw();
+        _redrawDatapointsClipped();
+        grid = renderGrid();
 
-                var datapointWindowPos = $( this ).offset();
-                plot.datapointInfoBox = plot.infoBox(
-                    datapointWindowPos.top, datapointWindowPos.left,
-                    plot.infoHtml( xCol[ i ], yCol[ i ], ( ids )?( ids[ i ] ):( undefined ) )
-                );
-                $( 'body' ).append( plot.datapointInfoBox );
-            })
-            .on( 'mouseout', function(){
-                d3.select( this )
-                    .style( 'fill', 'black' )
-                    .style( 'fill-opacity', 0.2 );
-                plot.content.selectAll( '.hoverline' ).remove();
-                if( plot.datapointInfoBox ){
-                    plot.datapointInfoBox.remove();
-                }
-            });
-    },
-    
-    this.render = function( columnData, meta ){
-        this.log( this + '.render', arguments );
-        this.log( '\t config:', this.config );
+        $( svg.node() ).trigger( 'zoom.scatterplot', {
+            scale       : zoom.scale(),
+            translate   : zoom.translate()
+        });
+    }
+    //TODO: programmatically set zoom/pan and save in config
+    //TODO: set pan/zoom limits
+    zoom.on( "zoom", zoomed );
 
-        // prepare the data
-        //pre: columns passed are numeric
-        //pre: at least two columns are passed
-        //assume: first column is x, second column is y, any remaining aren't used
-        var xCol = columnData[0],
-            yCol = columnData[1],
-            ids = ( columnData.length > 2 )?( columnData[2] ):( undefined );
-        //this.log( this + '.render', xCol.length, yCol.length, this.config );
-        
-        //pre: xCol.len == yCol.len
-        xCol = this.preprocessData( xCol );
-        yCol = this.preprocessData( yCol );
-        this.log( 'xCol len', xCol.length, 'yCol len', yCol.length );
-        
-        this.findMinMaxes( xCol, yCol, meta );
-        //this.log( 'xMin, xMax, yMin, yMax:', this.xMin, this.xMax, this.yMin, this.yMax );
-        this.setUpScales();
+    function infoBox( top, left, d ){
+        // create an abs pos. element containing datapoint data (d) near the point (top, left)
+        //  with added padding to clear the mouse pointer
+        left += 8;
+        return $([
+            '<div class="chart-info-box" style="position: absolute">',
+                (( config.idColumn !== undefined )?( '<div>' + d[ config.idColumn ] + '</div>' ):( '' )),
+                '<div>', getX( d ), '</div>',
+                '<div>', getY( d ), '</div>',
+            '</div>'
+        ].join( '' ) ).css({ top: top, left: left, 'z-index': 2 });
+    }
 
-        // find (or build if it doesn't exist) the svg dom infrastructure
-        if( !this.svg ){ this.svg = d3.select( 'svg' ).attr( "class", "chart" ); }
-        if( !this.content ){
-            this.content = this.svg.append( "svg:g" ).attr( "class", "content" ).attr( 'id', this.config.id );
+    datapoints.on( 'mouseover', function( d, i ){
+        var datapoint = d3.select( this );
+        datapoint
+            .style( 'fill', 'red' )
+            .style( 'fill-opacity', 1 );
+
+        // create horiz line to axis
+        content.append( 'line' )
+            .attr( 'stroke', 'red' )
+            .attr( 'stroke-width', 1 )
+            // start not at center, but at the edge of the circle - to prevent mouseover thrashing
+            .attr( 'x1', datapoint.attr( 'cx' ) - config.datapointSize )
+            .attr( 'y1', datapoint.attr( 'cy' ) )
+            .attr( 'x2', 0 )
+            .attr( 'y2', datapoint.attr( 'cy' ) )
+            .classed( 'hoverline', true );
+
+        // create vertical line to axis - if not on the x axis
+        if( datapoint.attr( 'cy' ) < config.height ){
+            content.append( 'line' )
+                .attr( 'stroke', 'red' )
+                .attr( 'stroke-width', 1 )
+                .attr( 'x1', datapoint.attr( 'cx' ) )
+                // attributes are strings so, (accrd. to js) '3' - 1 = 2 but '3' + 1 = '31': coerce
+                .attr( 'y1', +datapoint.attr( 'cy' ) + config.datapointSize )
+                .attr( 'x2', datapoint.attr( 'cx' ) )
+                .attr( 'y2', config.height )
+                .classed( 'hoverline', true );
         }
-        //this.log( 'svg:', this.svg );
-        //this.log( 'content:', this.content );
 
-        this.adjustChartDimensions();
-        
-        if( !this.xAxis ){ this.xAxis = this.content.append( 'g' ).attr( 'class', 'axis' ).attr( 'id', 'x-axis' ); }
-        if( !this.xAxisLabel ){
-            this.xAxisLabel = this.xAxis.append( 'text' ).attr( 'class', 'axis-label' ).attr( 'id', 'x-axis-label' );
-        }
-        //this.log( 'xAxis:', this.xAxis, 'xAxisLabel:', this.xAxisLabel );
-
-        if( !this.yAxis ){ this.yAxis = this.content.append( 'g' ).attr( 'class', 'axis' ).attr( 'id', 'y-axis' ); }
-        if( !this.yAxisLabel ){
-            this.yAxisLabel = this.yAxis.append( 'text' ).attr( 'class', 'axis-label' ).attr( 'id', 'y-axis-label' );
-        }
-        //this.log( 'yAxis:', this.yAxis, 'yAxisLabel:', this.yAxisLabel );
-        
-        this.setUpXAxis();
-        this.setUpYAxis();
-        
-        this.renderGrid();
-        this.renderDatapoints( xCol, yCol, ids );
-    };
-
-    this.infoHtml = function( x, y, id ){
-        var retDiv = $( '<div/>' );
-        if( id ){
-            $( '<div/>' ).text( id ).css( 'font-weight', 'bold' ).appendTo( retDiv );
-        }
-        $( '<div/>' ).text( x ).appendTo( retDiv );
-        $( '<div/>' ).text( y ).appendTo( retDiv );
-        return retDiv.html();
-    };
-
-    //TODO: html for now
-    this.infoBox = function( top, left, html, adjTop, adjLeft ){
-        adjTop  = adjTop  || 0;
-        adjLeft = adjLeft || 20;
-        var infoBox = $( '<div />' )
-            .addClass( 'chart-info-box' )
-            .css({
-                'position'  : 'absolute',
-                'top'       : top  + adjTop,
-                'left'      : left + adjLeft
-            });
-        infoBox.html( html );
-        return infoBox;
-    };
+        // show the info box and trigger an event
+        var bbox = this.getBoundingClientRect();
+        $( 'body' ).append( infoBox( bbox.top, bbox.right, d ) );
+        $( svg.node() ).trigger( 'mouseover-datapoint.scatterplot', [ this, d, i ] );
+    });
 
+    datapoints.on( 'mouseout', function(){
+        // return the point to normal, remove hoverlines and info box
+        d3.select( this )
+            .style( 'fill', 'black' )
+            .style( 'fill-opacity', 0.2 );
+        content.selectAll( '.hoverline' ).remove();
+        $( '.chart-info-box' ).remove();
+    });
 }
 
 //==============================================================================

config/plugins/visualizations/scatterplot/src/scatterplotControlForm.js

@@ -1,631 +0,0 @@
-/* =============================================================================
-todo:
-    I'd like to move the svg creation out of the splot constr. to:
-        allow adding splots to an existing canvas
-        allow mult. splots sharing a canvas
-
-
-    outside this:
-        BUG: setting width, height in plot controls doesn't re-interpolate data locations!!
-        BUG?: get metadata_column_names (from datatype if necessary)
-        BUG: single vis in popupmenu should have tooltip with that name NOT 'Visualizations'
-    
-    wire label setters, anim setter
-    
-    TwoVarScatterplot:
-        ??: maybe better to do this with a canvas...
-        save as visualization
-        to seperate file?
-        remove underscore dependencies
-        add interface to change values (seperate)?
-        download svg -> base64 encode
-        incorporate glyphs, glyph state renderers
-        
-    ScatterplotSettingsForm:
-        some css bug that lowers the width of settings form when plot-controls tab is open
-            causes chart to shift
-        what can be abstracted/reused for other graphs?
-        avoid direct manipulation of this.plot
-        allow option to put plot into seperate tab of interface (for small multiples)
-    
-        provide callback in view to load data incrementally - for large sets
-            paginate
-                handle rerender
-                use endpoint (here and on the server (fileptr))
-            fetch (new?) data
-                handle rerender
-            use d3.TSV?
-            render warning on long data (> maxDataPoints)
-                adjust endpoint
-        
-        selectable list of preset column comparisons (rnaseq etc.)
-            how to know what sort of Tabular the data is?
-        smarter about headers
-        validate columns selection (here or server)
-
-        set stats column names by selected columns
-        move chart into tabbed area...
-        
-    Scatterplot.mako:
-        multiple plots on one page (small multiples)
-        ?? ensure svg styles thru d3 or css?
-            d3: configable (easily)
-            css: standard - better maintenance
-            ? override at config
-
-============================================================================= */
-/**
- *  Scatterplot control UI as a backbone view
- *      handles:
- *          getting the desired data
- *          configuring the plot display
- *          showing (general) statistics
- *
- *  initialize attributes REQUIRES a dataset and an apiDatasetsURL
- */
-var ScatterplotControlForm = BaseView.extend( LoggableMixin ).extend({
-    //logger      : console,
-    className   : 'scatterplot-control-form',
-    
-    //NOTE: should include time needed to render
-    dataLoadDelay : 4000,
-    dataLoadSize  : 5000,
-
-    loadingIndicatorImage : 'loading_small_white_bg.gif',
-    fetchMsg    : 'Fetching data...',
-    renderMsg   : 'Rendering...',
-    
-    initialize : function( attributes ){
-        this.log( this + '.initialize, attributes:', attributes );
-        
-        this.dataset = null;
-        this.chartConfig = null;
-        this.chart = null;
-        this.loader = null;
-
-        // set up refs to the four tab areas
-        this.$dataControl = null;
-        this.$chartControl = null;
-        this.$statsDisplay = null;
-        this.$chartDisplay = null;
-
-        this.dataFetch = null;
-
-        this.initializeFromAttributes( attributes );
-        this.initializeChart( attributes );
-        this.initializeDataLoader( attributes );
-    },
-
-    initializeFromAttributes : function( attributes ){
-        // required settings: ensure certain vars we need are passed in attributes
-        if( !attributes || !attributes.dataset ){
-            throw( "ScatterplotView requires a dataset" );
-        } else {
-            this.dataset = attributes.dataset;
-        }
-        if( jQuery.type( this.dataset.metadata_column_types ) === 'string' ){
-            this.dataset.metadata_column_types = this.dataset.metadata_column_types.split( ', ' );
-        }
-        this.log( '\t dataset:', this.dataset );
-
-        // attempt to get possible headers from the data's first line
-        if( this.dataset.comment_lines && this.dataset.comment_lines.length ){
-            //TODO:??
-            var firstLine = this.dataset.comment_lines[0],
-                possibleHeaders = firstLine.split( '\t' );
-            if( possibleHeaders.length === this.dataset.metadata_column_types.length ){
-                this.possibleHeaders = possibleHeaders;
-            }
-        }
-
-        // passed from mako helper
-        //TODO: integrate to galaxyPaths
-        //TODO: ?? seems like data loader section would be better
-        if( !attributes.apiDatasetsURL ){
-            throw( "ScatterplotView requires a apiDatasetsURL" );
-        } else {
-            this.dataURL = attributes.apiDatasetsURL + '/' + this.dataset.id + '?';
-        }
-        this.log( '\t dataURL:', this.dataURL );
-    },
-
-    initializeChart : function( attributes ){
-        // set up the basic chart infrastructure and config (if any)
-        this.chartConfig = attributes.chartConfig || {};
-        //if( this.logger ){ this.chartConfig.debugging = true; }
-        this.log( '\t initial chartConfig:', this.chartConfig );
-
-        this.chart = new TwoVarScatterplot( this.chartConfig );
-        //TODO: remove 2nd ref, use this.chart.config
-        this.chartConfig = this.chart.config;
-    },
-
-    initializeDataLoader : function( attributes ){
-        // set up data loader
-        var view = this;
-        this.loader = new LazyDataLoader({
-            //logger  : ( this.logger )?( this.logger ):( null ),
-            // we'll generate this when columns are chosen
-            url     : null,
-            start   : attributes.start || 0,
-            //NOTE: metadata_data_lines can be null (so we won't know the total)
-            total   : attributes.total || this.dataset.metadata_data_lines,
-            delay   : this.dataLoadDelay,
-            size    : this.dataLoadSize,
-
-            buildUrl : function( start, size ){
-                // currently VERY SPECIFIC to using data_providers.py start_val, max_vals params
-                return this.url + '&' + jQuery.param({
-                    start_val: start,
-                    max_vals:  size
-                });
-            }
-        });
-        $( this.loader ).bind( 'error', function( event, status, error ){
-            view.log( 'ERROR:', status, error );
-            alert( 'ERROR fetching data:\n' + status + '\n' + error );
-            view.hideLoadingIndicator();
-        });
-    },
-    
-    // ------------------------------------------------------------------------- CONTROLS RENDERING
-    render : function(){
-        this.log( this + '.render' );
-
-        // render the tab controls, areas and loading indicator
-        this.$el.append( ScatterplotControlForm.templates.mainLayout({
-            loadingIndicatorImagePath   : '/static/images/' + this.loadingIndicatorImage,
-            message                     : ''
-        }));
-
-        // render the tab content
-        this.$dataControl   = this._render_dataControl();
-        this.$chartControl  = this._render_chartControl();
-        this.$statsDisplay  = this.$el.find( '.tab-pane#stats-display' );
-        this.$chartDisplay  = this._render_chartDisplay();
-
-        // auto render if given both x, y column choices in query for page
-        //TODO:?? add autoRender=1 to query maybe?
-        if( this.chartConfig.xColumn && this.chartConfig.yColumn ){
-            this.renderChart();
-        }
-
-        // set up behaviours
-        this.$el.find( '.tooltip' ).tooltip();
-
-        // uncomment any of the following to have that tab show on initial load (for testing)
-        //this.$el.find( 'ul.nav' ).find( 'a[href="#data-control"]' ).tab( 'show' );
-        //this.$el.find( 'ul.nav' ).find( 'a[href="#chart-control"]' ).tab( 'show' );
-        //this.$el.find( 'ul.nav' ).find( 'a[href="#stats-display"]' ).tab( 'show' );
-        //this.$el.find( 'ul.nav' ).find( 'a[href="#chart-display"]' ).tab( 'show' );
-        return this;
-    },
-
-    _render_dataControl : function(){
-        // controls for which columns are used to plot datapoints (and ids/additional info to attach if desired)
-        var view = this,
-            allColumns = [],
-            numericColumns = [],
-            usePossibleHeaders = ( this.possibleHeaders && this.$dataControl )?
-                ( this.$dataControl.find( '#first-line-header-checkbox' ).is( ':checked' ) ):( false );
-
-        // gather column indeces (from metadata_column_types) and names (from metadata_columnnames)
-        _.each( this.dataset.metadata_column_types, function( type, index ){
-            // use a 1 based index in names/values within the form (will be dec. when parsed out)
-            var oneBasedIndex = index + 1,
-                // default name is 'column <index>'...
-                name = 'column ' + oneBasedIndex;
-
-            // ...but label with the name if available...
-            if( view.dataset.metadata_column_names ){
-                name = view.dataset.metadata_column_names[ index ];
-
-            // ...or, use the first line as headers if the user wants
-            } else if( usePossibleHeaders ){
-                name = view.possibleHeaders[ index ];
-            }
-            
-            // cache all columns here
-            allColumns.push({ index: oneBasedIndex, name: name });
-
-            // filter numeric columns to their own list
-            if( type === 'int' || type === 'float' ){
-                numericColumns.push({ index: oneBasedIndex, name: name });
-            }
-        });
-        //TODO: other vals: max_vals, start_val, pagination (chart-settings)
-            
-        // render the html
-        var $dataControl = this.$el.find( '.tab-pane#data-control' );
-        $dataControl.html( ScatterplotControlForm.templates.dataControl({
-            allColumns      : allColumns,
-            numericColumns  : numericColumns,
-            possibleHeaders : ( this.possibleHeaders )?( this.possibleHeaders.join( ', ' ) ):( '' ),
-            usePossibleHeaders : usePossibleHeaders
-        }));
-
-        if( !this.dataset.metadata_column_names && this.possibleHeaders ){
-            $dataControl.find( '#first-line-header' ).show();
-        }
-
-        // preset to column selectors if they were passed in the config in the query string
-        $dataControl.find( '#X-select' ).val( this.chartConfig.xColumn );
-        $dataControl.find( '#Y-select' ).val( this.chartConfig.yColumn );
-        if( this.chartConfig.idColumn !== undefined ){
-            $dataControl.find( '#include-id-checkbox' )
-                .attr( 'checked', true ).trigger( 'change' );
-            $dataControl.find( '#ID-select' ).val( this.chartConfig.idColumn );
-        }
-
-        return $dataControl;
-    },
-    
-    _render_chartControl : function(){
-        // tab content to control how the chart is rendered (data glyph size, chart size, etc.)
-        var view = this,
-            $chartControl = this.$el.find( '.tab-pane#chart-control' ),
-            // limits for controls (by control/chartConfig id)
-            //TODO: move into TwoVarScatterplot
-            controlRanges = {
-                'datapointSize' : { min: 2, max: 10, step: 1 },
-                'width'         : { min: 200, max: 800, step: 20 },
-                'height'        : { min: 200, max: 800, step: 20 }
-            };
-
-        // render the html
-        $chartControl.append( ScatterplotControlForm.templates.chartControl( this.chartConfig ) );
-
-        // set up behaviours, js on sliders
-        $chartControl.find( '.numeric-slider-input' ).each( function(){
-            var $this = $( this ),
-                $output = $this.find( '.slider-output' ),
-                $slider = $this.find( '.slider' ),
-                id = $this.attr( 'id' );
-            //chartControl.log( 'slider set up', 'this:', $this, 'slider:', $slider, 'id', id );
-
-            // what to do when the slider changes: update display and update chartConfig
-            //TODO: move out of loop
-            function onSliderChange(){
-                var $this = $( this ),
-                    newValue = $this.slider( 'value' );
-                //chartControl.log( 'slider change', 'this:', $this, 'output:', $output, 'value', newValue );
-                $output.text( newValue );
-                //chartControl.chartConfig[ id ] = newValue;
-            }
-
-            $slider.slider( _.extend( controlRanges[ id ], {
-                value   : view.chartConfig[ id ],
-                change  : onSliderChange,
-                slide   : onSliderChange
-            }));
-        });
-
-        return $chartControl;
-    },
-
-    _render_chartDisplay : function(){
-        // render the tab content where the chart is displayed (but not the chart itself)
-        var $chartDisplay = this.$el.find( '.tab-pane#chart-display' );
-        $chartDisplay.append( ScatterplotControlForm.templates.chartDisplay( this.chartConfig ) );
-        return $chartDisplay;
-    },
-
-    // ------------------------------------------------------------------------- EVENTS
-    events : {
-        'change #include-id-checkbox'          : 'toggleThirdColumnSelector',
-        'change #first-line-header-checkbox'   : 'rerenderDataControl',
-        'click #data-control #render-button'   : 'renderChart',
-        'click #chart-control #render-button'  : 'changeChartSettings'
-    },
-
-    toggleThirdColumnSelector : function(){
-        // show/hide the id selector on the data settings panel
-        this.$el.find( 'select[name="ID"]' ).parent().toggle();
-    },
-
-    rerenderDataControl : function(){
-        this.$dataControl = this._render_dataControl();
-    },
-    
-    showLoadingIndicator : function( message, callback ){
-        // display the loading indicator over the tab panels if hidden, update message (if passed)
-        message = message || '';
-        var indicator = this.$el.find( 'div#loading-indicator' );
-            messageBox = indicator.find( '.loading-message' );
-            
-        if( indicator.is( ':visible' ) ){
-            if( message ){
-                messageBox.fadeOut( 'fast', function(){
-                    messageBox.text( message );
-                    messageBox.fadeIn( 'fast', callback );
-                });
-            } else {
-                callback();
-            }
-            
-        } else {
-            if( message ){ messageBox.text( message ); }
-            indicator.fadeIn( 'fast', callback );
-        }
-    },
-
-    hideLoadingIndicator : function( callback ){
-        this.$el.find( 'div#loading-indicator' ).fadeOut( 'fast', callback );
-    },
-    
-    // ------------------------------------------------------------------------- CHART/STATS RENDERING
-    renderChart : function(){
-        // fetch the data, (re-)render the chart
-        this.log( this + '.renderChart' );
-
-        //TODO: separate data fetch
-
-        // this is a complete re-render, so clear the prev. data
-        this.data = null;
-        this.meta = null;
-        
-        // update the chartConfig (here and chart) using chart settings
-        //TODO: separate and improve (used in changeChartSettings too)
-        _.extend( this.chartConfig, this.getChartSettings() );
-        this.log( '\t chartConfig:', this.chartConfig );
-        this.chart.updateConfig( this.chartConfig, false );
-        
-        // build the url with the current data settings
-        this.loader.url = this.dataURL + '&' + jQuery.param( this.getDataSettings() );
-        this.log( '\t loader: total lines:', this.loader.total, ' url:', this.loader.url );
-
-        // bind the new data event to: aggregate data, update the chart and stats with new data
-        var view = this;
-        $( this.loader ).bind( 'loaded.new', function( event, response ){
-            view.log( view + ' loaded.new', response );
-
-            // aggregate data and meta
-            view.postProcessDataFetchResponse( response );
-            view.log( '\t postprocessed data:', view.data );
-            view.log( '\t postprocessed meta:', view.meta );
-
-            // update the chart and stats
-            view.showLoadingIndicator( view.renderMsg, function(){
-                view.chart.render( view.data, view.meta );
-                view.renderStats( view.data, view.meta );
-                view.hideLoadingIndicator();
-            });
-        });
-        // when all data loaded - unbind (or we'll start doubling event handlers)
-        $( this.loader ).bind( 'complete', function( event, data ){
-            view.log( view + ' complete', data );
-            $( view.loader ).unbind();
-        });
-
-        // begin loading the data, switch to the chart display tab
-        view.showLoadingIndicator( view.fetchMsg, function(){
-            view.$el.find( 'ul.nav' ).find( 'a[href="#chart-display"]' ).tab( 'show' );
-            view.loader.load();
-        });
-    },
-
-    renderStats : function(){
-        this.log( this + '.renderStats' );
-        // render the stats table in the stats panel
-        //TODO: there's a better way
-        this.$statsDisplay.html( ScatterplotControlForm.templates.statsDisplay({
-            stats:  [
-                { name: 'Count',    xval: this.meta[0].count,   yval: this.meta[1].count },
-                { name: 'Min',      xval: this.meta[0].min,     yval: this.meta[1].min },
-                { name: 'Max',      xval: this.meta[0].max,     yval: this.meta[1].max },
-                { name: 'Sum',      xval: this.meta[0].sum,     yval: this.meta[1].sum },
-                { name: 'Mean',     xval: this.meta[0].mean,    yval: this.meta[1].mean },
-                { name: 'Median',   xval: this.meta[0].median,  yval: this.meta[1].median }
-            ]
-        }));
-    },
-
-    changeChartSettings : function(){
-        // re-render the chart with new chart settings and OLD data
-        var view = this;
-            newChartSettings = this.getChartSettings();
-
-        // update the chart config from the chartSettings panel controls
-        _.extend( this.chartConfig, newChartSettings );
-        this.log( 'this.chartConfig:', this.chartConfig );
-        this.chart.updateConfig( this.chartConfig, false );
-
-        // if there's current data, call chart.render with it (no data fetch)
-        if( view.data && view.meta ){
-            view.showLoadingIndicator( view.renderMsg, function(){
-                view.$el.find( 'ul.nav' ).find( 'a[href="#chart-display"]' ).tab( 'show' );
-                view.chart.render( view.data, view.meta );
-                view.hideLoadingIndicator();
-            });
-
-        // no current data, call renderChart instead (which will fetch data)
-        } else {
-            this.renderChart();
-        }
-    },
-
-    // ------------------------------------------------------------------------- DATA AGGREGATION
-    postProcessDataFetchResponse : function( response ){
-        // the loader only returns new data - it's up to this to munge the fetches together properly
-        //TODO: we're now storing data in two places: loader and here
-        //  can't we reduce incoming data into loader.data[0]? are there concurrency problems?
-        this.postProcessData( response.data );
-        this.postProcessMeta( response.meta );
-    },
-    
-    postProcessData : function( newData ){
-        // stack the column data on top of each other into this.data
-        //this.log( this + '.postProcessData:', newData );
-        var view = this;
-
-        // if we already have data: aggregate
-        if( view.data ){
-            _.each( newData, function( newColData, colIndex ){
-                //view.log( colIndex + ' data:', newColData );
-                //TODO??: time, space efficiency of this?
-                view.data[ colIndex ] = view.data[ colIndex ].concat( newColData );
-            });
-            
-        // otherwise: assign (first load)
-        } else {
-            view.data = newData;
-        }
-    },
-
-    postProcessMeta : function( newMeta ){
-        // munge the meta data (stats) from the server fetches together
-        //pre: this.data must be preprocessed (needed for medians)
-        //this.log( this + '.postProcessMeta:', newMeta );
-        var view = this,
-            colTypes = this.dataset.metadata_column_types;
-
-        // if we already have meta: aggregate
-        if( view.meta ){
-            _.each( newMeta, function( newColMeta, colIndex ){
-                var colMeta = view.meta[ colIndex ],
-                    colType = colTypes[ colIndex ];
-                //view.log( '\t ' + colIndex + ' postprocessing meta:', newColMeta );
-                //view.log( colIndex + ' old meta:',
-                //    'min:',     colMeta.min,
-                //    'max:',     colMeta.max,
-                //    'sum:',     colMeta.sum,
-                //    'mean:',    colMeta.mean,
-                //    'median:',  colMeta.median
-                //);
-
-                //!TODO: at what point are we getting int/float overflow on these?!
-                //??: need to be null safe?
-                colMeta.count += ( newColMeta.count )?( newColMeta.count ):( 0 );
-                //view.log( colIndex, 'count:', colMeta.count );
-                
-                if( ( colType === 'int' ) || ( colType === 'float' ) ){
-                    //view.log( colIndex + ' incoming meta:',
-                    //    'min:',     newColMeta.min,
-                    //    'max:',     newColMeta.max,
-                    //    'sum:',     newColMeta.sum,
-                    //    'mean:',    newColMeta.mean,
-                    //    'median:',  newColMeta.median
-                    //);
-
-                    colMeta.min  = Math.min( newColMeta.min, colMeta.min );
-                    colMeta.max  = Math.max( newColMeta.max, colMeta.max );
-                    colMeta.sum  = newColMeta.sum + colMeta.sum;
-                    colMeta.mean = ( colMeta.count )?( colMeta.sum / colMeta.count ):( null );
-
-                    // median's a pain bc of sorting (requires the data as well)
-                    var sortedCol = view.data[ colIndex ].slice().sort(),
-                        middleIndex = Math.floor( sortedCol.length / 2 );
-
-                    if( sortedCol.length % 2 === 0 ){
-                        colMeta.median = ( ( sortedCol[ middleIndex ] + sortedCol[( middleIndex + 1 )] ) / 2 );
-
-                    } else {
-                        colMeta.median = sortedCol[ middleIndex ];
-                    }
-
-                    //view.log( colIndex + ' new meta:',
-                    //    'min:',     colMeta.min,
-                    //    'max:',     colMeta.max,
-                    //    'sum:',     colMeta.sum,
-                    //    'mean:',    colMeta.mean,
-                    //    'median:',  colMeta.median
-                    //);
-                }
-            });
-
-        // otherwise: assign (first load)
-        } else {
-            view.meta = newMeta;
-            //view.log( '\t meta (first load):', view.meta );
-        }
-    },
-
-    // ------------------------------------------------------------------------- GET DATA/CHART SETTINGS
-    getDataSettings : function(){
-        // parse the column values for both indeces (for the data fetch) and names (for the chart)
-        var columnSelections = this.getColumnSelections(),
-            columns = [];
-        this.log( '\t columnSelections:', columnSelections );
-            
-        //TODO: validate columns - minimally: we can assume either set by selectors or via a good query string
-
-        // get column indices for params, include the desired ID column (if any)
-        //NOTE: these are presented in human-readable 1 base index (to match the data.peek) - adjust
-        columns = [
-            columnSelections.X.colIndex - 1,
-            columnSelections.Y.colIndex - 1
-        ];
-        if( this.$dataControl.find( '#include-id-checkbox' ).attr( 'checked' ) ){
-            columns.push( columnSelections.ID.colIndex - 1 );
-        }
-        //TODO: other vals: max, start, page
-        
-        var params = {
-            data_type   : 'raw_data',
-            provider    : 'column_with_stats',
-            columns     : '[' + columns + ']'
-        };
-        this.log( '\t data settings (url params):', params );
-        return params;
-    },
-
-    getColumnSelections : function(){
-        // gets the current user-selected values for which columns to fetch from the data settings panel
-        // returns a map: { column-select name (eg. X) : { colIndex : column-selector val,
-        //                                                 colName : selected option text }, ... }
-        var selections = {};
-        this.$dataControl.find( 'div.column-select select' ).each( function(){
-            var $this   = $( this ),
-                val     = $this.val();
-            selections[ $this.attr( 'name' ) ] = {
-                colIndex : val,
-                colName  : $this.children( '[value="' + val + '"]' ).text()
-            };
-        });
-        return selections;
-    },
-
-    getChartSettings : function(){
-        // gets the user-selected chartConfig from the chart settings panel
-        var settings = {},
-            colSelections = this.getColumnSelections();
-        //this.log( 'colSelections:', colSelections );
-
-        //TODO: simplify with keys and loop
-        settings.datapointSize = this.$chartControl.find( '#datapointSize.numeric-slider-input' )
-            .find( '.slider' ).slider( 'value' );
-        settings.width = this.$chartControl.find( '#width.numeric-slider-input' )
-            .find( '.slider' ).slider( 'value' );
-        settings.height = this.$chartControl.find( '#height.numeric-slider-input' )
-            .find( '.slider' ).slider( 'value' );
-
-        // update axes labels using chartSettings inputs (if not at defaults), otherwise the selects' colName
-        //TODO: a little confusing
-        var chartSettingsXLabel = this.$chartControl.find( 'input#X-axis-label' ).val(),
-            chartSettingsYLabel = this.$chartControl.find( 'input#Y-axis-label' ).val();
-        settings.xLabel = ( chartSettingsXLabel === 'X' )?
-                            ( colSelections.X.colName ):( chartSettingsXLabel );
-        settings.yLabel = ( chartSettingsYLabel === 'Y' )?
-                            ( colSelections.Y.colName ):( chartSettingsYLabel );
-
-        settings.animDuration = ( this.$chartControl.find( '#animate-chart' ).is( ':checked' ) )?
-            ( this.chart.defaults.animDuration ):( 0 );
-
-        this.log( '\t chartSettings:', settings );
-        return settings;
-    },
-
-    toString : function(){
-        return 'ScatterplotControlForm(' + (( this.dataset )?( this.dataset.id ):( '' )) + ')';
-    }
-});
-
-ScatterplotControlForm.templates = {
-    mainLayout      : Templates.scatterplotControlForm,
-    dataControl     : Templates.dataControl,
-    chartControl    : Templates.chartControl,
-    statsDisplay    : Templates.statsDisplay,
-    chartDisplay    : Templates.chartDisplay
-};
-
-//==============================================================================

config/plugins/visualizations/scatterplot/src/visualization-templates.html

@@ -1,182 +0,0 @@
-<script type="text/template" class="template-visualization" id="template-visualization-scatterplotControlForm">
-{{! main layout }}
-
-<div class="scatterplot-container chart-container tabbable tabs-left">
-    {{! tab buttons/headers using Bootstrap }}
-    <ul class="nav nav-tabs">
-        {{! start with the data controls as the displayed tab }}
-        <li class="active"><a href="#data-control" data-toggle="tab" class="tooltip"
-            title="Use this tab to change which data are used">Data Controls</a></li>
-        <li><a href="#chart-control" data-toggle="tab" class="tooltip"
-            title="Use this tab to change how the chart is drawn">Chart Controls</a></li>
-        <li><a href="#stats-display" data-toggle="tab" class="tooltip"
-            title="This tab will display overall statistics for your data">Statistics</a></li>
-        <li><a href="#chart-display" data-toggle="tab" class="tooltip"
-            title="This tab will display the chart">Chart</a>
-            {{! loading indicator - initially hidden }}
-            <div id="loading-indicator" style="display: none;">
-                <img class="loading-img" src="{{loadingIndicatorImagePath}}" />
-                <span class="loading-message">{{message}}</span>
-            </div>
-        </li>
-    </ul>
-
-    {{! data form, chart config form, stats, and chart all get their own tab }}
-    <div class="tab-content">
-        {{! ---------------------------- tab for data settings form }}
-        <div id="data-control" class="tab-pane active">
-            {{! rendered separately }}
-        </div>
-    
-        {{! ---------------------------- tab for chart graphics control form }}
-        <div id="chart-control" class="tab-pane">
-            {{! rendered separately }}
-        </div>
-
-        {{! ---------------------------- tab for data statistics }}
-        <div id="stats-display" class="tab-pane">
-            {{! rendered separately }}
-        </div>
-
-        {{! ---------------------------- tab for actual chart }}
-        <div id="chart-display" class="tab-pane">
-            {{! chart rendered separately }}
-        </div>
-
-    </div>{{! end .tab-content }}
-</div>{{! end .chart-control }}
-</script>
-
-<script type="text/template" class="template-visualization" id="template-visualization-dataControl">
-
-    <p class="help-text">
-        Use the following controls to change the data used by the chart.
-        Use the 'Draw' button to render (or re-render) the chart with the current settings.
-    </p>
-
-    {{! column selector containers }}
-    <div class="column-select">
-        <label for="X-select">Data column for X: </label>
-        <select name="X" id="X-select">
-        {{#each numericColumns}}
-            <option value="{{index}}">{{name}}</option>
-        {{/each}}
-        </select>
-    </div>
-    <div class="column-select">
-        <label for="Y-select">Data column for Y: </label>
-        <select name="Y" id="Y-select">
-        {{#each numericColumns}}
-            <option value="{{index}}">{{name}}</option>
-        {{/each}}
-        </select>
-    </div>
-
-    {{! optional id column }}
-    <div id="include-id">
-        <label for="include-id-checkbox">Include a third column as data point IDs?</label>
-        <input type="checkbox" name="include-id" id="include-id-checkbox" />
-        <p class="help-text-small">
-            These will be displayed (along with the x and y values) when you hover over
-            a data point.
-        </p>
-    </div>
-    <div class="column-select" style="display: none">
-        <label for="ID-select">Data column for IDs: </label>
-        <select name="ID" id="ID-select">
-        {{#each allColumns}}
-            <option value="{{index}}">{{name}}</option>
-        {{/each}}
-        </select>
-    </div>
-
-    {{! if we're using generic column selection names ('column 1') - allow the user to use the first line }}
-    <div id="first-line-header" style="display: none;">
-        <p>Possible headers: {{ possibleHeaders }}
-        </p>
-        <label for="first-line-header-checkbox">Use the above as column headers?</label>
-        <input type="checkbox" name="include-id" id="first-line-header-checkbox"
-            {{#if usePossibleHeaders }}checked="true"{{/if}}/>
-        <p class="help-text-small">
-            It looks like Galaxy couldn't get proper column headers for this data.
-            Would you like to use the column headers above as column names to select columns?
-        </p>
-    </div>
-
-    <input id="render-button" type="button" value="Draw" />
-    <div class="clear"></div>
-</script>
-
-<script type="text/template" class="template-visualization" id="template-visualization-chartControl">
-    <p class="help-text">
-        Use the following controls to how the chart is displayed.
-        The slide controls can be moved by the mouse or, if the 'handle' is in focus, your keyboard's arrow keys.
-        Move the focus between controls by using the tab or shift+tab keys on your keyboard.
-        Use the 'Draw' button to render (or re-render) the chart with the current settings.
-    </p>
-
-    <div id="datapointSize" class="form-input numeric-slider-input">
-        <label for="datapointSize">Size of data point: </label>
-        <div class="slider-output">{{datapointSize}}</div>
-        <div class="slider"></div>
-        <p class="form-help help-text-small">
-            Size of the graphic representation of each data point
-        </p>
-    </div>
-
-    <div id="animDuration" class="form-input checkbox-input">
-        <label for="animate-chart">Animate chart transitions?: </label>
-        <input type="checkbox" id="animate-chart"
-            class="checkbox control"{{#if animDuration}} checked="true"{{/if}} />
-        <p class="form-help help-text-small">
-            Uncheck this to disable the animations used on the chart
-        </p>
-    </div>
-
-    <div id="width" class="form-input numeric-slider-input">
-        <label for="width">Chart width: </label>
-        <div class="slider-output">{{width}}</div>
-        <div class="slider"></div>
-        <p class="form-help help-text-small">
-            (not including chart margins and axes)
-        </p>
-    </div>
-
-    <div id="height" class="form-input numeric-slider-input">
-        <label for="height">Chart height: </label>
-        <div class="slider-output">{{height}}</div>
-        <div class="slider"></div>
-        <p class="form-help help-text-small">
-            (not including chart margins and axes)
-        </p>
-    </div>
-
-    <div id="X-axis-label"class="text-input form-input">
-        <label for="X-axis-label">Re-label the X axis: </label>
-        <input type="text" name="X-axis-label" id="X-axis-label" value="{{xLabel}}" />
-        <p class="form-help help-text-small"></p>
-    </div>
-
-    <div id="Y-axis-label" class="text-input form-input">
-        <label for="Y-axis-label">Re-label the Y axis: </label>
-        <input type="text" name="Y-axis-label" id="Y-axis-label" value="{{yLabel}}" />
-        <p class="form-help help-text-small"></p>
-    </div>
-
-    <input id="render-button" type="button" value="Draw" />
-</script>
-
-<script type="text/template" class="template-visualization" id="template-visualization-statsDisplay">
-    <p class="help-text">By column:</p>
-    <table id="chart-stats-table">
-        <thead><th></th><th>X</th><th>Y</th></thead>
-        {{#each stats}}
-        <tr><td>{{name}}</td><td>{{xval}}</td><td>{{yval}}</td></tr>
-        </tr>
-        {{/each}}
-    </table>
-</script>
-
-<script type="text/template" class="template-visualization" id="template-visualization-chartDisplay">
-    <svg width="{{width}}" height="{{height}}"></svg>
-</script>

config/plugins/visualizations/scatterplot/static/numeric-column-stats.js

@@ -0,0 +1,89 @@
+var numericColumnStats = function( data, keys ){
+    var console = console || { debug: function(){} },
+        stats = {};
+
+    // precondition: keys is an array of keys accessible on each data in data
+    if( !keys || !keys.length ){
+        throw new Error( 'keys is a required parameter and must be an array:' + keys );
+    }
+    if( !data || !data.length ){
+        return stats;
+    }
+
+    function parseVal( val ){
+        if( val === null ){
+            throw new Error( 'Null value' );
+        }
+        val = Number( val );
+        if( isNaN( val ) || !isFinite( val ) ){
+            throw new Error( 'NaN or non-finite number' );
+        }
+        return val;
+    }
+
+    // does this effect the data in the other thread?
+    [ 'min', 'max', 'sum', 'mean', 'median', 'count' ].forEach( function( name, i ){
+        stats[ name ] = new Array( keys.length );
+    });
+
+    var keyIndex = 0,
+        separatedCols = new Array( keys.length );
+    keys.forEach( function( key, keyIndex ){
+        stats.min[ keyIndex ] = 0;
+        stats.max[ keyIndex ] = 0;
+        stats.sum[ keyIndex ] = 0;
+        separatedCols[ keyIndex ] = [];
+    });
+
+    // work backwards to prevent co-modification problems while splitting up data into columns
+    for( var dataIndex=( data.length - 1 ); dataIndex>=0; dataIndex-=1 ){
+        var datum = data.pop( dataIndex );
+
+        for( keyIndex=0; keyIndex<keys.length; keyIndex+=1 ){
+            var key = keys[ keyIndex ],
+                datumColVal = datum[ key ];
+
+            try {
+                //NOTE: parsing value as number
+                datumColVal = parseVal( datumColVal );
+            } catch( e ){
+                continue;
+            }
+
+            // separate the columns
+            separatedCols[ keyIndex ].unshift( datumColVal );
+            // get the other stats
+            stats.min[ keyIndex ] = Math.min( stats.min[ keyIndex ], datumColVal );
+            stats.max[ keyIndex ] = Math.max( stats.max[ keyIndex ], datumColVal );
+            stats.sum[ keyIndex ] += datumColVal;
+        }
+    }
+
+    // get counts, mean, median
+    function comparator( a, b ){
+        if( a < b ){ return -1; }
+        if( a < b ){ return 1; }
+        return 0;
+    }
+
+    for( keyIndex=0; keyIndex<keys.length; keyIndex+=1 ){
+        var count = separatedCols[ keyIndex ].length,
+            sum = stats.sum[ keyIndex ];
+        stats.count[ keyIndex ] = count;
+        stats.mean[ keyIndex ] = ( sum / count );
+
+        // sort columns for median
+        separatedCols[ keyIndex ].sort( comparator );
+
+        // odd count -> straight forward median
+        var middleDataIndex = Math.floor( count / 2 );
+        if( count % 2 === 1 ){
+            stats.median[ keyIndex ] = separatedCols[ keyIndex ][ middleDataIndex ];
+        } else {
+            var middleValA = separatedCols[ keyIndex ][ middleDataIndex ],
+                middleValB = separatedCols[ keyIndex ][( middleDataIndex + 1 )];
+            stats.median[ keyIndex ] = ( middleValA + middleValB ) / 2;
+        }
+    }
+    return stats;
+};

config/plugins/visualizations/scatterplot/static/scatterplot.css

@@ -1,171 +1,213 @@
-/*TODO: use/move into base.less*/
-* { margin: 0px; padding: 0px; }
-
-/* -------------------------------------------- general layout */
-div.tab-pane {
-    padding: 8px;
-}
+* { margin: 0; padding: 0; }
 
 /* -------------------------------------------- header */
-.header {
-    margin-bottom: 8px;
-}
+.chart-header > * { margin: 0; padding: 0; }
 
-#chart-header {
-    padding : 8px;
-    background-color: #ebd9b2;
+.chart-header {
     margin-bottom: 16px;
     overflow: auto;
+    background-color: #ebd9b2;
+    padding : 16px 12px 12px 12px;
 }
 
-#chart-header .subtitle {
-    margin: -4px 0px 0px 4px;
-    padding : 0;
+.chart-header h2 {
+    margin-bottom: 4px;
+}
+
+.chart-header p {
     color: white;
     font-size: small;
 }
 
 /* -------------------------------------------- main layout */
-#scatterplot {
-    /*from width + margin of chart?*/
+.scatterplot-editor > * { margin: 0; padding: 0; }
+.scatterplot-editor .tab-pane {
+    padding: 8px;
 }
 
-.scatterplot-container .tab-pane {
+.scatterplot-editor .help-text,
+.scatterplot-editor .help-text-small {
+    color: grey;
+    margin-bottom: 8px;
 }
 
-/* -------------------------------------------- all controls */
+.scatterplot-editor .help-text {
+    margin-top: 4px;
+}
 
-#scatterplot input[type=button],
-#scatterplot select {
+.scatterplot-editor .help-text-small {
+    font-size: smaller;
+}
+
+.scatterplot-editor .file-controls {
+    float: right;
+    /*margin-right: 16px;*/
+}
+
+.file-controls .btn {
+    height: 24px;
+    padding: 0px 10px 0px 10px;
+    line-height: 24px;
+}
+
+.file-controls .copy-btn {
+    display: none;
+}
+
+
+/* ============================================ config controls */
+.scatterplot-config-control {
+    max-width: 768px;
+}
+
+.scatterplot-config-control input[type="button"],
+.scatterplot-config-control button,
+.scatterplot-config-control select {
     width: 100%;
     max-width: 256px;
     margin-bottom: 8px;
 }
-
-#scatterplot .help-text,
-#scatterplot .help-text-small {
-    color: grey;
+.scatterplot-config-control input[type="text"] {
+    width: 100%;
+    max-width: 512px;
+    border: 1px solid lightgrey;
 }
 
-#scatterplot .help-text {
-    padding-bottom: 16px;
+.scatterplot-config-control input[type="checkbox"] {
+    display: inline-block;
+    margin-top: -3px;
 }
 
-#scatterplot .help-text-small {
-    padding: 4px;
-    font-size: smaller;
-}
-
-#scatterplot > * {
-}
-
-#scatterplot input[value=Draw] {
+.scatterplot-config-control .render-button {
     display: block;
     margin-top: 16px;
 }
 
-#scatterplot .numeric-slider-input {
-    max-width: 70%;
-}
-
-/* -------------------------------------------- data controls */
-
-/* -------------------------------------------- chart controls */
-#chart-control .form-input {
-    /*display: table-row;*/
-}
-
-#chart-control label {
-    /*text-align: right;*/
+.scatterplot-config-control .form-input {
     margin-bottom: 8px;
-    /*display: table-cell;*/
 }
 
-#chart-control .slider {
-    /*display: table-cell;*/
+.scatterplot-config-control .slider {
     height: 8px;
     display: block;
-    margin: 8px 0px 0px 8px;
+    margin-bottom: 8px;
 }
 
-#chart-control .slider-output {
-    /*display: table-cell;*/
+.scatterplot-config-control .slider-output {
     float: right;
 }
 
-#chart-control input[type="text"] {
-    border: 1px solid lightgrey;
-}
-
-
-/* -------------------------------------------- statistics */
-#stats-display table#chart-stats-table {
-    width: 100%;
-}
-
-#stats-display #chart-stats-table th {
-    width: 30%;
-    padding: 4px;
-    font-weight: bold;
-    color: grey;
-}
-
-#stats-display #chart-stats-table td {
-    border: solid lightgrey;
-    border-width: 1px 0px 0px 1px;
-    padding: 4px;
-}
-
-#stats-display #chart-stats-table td:nth-child(1) {
-    border-width: 1px 0px 0px 0px;
-    padding-right: 1em;
-    text-align: right;
-    font-weight: bold;
-    color: grey;
-}
-
+/* ============================================ plot display */
 /* -------------------------------------------- load indicators */
-#loading-indicator {
-    margin: 12px 0px 0px 8px;
+.scatterplot-display .loading-indicator {
+    color: grey;
 }
 
-#scatterplot #loading-indicator .loading-message {
+.scatterplot-display .loading-indicator .loading-indicator-message {
+    display: inline-block;
+    margin-left: 4px;
     font-style: italic;
-    font-size: smaller;
-    color: grey;
+}
+
+/* -------------------------------------------- display controls */
+.scatterplot-display .controls {
+    width: 100%;
+    margin-bottom: 4px;
+    border: 1px solid lightgrey;
+    border-radius: 3px;
+    padding: 8px 8px 4px 8px;
+}
+
+.scatterplot-display .controls button {
+    display: inline-block;
+    min-width: 0;
+    width: auto;
+    margin: 0px;
+}
+
+.scatterplot-display .controls .left {
+    float: left;
+}
+
+.scatterplot-display .controls .pagination {
+    display: inline-block;
+    margin: 0px;
+}
+
+.scatterplot-display .controls .data-prev-next {
+    display: inline-block;
+    margin: 0px 4px 0px 0px;
+}
+
+.scatterplot-display .controls .pagination-container {
+    display: inline-block;
+    overflow: auto;
+    max-width: 256px;
+    /*very tweaky*/
+    margin: 0px 8px 0px 0px;
+    height: 27px;
+    padding-top: 5px;
+}
+.scatterplot-display .controls .pagination-container a {
+    display: inline;
+    /* WTH, bootstrap? */
+    position: static;
+    float: none;
+    /* up from 4px -> 5px */
+    padding: 5px 10px 5px 10px;
+}
+
+.scatterplot-display .controls .right {
+    float: right;
+    margin-bottom: 4px;
+}
+
+.scatterplot-display .controls p {
+    display: inline-block;
+    margin: 0px 4px 0px 4px;
+    /*to better match the buttons*/
+    vertical-align: middle;
+    line-height: 24px;
+}
+
+.scatterplot-display button.rerender-btn {
+    margin-left: 4px;
+}
+
+.scatterplot-display button.stats-toggle-btn {
+    margin-left: 4px;
 }
 
 /* -------------------------------------------- chart area */
-#chart-holder {
-    overflow: auto;
-    margin-left: 8px;
+svg.scatterplot {
+    border: 1px solid lightgrey;
+    border-radius: 3px;
 }
 
-svg .grid-line {
+svg.scatterplot .grid-line {
     fill: none;
     stroke: lightgrey;
-    stroke-opacity: 0.5;
+    stroke-opacity: 0.2;
     shape-rendering: crispEdges;
-    stroke-dasharray: 3, 3;
 }
 
-svg .axis path, svg .axis line {
+svg.scatterplot .axis path,
+svg.scatterplot .axis line {
     fill: none;
     stroke: black;
     shape-rendering: crispEdges;
 }
 
-svg .axis text {
-    font-family: monospace;
-    font-size: 12px;
+svg.scatterplot .axis text {
+    font-size: 9px;
 }
 
-svg #x-axis-label, svg #y-axis-label {
+svg.scatterplot .axis-label {
     font-family: sans-serif;
-    font-size: 10px;
+    font-size: 13px;
 }
 
-svg .glyph {
+svg.scatterplot .glyph {
     stroke: none;
     fill: black;
     fill-opacity: 0.2;

config/plugins/visualizations/scatterplot/static/worker-stats.js

@@ -0,0 +1,5 @@
+onmessage = function( event ){
+    importScripts( 'numeric-column-stats.js' );
+    postMessage( numericColumnStats( event.data.data, event.data.keys ) );
+    self.close();
+};

config/plugins/visualizations/scatterplot/templates/scatterplot.mako

@@ -1,8 +1,19 @@
+<%
+    default_title = "Scatterplot of '" + hda.name + "'"
+    info = hda.name
+    if hda.info:
+        info += ' : ' + hda.info
+
+    # optionally bootstrap data from dprov
+    ##data = list( hda.datatype.dataset_column_dataprovider( hda, limit=10000 ) )
+%>
+## ----------------------------------------------------------------------------
+
 <!DOCTYPE HTML>
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-<title>${hda.name} | ${visualization_name}</title>
+<title>${title or default_title} | ${visualization_display_name}</title>
 
 ## ----------------------------------------------------------------------------
 <link type="text/css" rel="Stylesheet" media="screen" href="/static/style/base.css">
@@ -13,55 +24,50 @@
 ## ----------------------------------------------------------------------------
 <script type="text/javascript" src="/static/scripts/libs/jquery/jquery.js"></script>
 <script type="text/javascript" src="/static/scripts/libs/jquery/jquery.migrate.js"></script>
+<script type="text/javascript" src="/static/scripts/libs/jquery/jquery-ui.js"></script>
+<script type="text/javascript" src="/static/scripts/libs/bootstrap.js"></script>
 <script type="text/javascript" src="/static/scripts/libs/underscore.js"></script>
 <script type="text/javascript" src="/static/scripts/libs/backbone/backbone.js"></script>
-<script type="text/javascript" src="/static/scripts/libs/backbone/backbone-relational.js"></script>
 <script type="text/javascript" src="/static/scripts/libs/handlebars.runtime.js"></script>
 <script type="text/javascript" src="/static/scripts/libs/d3.js"></script>
-<script type="text/javascript" src="/static/scripts/libs/bootstrap.js"></script>
-<script type="text/javascript" src="/static/scripts/libs/jquery/jquery-ui.js"></script>
-<script type="text/javascript" src="/static/scripts/utils/LazyDataLoader.js"></script>
+
 <script type="text/javascript" src="/static/scripts/mvc/base-mvc.js"></script>
+<script type="text/javascript" src="/static/scripts/mvc/visualization/visualization-model.js"></script>
 
-<script type="text/javascript" src="/plugins/visualizations/scatterplot/static/scatterplot.js"></script>
-
+<script type="text/javascript" src="/plugins/visualizations/scatterplot/static/scatterplot-edit.js"></script>
 </head>
 
 ## ----------------------------------------------------------------------------
 <body>
 %if not embedded:
 ## dataset info: only show if on own page
-<div id="chart-header" class="header">
-    <h2 class="title">Scatterplot of '${hda.name}'</h2>
-    <p class="subtitle">${hda.info}</p>
+<div class="chart-header">
+    <h2>${title or default_title}</h2>
+    <p>${info}</p>
 </div>
-%endif
-
-<div id="scatterplot" class="scatterplot-control-form"></div>
 
+<div class="scatterplot-editor"></div>
 <script type="text/javascript">
 $(function(){
-    var hda             = ${h.to_json_string( trans.security.encode_dict_ids( hda.to_dict() ) )},
-        querySettings   = ${h.to_json_string( query_args )},
-        chartConfig     = _.extend( querySettings, {
-            containerSelector : '#chart',
-            //TODO: move to ScatterplotControlForm.initialize
-            marginTop   : ( querySettings.marginTop > 20 )?( querySettings.marginTop ):( 20 ),
-
-            xColumn     : querySettings.xColumn,
-            yColumn     : querySettings.yColumn,
-            idColumn    : querySettings.idColumn
-        });
-    //console.debug( querySettings );
-
-    var settingsForm = new ScatterplotControlForm({
-        dataset         : hda,
-        apiDatasetsURL  : "${h.url_for( controller='/api/datasets', action='index' )}",
-        el              : $( '#scatterplot' ),
-        chartConfig     : chartConfig
-    }).render();
+    var model   = new ScatterplotModel({
+            id      : ${h.to_json_string( visualization_id )} || undefined,
+            title   : "${title or default_title}",
+            config  : ${h.to_json_string( config, indent=2 )}
+        }),
+        hdaJson = ${h.to_json_string( trans.security.encode_dict_ids( hda.to_dict() ), indent=2 )},
+        editor  = new ScatterplotConfigEditor({
+            el      : $( '.scatterplot-editor' ).attr( 'id', 'scatterplot-editor-hda-' + hdaJson.id ),
+            model   : model,
+            dataset : hdaJson
+        }).render();
+    window.editor = editor;
 
+    model.on( 'change:title', function(){
+        $( '.chart-header h2' ).text( model.get( 'title' ) );
+    })
 });
+
 </script>
+%endif
 
 </body>

config/plugins/visualizations/sweepster/config/sweepster.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE visualization SYSTEM "../../visualization.dtd">
-<visualization name="sweepster">
+<visualization name="Sweepster" disabled="true">
     <data_sources>
         <data_source>
             <model_class>HistoryDatasetAssociation</model_class>
@@ -23,5 +23,5 @@
     <!-- template_root and template are currently ignored for the 'built-in' visualizations -->
     <template_root>webapps/galaxy/visualization</template_root>
     <template>sweepster.mako</template>
-    <render_location>_top</render_location>
+    <render_target>_top</render_target>
 </visualization>

config/plugins/visualizations/trackster/config/trackster.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE visualization SYSTEM "../../visualization.dtd">
-<visualization name="trackster">
+<visualization name="Trackster">
     <!--not tested yet -->
     <data_sources>
         <data_source>
@@ -26,5 +26,5 @@
     <!-- template_root and template are currently ignored for the 'built-in' visualizations -->
     <template_root>webapps/galaxy/visualization/tracks</template_root>
     <template>browser.mako</template>
-    <render_location>_top</render_location>
+    <render_target>_top</render_target>
 </visualization>

config/plugins/visualizations/visualization.dtd

@@ -1,10 +1,17 @@
 <!-- each visualization must have a template (all other elements are optional) -->
-<!ELEMENT visualization (data_sources*,params*,template_root*,template,link_text*,render_location*)>
-<!-- visualization name (e.g. 'trackster', 'scatterplot', etc.) is required -->
+<!ELEMENT visualization (description*,data_sources*,params*,template_root*,template,render_target*)>
+<!-- visualization
+        name: the title/display name of the visualization (e.g. 'Trackster', 'Fastq Stats', etc.) REQUIRED
+        disabled: if included (value does not matter), this attribute will prevent the visualization being loaded
+-->
 <!ATTLIST visualization
     name        CDATA #REQUIRED
+    disabled    CDATA #IMPLIED
 >
 
+<!ELEMENT description (#PCDATA)>
+<!-- a text description of what the visualization does -->
+
 <!ELEMENT data_sources (data_source*)>
 <!-- data sources are elements that describe what objects (HDAs, LDDAs, Job, User, etc.)
         are applicable to a visualization. Often these are used to fetch applicable links
@@ -121,10 +128,8 @@
 <!ELEMENT template_root (#PCDATA)>
 <!-- template: the template used to render the visualization. REQUIRED -->
 <!ELEMENT template (#PCDATA)>
-<!-- link_text: the text component of an html anchor displayed when the registry builds the link information -->
-<!ELEMENT link_text (#PCDATA)>
-<!-- render_location: used as the target attribute of the link to the visualization.
+<!-- render_target: used as the target attribute of the link to the visualization.
       Can be 'galaxy_main', '_top', '_blank'. DEFAULT: 'galaxy_main'
 -->
 <!-- TODO: rename -> render_target -->
-<!ELEMENT render_location (#PCDATA)>
+<!ELEMENT render_target (#PCDATA)>

doc/source/lib/galaxy.tools.util.rst

@@ -9,14 +9,6 @@ util Package
     :undoc-members:
     :show-inheritance:
 
-:mod:`hyphy_util` Module
-------------------------
-
-.. automodule:: galaxy.tools.util.hyphy_util
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 :mod:`maf_utilities` Module
 ---------------------------
 

doc/source/lib/galaxy.webapps.galaxy.api.rst

@@ -205,6 +205,11 @@ used in the web form, like 'link_data_only' and etc.
 
 The request and response format should be considered alpha and are subject to change.
 
+API Return Codes and Formats
+==================
+
+A set of error codes for API requests is being established and will be
+documented here. This is a long-term project however so stayed tuned.
 
 API Controllers
 ===============
@@ -297,6 +302,14 @@ API Controllers
     :undoc-members:
     :show-inheritance:
 
+:mod:`lda_datasets` Module
+--------------------------
+
+.. automodule:: galaxy.webapps.galaxy.api.lda_datasets
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 :mod:`libraries` Module
 -----------------------
 
@@ -393,3 +406,67 @@ API Controllers
     :undoc-members:
     :show-inheritance:
 
+
+API Design Guidelines
+=====================
+
+The following section outlines guidelines related to extending and/or modifing
+the Galaxy API. The Galaxy API has grown in an ad-hoc fashion over time by
+many contributors and so clients SHOULD NOT expect the API will conform to
+these guidelines - but developers contributing to the Galaxy API SHOULD follow
+these guidelines.
+
+    - API functionality should include docstring documentation for consumption
+      by readthedocs.org.
+    - Developers should familarize themselves with the HTTP status code definitions
+      http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html. The API responses
+      should properly set the status code according to the result - in particular
+      2XX responses should be used for successful requests, 4XX for various
+      kinds of client errors, and 5XX for errors on the server side.
+    - If there is an error processing some part of request (one item in a list
+      for instance), the status code should be set to reflect the error and the
+      partial result may or may not be returned depending on the controller -
+      this behavior should be documented.
+    - (TODO) API methods should throw a finite number of exceptions (defined in)
+      `galaxy.exceptions` and these should subclass `MessageException` and not
+      paste/wsgi HTTP exceptions. When possible, the framework itself should be
+      responsible catching these exceptions, setting the status code, and
+      building an error response.
+    - Error responses should not consist of plain text strings - they should be
+      dictionaries describing the error and containing the following keys (TODO:
+      spell out nature of this.) Various error conditions (once a format has
+      been chosen and framework to enforce it in place) should be spelled out
+      in this document.
+    - Backward compatibility is important and should maintained when possible.
+      If changing behavior in a non-backward compatibile way please ensure one
+      of the following holds - there is a strong reason to believe no consumers
+      depend on a behavior, the behavior is effectively broken, or the API
+      method being modified has not been part of a tagged dist release.
+
+The following bullet points represent good practices more than guidelines, please
+consider them when modifying the API.
+
+    - Functionality should not be copied and pasted between controllers -
+      consider refactoring functionality into associated classes or short of
+      that into Mixins (http://en.wikipedia.org/wiki/Composition_over_inheritance).
+    - API additions are more permanent changes to Galaxy than many other potential
+      changes and so a second opinion on API changes should be sought. (Consider a
+      pull request!)
+    - New API functionality should include functional tests. These functional
+      tests should be implemented in Python and placed in
+      `test/functional/api`. (Once such a framework is in place - it is not
+      right now).
+    - Changes to reflect modifications to the API should be pushed upstream to
+      the BioBlend project possible.
+
+Longer term goals/notes.
+
+    - It would be advantageous to have a clearer separation of anonymous and
+      admin handling functionality.
+    - If at some point in the future, functionality needs to be added that
+      breaks backward compatibility in a significant way to a compontent used by
+      the community should be alerted  - a "dev" variant of the API will be
+      established and the community should be alerted and given a timeframe
+      for when the old behavior will be replaced with the new behavior.
+    - Consistent standards for range-based requests, batch requests, filtered
+      requests, etc... should be established and documented here.

eggs.ini

@@ -26,9 +26,8 @@ pycrypto = 2.5
 pysam = 0.4.2
 pysqlite = 2.5.6
 python_lzo = 1.08_2.03_static
-simplejson = 2.1.1
 threadframe = 0.2
-guppy = 0.1.8
+guppy = 0.1.10
 SQLAlchemy = 0.7.9
 ; msgpack_python = 0.2.4
 
@@ -54,6 +53,7 @@ PasteDeploy = 1.5.0
 pexpect = 2.4
 python_openid = 2.2.5
 python_daemon = 1.5.5
+pytz = 2013.9
 Routes = 1.12.3
 sqlalchemy_migrate = 0.7.2
 ssh = 1.7.14
@@ -61,10 +61,10 @@ SVGFig = 1.1.6
 Tempita = 0.5.1
 twill = 0.9
 WebError = 0.8a
-WebHelpers = 0.2
+WebHelpers = 1.3
 WebOb = 0.8.5
 wsgiref = 0.1.2
-Babel = 0.9.4
+Babel = 1.3
 wchartype = 0.1
 Whoosh = 0.3.18
 ; fluent_logger = 0.3.3

install_and_test_tool_shed_repositories.sh

@@ -2,20 +2,31 @@
 
 # A good place to look for nose info: http://somethingaboutorange.com/mrl/projects/nose/
 
-# The test/install_and_test_tool_shed_repositories/functional_tests.py can not be executed directly, because it must have certain functional test definitions
-# in sys.argv. Running it through this shell script is the best way to ensure that it has the required definitions.
+# The test/install_and_test_tool_shed_repositories/functional_tests.py cannot be executed directly because it must
+# have certain functional test definitions in sys.argv.  Running it through this shell script is the best way to
+# ensure that it has the required definitions.
 
-# This script requires the following environment variables:
+# This script requires setting of the following environment variables:
 # GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY - must be set to the API key for the tool shed that is being checked.
 # GALAXY_INSTALL_TEST_TOOL_SHED_URL - must be set to a URL that the tool shed is listening on.
-# If the tool shed url is not specified in tool_sheds_conf.xml, GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF must be set to a tool sheds configuration file
-# that does specify that url, otherwise repository installation will fail.
+
+# If the tool shed url is not specified in tool_sheds_conf.xml, GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF must be set to
+# a tool sheds configuration file that does specify that url or repository installation will fail.
+
+# This script accepts the command line option -w to select which set of tests to run. The default behavior is to test
+# first tool_dependency_definition repositories and then repositories with tools. Provide the value 'dependencies'
+# to test only tool_dependency_definition repositories or 'tools' to test only repositories with tools. 
 
 if [ -z $GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY ] ; then
 	echo "This script requires the GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY environment variable to be set and non-empty."
 	exit 1
 fi
 
+if [ -z $GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR ] ; then
+	echo "This script requires the GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR environment variable to be set to your configured tool dependency path."
+	exit 1
+fi
+
 if [ -z $GALAXY_INSTALL_TEST_TOOL_SHED_URL ] ; then
 	echo "This script requires the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable to be set and non-empty."
 	exit 1
@@ -37,7 +48,66 @@ else
 	fi
 fi
 
-python test/install_and_test_tool_shed_repositories/functional_tests.py $* -v --with-nosehtml --html-report-file \
-	test/install_and_test_tool_shed_repositories/run_functional_tests.html \
-	test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py \
-	test/functional/test_toolbox.py  
+if [ -z $GALAXY_INSTALL_TEST_SHED_TOOL_PATH ] ; then
+	export GALAXY_INSTALL_TEST_SHED_TOOL_PATH='/tmp/shed_tools'
+fi
+
+if [ ! -d $GALAXY_INSTALL_TEST_SHED_TOOL_PATH ] ; then
+	mkdir -p $GALAXY_INSTALL_TEST_SHED_TOOL_PATH
+fi
+
+test_tool_dependency_definitions () {
+    # Test installation of repositories of type tool_dependency_definition.
+	if [ -f $GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR/stage_1_complete ] ; then
+		rm $GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR/stage_1_complete
+	fi
+    echo "Starting stage 1, tool dependency definitions."
+    python test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py $* -v --with-nosehtml --html-report-file \
+        test/install_and_test_tool_shed_repositories/tool_dependency_definitions/run_functional_tests.html \
+        test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py \
+        test/functional/test_toolbox.py
+    echo "Stage 1 complete, exit code $?"
+    touch $GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR/stage_1_complete
+}
+
+test_repositories_with_tools () {
+	if [ ! -f $GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR/stage_1_complete ] ; then
+		echo 'Stage 1 did not complete its run, exiting.'
+		exit 1
+	fi
+    echo "Starting stage 2, repositories with tools."
+    # Test installation of repositories that contain valid tools with defined functional tests and a test-data directory containing test files.
+    python test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py $* -v --with-nosehtml --html-report-file \
+        test/install_and_test_tool_shed_repositories/repositories_with_tools/run_functional_tests.html \
+        test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py \
+        test/functional/test_toolbox.py
+    echo "Stage 2 complete, exit code $?"
+    rm $GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR/stage_1_complete
+}
+
+which='both'
+
+while getopts "w:" arg; do
+    case $arg in
+        w)
+            which=$OPTARG
+            ;;
+    esac
+done
+
+case $which in
+    # Use "-w tool_dependency_definitions" when you want to test repositories of type tool_dependency_definition.
+    tool_dependency_definitions)
+        test_tool_dependency_definitions
+        ;;
+    # Use "-w repositories_with_tools" parameter when you want to test repositories that contain tools.
+    repositories_with_tools)
+        test_repositories_with_tools
+        ;;
+    # No received parameters or any received parameter not in [ tool_dependency_definitions, repositories_with_tools ]
+    # will execute both scripts.
+    *)
+        test_tool_dependency_definitions
+        test_repositories_with_tools
+        ;;
+esac        

job_conf.xml.sample_advanced

@@ -6,7 +6,19 @@
           -->
         <plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner"/>
         <plugin id="pbs" type="runner" load="galaxy.jobs.runners.pbs:PBSJobRunner" workers="2"/>
-        <plugin id="drmaa" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner"/>
+        <plugin id="drmaa" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner">
+            <!-- Different DRMs handle successfully completed jobs differently,
+                 these options can be changed to handle such differences and
+                 are explained in detail on the Galaxy wiki. Defaults are shown -->
+            <param id="invalidjobexception_state">ok</param>
+            <param id="invalidjobexception_retries">0</param>
+            <param id="internalexception_state">ok</param>
+            <param id="internalexception_retries">0</param>
+        </plugin>
+        <plugin id="sge" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner">
+            <!-- Override the $DRMAA_LIBRARY_PATH environment variable -->
+            <param id="drmaa_library_path">/sge/lib/libdrmaa.so</param>
+        </plugin>
         <plugin id="lwr" type="runner" load="galaxy.jobs.runners.lwr:LwrJobRunner">
           <!-- More information on LWR can be found at https://lwr.readthedocs.org -->
           <!-- Uncomment following line to use libcurl to perform HTTP calls (defaults to urllib) -->
@@ -14,6 +26,7 @@
         </plugin>
         <plugin id="cli" type="runner" load="galaxy.jobs.runners.cli:ShellJobRunner" />
         <plugin id="condor" type="runner" load="galaxy.jobs.runners.condor:CondorJobRunner" />
+        <plugin id="slurm" type="runner" load="galaxy.jobs.runners.slurm:SlurmJobRunner" />
     </plugins>
     <handlers default="handlers">
         <!-- Additional job handlers - the id should match the name of a
@@ -21,6 +34,15 @@
          -->
         <handler id="handler0" tags="handlers"/>
         <handler id="handler1" tags="handlers"/>
+        <!-- Handlers will load all plugins defined in the <plugins> collection
+             above by default, but can be limited to a subset using <plugin>
+             tags. This is useful for heterogenous environments where the DRMAA
+             plugin would need to be loaded more than once with different
+             configs.
+         -->
+        <handler id="sge_handler">
+            <plugin id="sge"/>
+        </handler>
         <handler id="special_handler0" tags="special_handlers"/>
         <handler id="special_handler1" tags="special_handlers"/>
         <handler id="trackster_handler"/>
@@ -52,15 +74,44 @@
             <param id="private_token">123456789changeme</param>
             <!-- Uncomment the following statement to disable file staging (e.g.
                  if there is a shared file system between Galaxy and the LWR 
-                 server). -->
+                 server). Alternatively action can be set to 'copy' - to replace 
+                 http transfers with file system copies. -->
             <!-- <param id="default_file_action">none</param> -->
+            <!-- The above option is just the default, the transfer behavior
+                 none|copy|http can be configured on a per path basis via the
+                 following file. See lib/galaxy/jobs/runners/lwr_client/action_mapper.py
+                 for examples of how to configure this file. This is very beta
+                 and nature of file will likely change.
+            -->
+            <!-- <param id="file_action_config">file_actions.json</param> -->
+            <!-- Uncomment following option to disable Galaxy tool dependency
+                 resolution and utilize remote LWR's configuraiton of tool
+                 dependency resolution instead (same options as Galaxy for
+                 dependency resolution are available in LWR).
+            -->
+            <!-- <param id="dependency_resolution">remote</params> -->
+            <!-- Uncomment following option to enable setting metadata on remote
+                 LWR server. The 'use_remote_datatypes' option is available for
+                 determining whether to use remotely configured datatypes or local
+                 ones (both alternatives are a little brittle). -->
+            <!-- <param id="remote_metadata">true</param> -->
+            <!-- <param id="use_remote_datatypes">false</param> -->
+            <!-- If remote LWR server is configured to run jobs as the real user,
+                 uncomment the following line to pass the current Galaxy user
+                 along. -->
+            <!-- <param id="submit_user">$__user_name__</param> -->
+            <!-- Various other submission parameters can be passed along to the LWR
+                 whose use will depend on the remote LWR's configured job manager.
+                 For instance:
+            -->
+            <!-- <param id="submit_native_specification">-P bignodes -R y -pe threads 8</param> -->            
         </destination>
         <destination id="ssh_torque" runner="cli">
             <param id="shell_plugin">SecureShell</param>
             <param id="job_plugin">Torque</param>
             <param id="shell_username">foo</param>
             <param id="shell_hostname">foo.example.org</param>
-            <param id="Job_Execution_Time">24:00:00</param>
+            <param id="job_Resource_List">walltime=24:00:00,ncpus=4</param>
         </destination>
         <destination id="condor" runner="condor">
             <!-- With no params, jobs are submitted to the 'vanilla' universe with:

lib/galaxy/actions/admin.py

@@ -150,7 +150,7 @@ class AdminActions( object ):
         message += ', '.join( names )
         return message
 
-    def _undelete_quota( self, quota ):
+    def _undelete_quota( self, quota, params = None):
         quotas = util.listify( quota )
         names = []
         for q in quotas:

lib/galaxy/app.py

@@ -1,16 +1,10 @@
 from __future__ import absolute_import
-import sys, os, atexit
+import sys
+import os
 
-from galaxy import config, jobs, util, tools, web
-import galaxy.tools.search
-import galaxy.tools.data
-import tool_shed.galaxy_install
-import tool_shed.tool_shed_registry
-from galaxy.web import security
+from galaxy import config, jobs
 import galaxy.model
-import galaxy.datatypes.registry
 import galaxy.security
-from galaxy.objectstore import build_object_store_from_config
 import galaxy.quota
 from galaxy.tags.tag_handler import GalaxyTagHandler
 from galaxy.visualization.genomes import Genomes
@@ -27,7 +21,8 @@ from galaxy.web.base import pluginframework
 import logging
 log = logging.getLogger( __name__ )
 
-class UniverseApplication( object ):
+
+class UniverseApplication( object, config.ConfiguresGalaxyMixin ):
     """Encapsulates the state of a Universe application"""
     def __init__( self, **kwargs ):
         print >> sys.stderr, "python path is: " + ", ".join( sys.path )
@@ -38,72 +33,38 @@ class UniverseApplication( object ):
         self.config.check()
         config.configure_logging( self.config )
         self.configure_fluent_log()
-        # Determine the database url
-        if self.config.database_connection:
-            db_url = self.config.database_connection
-        else:
-            db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
-        # Set up the tool sheds registry
-        if os.path.isfile( self.config.tool_sheds_config ):
-            self.tool_shed_registry = tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config )
-        else:
-            self.tool_shed_registry = None
-        # Initialize database / check for appropriate schema version.  # If this
-        # is a new installation, we'll restrict the tool migration messaging.
-        from galaxy.model.migrate.check import create_or_verify_database
-        create_or_verify_database( db_url, kwargs.get( 'global_conf', {} ).get( '__file__', None ), self.config.database_engine_options, app=self )
-        # Alert the Galaxy admin to tools that have been moved from the distribution to the tool shed.
-        from tool_shed.galaxy_install.migrate.check import verify_tools
-        verify_tools( self, db_url, kwargs.get( 'global_conf', {} ).get( '__file__', None ), self.config.database_engine_options )
-        # Object store manager
-        self.object_store = build_object_store_from_config(self.config, fsmon=True)
+
+        self._configure_tool_shed_registry()
+
+        self._configure_object_store( fsmon=True )
+
         # Setup the database engine and ORM
-        from galaxy.model import mapping
-        self.model = mapping.init( self.config.file_path,
-                                   db_url,
-                                   self.config.database_engine_options,
-                                   database_query_profiling_proxy = self.config.database_query_profiling_proxy,
-                                   object_store = self.object_store,
-                                   trace_logger=self.trace_logger,
-                                   use_pbkdf2=self.config.get_bool( 'use_pbkdf2', True ) )
+        config_file = kwargs.get( 'global_conf', {} ).get( '__file__', None )
+        self._configure_models( check_migrate_databases=True, check_migrate_tools=True, config_file=config_file )
+
         # Manage installed tool shed repositories.
-        self.installed_repository_manager = tool_shed.galaxy_install.InstalledRepositoryManager( self )
-        # Create an empty datatypes registry.
-        self.datatypes_registry = galaxy.datatypes.registry.Registry()
-        # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories.  We
-        # load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some
-        # generic sniffers (eg text,xml) which catch anything, so it's impossible for proprietary sniffers to be used.
-        # However, if there is a conflict (2 datatypes with the same extension) between a proprietary datatype and a datatype
-        # in the Galaxy distribution, the datatype in the Galaxy distribution will take precedence.  If there is a conflict
-        # between 2 proprietary datatypes, the datatype from the repository that was installed earliest will take precedence.
-        self.installed_repository_manager.load_proprietary_datatypes()
-        # Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config.
-        self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config )
+        from tool_shed.galaxy_install import installed_repository_manager
+        self.installed_repository_manager = installed_repository_manager.InstalledRepositoryManager( self )
+
+        self._configure_datatypes_registry( self.installed_repository_manager )
         galaxy.model.set_datatypes_registry( self.datatypes_registry )
+
         # Security helper
-        self.security = security.SecurityHelper( id_secret=self.config.id_secret )
+        self._configure_security()
         # Tag handler
         self.tag_handler = GalaxyTagHandler()
         # Genomes
         self.genomes = Genomes( self )
         # Data providers registry.
         self.data_provider_registry = DataProviderRegistry()
-        # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path.
-        self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( tool_data_path=self.config.tool_data_path,
-                                                                        config_filename=self.config.tool_data_table_config_path )
-        # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables.
-        self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config,
-                                                     tool_data_path=self.tool_data_tables.tool_data_path,
-                                                     from_shed_config=False )
+
+        self._configure_tool_data_tables( from_shed_config=False )
+
         # Initialize the job management configuration
         self.job_config = jobs.JobConfiguration(self)
-        # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file.
-        tool_configs = self.config.tool_configs
-        if self.config.migrated_tools_config not in tool_configs:
-            tool_configs.append( self.config.migrated_tools_config )
-        self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self )
-        # Search support for tools
-        self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox )
+
+        self._configure_toolbox()
+
         # Load Data Manager
         self.data_managers = DataManagers( self )
         # If enabled, poll respective tool sheds to see if updates are available for any installed tool shed repositories.
@@ -126,9 +87,9 @@ class UniverseApplication( object ):
         load_genome_index_tools( self.toolbox )
         # visualizations registry: associates resources with visualizations, controls how to render
         self.visualizations_registry = None
-        if self.config.visualizations_plugins_directory:
+        if self.config.visualization_plugins_directory:
             self.visualizations_registry = VisualizationsRegistry( self,
-                directories_setting=self.config.visualizations_plugins_directory,
+                directories_setting=self.config.visualization_plugins_directory,
                 template_cache_dir=self.config.template_cache )
         # Load security policy.
         self.security_agent = self.model.security_agent

lib/galaxy/config.py

@@ -1,6 +1,8 @@
 """
 Universe configuration builder.
 """
+# absolute_import needed for tool_shed package.
+from __future__ import absolute_import
 
 import sys, os, tempfile, re
 import logging, logging.config
@@ -33,15 +35,22 @@ class Configuration( object ):
         self.umask = os.umask( 077 ) # get the current umask
         os.umask( self.umask ) # can't get w/o set, so set it back
         self.gid = os.getgid() # if running under newgrp(1) we'll need to fix the group of data created on the cluster
+
         # Database related configuration
         self.database = resolve_path( kwargs.get( "database_file", "database/universe.sqlite" ), self.root )
         self.database_connection = kwargs.get( "database_connection", False )
         self.database_engine_options = get_database_engine_options( kwargs )
         self.database_create_tables = string_as_bool( kwargs.get( "database_create_tables", "True" ) )
         self.database_query_profiling_proxy = string_as_bool( kwargs.get( "database_query_profiling_proxy", "False" ) )
+
         # Don't set this to true for production databases, but probably should
         # default to True for sqlite databases.
         self.database_auto_migrate = string_as_bool( kwargs.get( "database_auto_migrate", "False" ) )
+
+        # Install database related configuration (if different).
+        self.install_database_connection = kwargs.get( "install_database_connection", None )
+        self.install_database_engine_options = get_database_engine_options( kwargs, model_prefix="install_" )
+
         # Where dataset files are stored
         self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root )
         self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root )
@@ -49,7 +58,7 @@ class Configuration( object ):
         self.openid_consumer_cache_path = resolve_path( kwargs.get( "openid_consumer_cache_path", "database/openid_consumer_cache" ), self.root )
         self.cookie_path = kwargs.get( "cookie_path", "/" )
         self.genome_data_path = kwargs.get( "genome_data_path", "tool-data/genome" )
-        self.rsync_url = kwargs.get( "rsync_url", "rsync://scofield.bx.psu.edu/indexes" )
+        self.rsync_url = kwargs.get( "rsync_url", "rsync://datacache.galaxyproject.org/indexes" )
         # Galaxy OpenID settings
         self.enable_openid = string_as_bool( kwargs.get( 'enable_openid', False ) )
         self.openid_config = kwargs.get( 'openid_config_file', 'openid_conf.xml' )
@@ -58,7 +67,7 @@ class Configuration( object ):
         self.enable_unique_workflow_defaults = string_as_bool( kwargs.get( 'enable_unique_workflow_defaults', False ) )
         self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root )
         self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() )
-        self.len_file_path = kwargs.get( "len_file_path", resolve_path(os.path.join(self.tool_data_path, 'shared','ucsc','chrom'), self.root) )
+        self.len_file_path = resolve_path( kwargs.get( "len_file_path", os.path.join( self.tool_data_path, 'shared','ucsc','chrom') ), self.root )
         self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root )
         # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution
         # and moved to the tool shed.
@@ -113,7 +122,9 @@ class Configuration( object ):
         self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" )
         self.retry_metadata_internally = string_as_bool( kwargs.get( "retry_metadata_internally", "True" ) )
         self.use_remote_user = string_as_bool( kwargs.get( "use_remote_user", "False" ) )
+        self.normalize_remote_user_email = string_as_bool( kwargs.get( "normalize_remote_user_email", "False" ) )
         self.remote_user_maildomain = kwargs.get( "remote_user_maildomain", None )
+        self.remote_user_header = kwargs.get( "remote_user_header", 'HTTP_REMOTE_USER' )
         self.remote_user_logout_href = kwargs.get( "remote_user_logout_href", None )
         self.require_login = string_as_bool( kwargs.get( "require_login", "False" ) )
         self.allow_user_creation = string_as_bool( kwargs.get( "allow_user_creation", "True" ) )
@@ -141,6 +152,7 @@ class Configuration( object ):
             h, m, s = [ int( v ) for v in self.job_walltime.split( ':' ) ]
             self.job_walltime_delta = timedelta( 0, s, 0, 0, m, h )
         self.admin_users = kwargs.get( "admin_users", "" )
+        self.reset_password_length = int( kwargs.get('reset_password_length', '15') )
         self.mailing_join_addr = kwargs.get('mailing_join_addr',"galaxy-announce-join@bx.psu.edu")
         self.error_email_to = kwargs.get( 'error_email_to', None )
         self.activation_email = kwargs.get( 'activation_email', None )
@@ -303,7 +315,7 @@ class Configuration( object ):
         self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' )
         # Cloud configuration options
         self.enable_cloud_launch = string_as_bool( kwargs.get( 'enable_cloud_launch', False ) )
-        self.cloudlaunch_default_ami = kwargs.get( 'cloudlaunch_default_ami', 'ami-118bfc78' )
+        self.cloudlaunch_default_ami = kwargs.get( 'cloudlaunch_default_ami', 'ami-a7dbf6ce' )
         # Galaxy messaging (AMQP) configuration options
         self.amqp = {}
         try:
@@ -316,6 +328,10 @@ class Configuration( object ):
         self.biostar_key_name = kwargs.get( 'biostar_key_name', None )
         self.biostar_key = kwargs.get( 'biostar_key', None )
         self.pretty_datetime_format = expand_pretty_datetime_format( kwargs.get( 'pretty_datetime_format', '$locale (UTC)' ) )
+        self.master_api_key = kwargs.get( 'master_api_key', None )
+        if self.master_api_key == "changethis":  # default in sample config file
+            raise Exception("Insecure configuration, please change master_api_key to something other than default (changethis)")
+
         # Experimental: This will not be enabled by default and will hide
         # nonproduction code.
         # The api_folders refers to whether the API exposes the /folders section.
@@ -329,7 +345,7 @@ class Configuration( object ):
         self.fluent_host = kwargs.get( 'fluent_host', 'localhost' )
         self.fluent_port = int( kwargs.get( 'fluent_port', 24224 ) )
         # visualization plugin framework
-        self.visualizations_plugins_directory = kwargs.get( 'visualizations_plugins_directory', None )
+        self.visualization_plugins_directory = kwargs.get( 'visualization_plugins_directory', None )
 
     @property
     def sentry_dsn_public( self ):
@@ -435,7 +451,7 @@ class Configuration( object ):
         admin_users = [ x.strip() for x in self.get( "admin_users", "" ).split( "," ) ]
         return ( user is not None and user.email in admin_users )
 
-def get_database_engine_options( kwargs ):
+def get_database_engine_options( kwargs, model_prefix='' ):
     """
     Allow options for the SQLAlchemy database engine to be passed by using
     the prefix "database_engine_option".
@@ -451,7 +467,7 @@ def get_database_engine_options( kwargs ):
         'pool_threadlocal': string_as_bool,
         'server_side_cursors': string_as_bool
     }
-    prefix = "database_engine_option_"
+    prefix = "%sdatabase_engine_option_" % model_prefix
     prefix_len = len( prefix )
     rval = {}
     for key, value in kwargs.iteritems():
@@ -462,6 +478,7 @@ def get_database_engine_options( kwargs ):
             rval[ key  ] = value
     return rval
 
+
 def configure_logging( config ):
     """
     Allow some basic logging configuration to be read from ini file.
@@ -502,3 +519,114 @@ def configure_logging( config ):
         sentry_handler.setLevel( logging.WARN )
         root.addHandler( sentry_handler )
 
+
+class ConfiguresGalaxyMixin:
+    """ Shared code for configuring Galaxy-like app objects.
+    """
+
+    def _configure_toolbox( self ):
+        # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file.
+        tool_configs = self.config.tool_configs
+        if self.config.migrated_tools_config not in tool_configs:
+            tool_configs.append( self.config.migrated_tools_config )
+        from galaxy import tools
+        self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self )
+        # Search support for tools
+        import galaxy.tools.search
+        self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox )
+
+    def _configure_tool_data_tables( self, from_shed_config ):
+        from galaxy.tools.data import ToolDataTableManager
+
+        # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path.
+        self.tool_data_tables = ToolDataTableManager( tool_data_path=self.config.tool_data_path,
+                                                      config_filename=self.config.tool_data_table_config_path )
+        # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables.
+        self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config,
+                                                     tool_data_path=self.tool_data_tables.tool_data_path,
+                                                     from_shed_config=from_shed_config )
+
+    def _configure_datatypes_registry( self, installed_repository_manager=None ):
+        from galaxy.datatypes import registry
+        # Create an empty datatypes registry.
+        self.datatypes_registry = registry.Registry()
+        if installed_repository_manager:
+            # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories.  We
+            # load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some
+            # generic sniffers (eg text,xml) which catch anything, so it's impossible for proprietary sniffers to be used.
+            # However, if there is a conflict (2 datatypes with the same extension) between a proprietary datatype and a datatype
+            # in the Galaxy distribution, the datatype in the Galaxy distribution will take precedence.  If there is a conflict
+            # between 2 proprietary datatypes, the datatype from the repository that was installed earliest will take precedence.
+            installed_repository_manager.load_proprietary_datatypes()
+        # Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config.
+        self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config )
+
+    def _configure_object_store( self, **kwds ):
+        from galaxy.objectstore import build_object_store_from_config
+        self.object_store = build_object_store_from_config( self.config, **kwds )
+
+    def _configure_security( self ):
+        from galaxy.web import security
+        self.security = security.SecurityHelper( id_secret=self.config.id_secret )
+
+    def _configure_tool_shed_registry( self ):
+        import tool_shed.tool_shed_registry
+
+        # Set up the tool sheds registry
+        if os.path.isfile( self.config.tool_sheds_config ):
+            self.tool_shed_registry = tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config )
+        else:
+            self.tool_shed_registry = None
+
+    def _configure_models( self, check_migrate_databases=False, check_migrate_tools=False, config_file=None ):
+        """
+        Preconditions: object_store must be set on self.
+        """
+        if self.config.database_connection:
+            db_url = self.config.database_connection
+        else:
+            db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
+        install_db_url = self.config.install_database_connection
+        # TODO: Consider more aggressive check here that this is not the same
+        # database file under the hood.
+        combined_install_database = not( install_db_url and install_db_url != db_url )
+        install_db_url = install_db_url or db_url
+
+        if check_migrate_databases:
+            # Initialize database / check for appropriate schema version.  # If this
+            # is a new installation, we'll restrict the tool migration messaging.
+            from galaxy.model.migrate.check import create_or_verify_database
+            create_or_verify_database( db_url, config_file, self.config.database_engine_options, app=self )
+            if not combined_install_database:
+                from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database
+                tsi_create_or_verify_database( install_db_url, self.config.install_database_engine_options, app=self )
+
+        if check_migrate_tools:
+            # Alert the Galaxy admin to tools that have been moved from the distribution to the tool shed.
+            from tool_shed.galaxy_install.migrate.check import verify_tools
+            if combined_install_database:
+                install_database_options = self.config.database_engine_options
+            else:
+                install_database_options = self.config.install_database_engine_options
+            verify_tools( self, install_db_url, config_file, install_database_options )
+
+        from galaxy.model import mapping
+        self.model = mapping.init( self.config.file_path,
+                                   db_url,
+                                   self.config.database_engine_options,
+                                   map_install_models=combined_install_database,
+                                   database_query_profiling_proxy=self.config.database_query_profiling_proxy,
+                                   object_store=self.object_store,
+                                   trace_logger=getattr(self, "trace_logger", None),
+                                   use_pbkdf2=self.config.get_bool( 'use_pbkdf2', True ) )
+
+        if combined_install_database:
+            log.info("Install database targetting Galaxy's database configuration.")
+            self.install_model = self.model
+        else:
+            from galaxy.model.tool_shed_install import mapping as install_mapping
+            install_db_url = self.config.install_database_connection
+            log.info("Install database using its own connection %s" % install_db_url)
+            install_db_engine_options = self.config.install_database_engine_options
+            self.install_model = install_mapping.init( install_db_url,
+                                                       install_db_engine_options )

lib/galaxy/datatypes/checkers.py

@@ -2,6 +2,8 @@ import os, gzip, re, gzip, zipfile, binascii, bz2, imghdr
 from galaxy import util
 from StringIO import StringIO
 
+HTML_CHECK_LINES = 100
+
 try:
     import Image as PIL
 except ImportError:
@@ -32,9 +34,11 @@ def check_html( file_path, chunk=None ):
     regexp1 = re.compile( "<A\s+[^>]*HREF[^>]+>", re.I )
     regexp2 = re.compile( "<IFRAME[^>]*>", re.I )
     regexp3 = re.compile( "<FRAMESET[^>]*>", re.I )
-    regexp4 = re.compile( "<META[^>]*>", re.I )
+    regexp4 = re.compile( "<META[\W][^>]*>", re.I )
     regexp5 = re.compile( "<SCRIPT[^>]*>", re.I )
     lineno = 0
+    # TODO: Potentially reading huge lines into string here, this should be
+    # reworked.
     for line in temp:
         lineno += 1
         matches = regexp1.search( line ) or regexp2.search( line ) or regexp3.search( line ) or regexp4.search( line ) or regexp5.search( line )
@@ -42,7 +46,7 @@ def check_html( file_path, chunk=None ):
             if chunk is None:
                 temp.close()
             return True
-        if lineno > 100:
+        if HTML_CHECK_LINES and (lineno > HTML_CHECK_LINES):
             break
     if chunk is None:
         temp.close()

lib/galaxy/datatypes/converters/bam_to_bai.xml

@@ -1,4 +1,7 @@
 <tool id="CONVERTER_Bam_Bai_0" name="Bam to Bai" hidden="true">
+  <requirements>
+      <requirement type="package">samtools</requirement>
+  </requirements>
   <command>samtools index $input1 $output1</command>
   <inputs>
     <page>
@@ -10,4 +13,4 @@
    </outputs>
   <help>
   </help>
-</tool>
+</tool>

lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml

@@ -1,5 +1,8 @@
 <tool id="CONVERTER_bedgraph_to_bigwig" name="Convert BedGraph to BigWig" hidden="true">
   <!-- Used internally to generate track indexes -->
+  <requirements>
+      <requirement type="package">ucsc_tools</requirement>
+  </requirements>
   <command>grep -v "^track" $input | wigToBigWig -clip stdin $chromInfo $output</command>
   <inputs>
       <param format="bedgraph" name="input" type="data" label="Choose wiggle"/>
@@ -9,4 +12,4 @@
    </outputs>
   <help>
   </help>
-</tool>
+</tool>

lib/galaxy/datatypes/converters/fasta_to_2bit.xml

@@ -1,6 +1,9 @@
 <tool id="CONVERTER_fasta_to_2bit" name="Convert FASTA to 2bit" version="1.0.0">
     <!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
     <!-- Used on the metadata edit page. -->
+    <requirements>
+        <requirement type="package">ucsc_tools</requirement>
+    </requirements>
     <command>faToTwoBit $input $output</command>
     <inputs>
         <param name="input" type="data" format="fasta" label="Fasta file"/>

lib/galaxy/datatypes/converters/fasta_to_bowtie_base_index_converter.xml

@@ -2,6 +2,9 @@
   <requirements><requirement type='package'>bowtie</requirement></requirements>
   <!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
   <!-- Used on the metadata edit page. -->
+  <requirements>
+      <requirement type="package">bowtie</requirement>
+  </requirements>
   <command>
   mkdir ${output.files_path}
   &amp;&amp; bowtie-build --quiet

lib/galaxy/datatypes/converters/fasta_to_bowtie_color_index_converter.xml

@@ -2,6 +2,9 @@
   <requirements><requirement type='package'>bowtie</requirement></requirements>
   <!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
   <!-- Used on the metadata edit page. -->
+  <requirements>
+      <requirement type="package">bowtie</requirement>
+  </requirements>
   <command>
   mkdir ${output.files_path}
   &amp;&amp; bowtie-build --quiet

lib/galaxy/datatypes/converters/sam_to_bam.xml

@@ -5,6 +5,9 @@
          To fix this: (a) merge sam_to_bam tool in tools with this conversion (like fasta_to_len 
          conversion); and (b) define a datatype-specific way to set converter parameters.
       -->
+    <requirements>
+        <requirement type="package">samtools</requirement>
+    </requirements>
     <command interpreter="python">sam_to_bam.py $input1 $output</command>
     <inputs>
         <param name="input1" type="data" format="sam" label="SAM file"/>

lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml

@@ -1,5 +1,8 @@
 <tool id="CONVERTER_wig_to_bigwig" name="Convert Wiggle to BigWig" hidden="true">
   <!-- Used internally to generate track indexes -->
+  <requirements>
+      <requirement type="package">ucsc_tools</requirement>
+  </requirements>
   <command>grep -v "^track" $input | wigToBigWig -clip stdin $chromInfo $output</command>
   <inputs>
     <page>
@@ -11,4 +14,4 @@
    </outputs>
   <help>
   </help>
-</tool>
+</tool>

lib/galaxy/datatypes/data.py

@@ -199,6 +199,26 @@ class Data( object ):
             out = "Can't create peek %s" % str( exc )
         return out
 
+    def _archive_main_file(self, archive, display_name, data_filename):
+        """Called from _archive_composite_dataset to add central file to archive.
+
+        Unless subclassed, this will add the main dataset file (argument data_filename)
+        to the archive, as an HTML file with its filename derived from the dataset name
+        (argument outfname).
+
+        Returns a tuple of boolean, string, string: (error, msg, messagetype)
+        """
+        error, msg, messagetype = False, "", ""
+        archname = '%s.html' % display_name  # fake the real nature of the html file
+        try:
+            archive.add(data_filename, archname)
+        except IOError:
+            error = True
+            log.exception("Unable to add composite parent %s to temporary library download archive" % data_filename)
+            msg = "Unable to create archive for download, please report this error"
+            messagetype = "error"
+        return error, msg, messagetype
+
     def _archive_composite_dataset( self, trans, data=None, **kwd ):
         # save a composite object into a compressed archive for downloading
         params = util.Params( kwd )
@@ -237,29 +257,27 @@ class Data( object ):
                 path = data.file_name
                 fname = os.path.split(path)[-1]
                 efp = data.extra_files_path
-                htmlname = os.path.splitext(outfname)[0]
-                if not htmlname.endswith(ext):
-                    htmlname = '%s_%s' % (htmlname,ext)
-                archname = '%s.html' % htmlname # fake the real nature of the html file
-                try:
-                    archive.add(data.file_name,archname)
-                except IOError:
-                    error = True
-                    log.exception( "Unable to add composite parent %s to temporary library download archive" % data.file_name)
-                    msg = "Unable to create archive for download, please report this error"
-                    messagetype = 'error'
-                for root, dirs, files in os.walk(efp):
-                    for fname in files:
-                        fpath = os.path.join(root,fname)
-                        rpath = os.path.relpath(fpath,efp)
-                        try:
-                            archive.add( fpath,rpath )
-                        except IOError:
-                            error = True
-                            log.exception( "Unable to add %s to temporary library download archive" % rpath)
-                            msg = "Unable to create archive for download, please report this error"
-                            messagetype = 'error'
-                            continue
+                #Add any central file to the archive,
+
+                display_name = os.path.splitext(outfname)[0]
+                if not display_name.endswith(ext):
+                    display_name = '%s_%s' % (display_name, ext)
+
+                error, msg, messagetype = self._archive_main_file(archive, display_name, path)
+                if not error:
+                    #Add any child files to the archive,
+                    for root, dirs, files in os.walk(efp):
+                        for fname in files:
+                            fpath = os.path.join(root,fname)
+                            rpath = os.path.relpath(fpath,efp)
+                            try:
+                                archive.add( fpath,rpath )
+                            except IOError:
+                                error = True
+                                log.exception( "Unable to add %s to temporary library download archive" % rpath)
+                                msg = "Unable to create archive for download, please report this error"
+                                messagetype = 'error'
+                                continue
                 if not error:
                     if params.do_action == 'zip':
                         archive.close()
@@ -288,7 +306,14 @@ class Data( object ):
         return open( dataset.file_name )
 
     def display_data(self, trans, data, preview=False, filename=None, to_ext=None, size=None, offset=None, **kwd):
-        """ Old display method, for transition """
+        """ Old display method, for transition - though still used by API and
+        test framework. Datatypes should be very careful if overridding this
+        method and this interface between datatypes and Galaxy will likely
+        change.
+
+        TOOD: Document alternatives to overridding this method (data
+        providers?).
+        """
         #Relocate all composite datatype display to a common location.
         composite_extensions = trans.app.datatypes_registry.get_composite_extensions( )
         composite_extensions.append('html') # for archiving composite datatypes
@@ -487,13 +512,14 @@ class Data( object ):
     def before_setting_metadata( self, dataset ):
         """This function is called on the dataset before metadata is set."""
         dataset.clear_associated_files( metadata_safe = True )
-    def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, space_to_tab = False, **kwds ):
+    def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, to_posix_lines = True, space_to_tab = False, **kwds ):
         kwds[ 'name' ] = name
         kwds[ 'optional' ] = optional
         kwds[ 'mimetype' ] = mimetype
         kwds[ 'description' ] = description
         kwds[ 'substitute_name_with_metadata' ] = substitute_name_with_metadata
         kwds[ 'is_binary' ] = is_binary
+        kwds[ 'to_posix_lines' ] = to_posix_lines
         kwds[ 'space_to_tab' ] = space_to_tab
         return Bunch( **kwds )
     def add_composite_file( self, name, **kwds ):

lib/galaxy/datatypes/dataproviders/decorators.py

@@ -18,6 +18,7 @@ DataProvider related decorators.
 
 from functools import wraps
 #from types import MethodType
+import urllib2
 import copy
 
 import logging
@@ -143,8 +144,9 @@ def _parse_query_string_settings( query_kwargs, settings=None ):
         'int'   : int,
         'float' : float,
         'bool'  : bool,
-        'list:str'  : lambda s: list_from_query_string( s ),
-        'list:int'  : lambda s: [ int( i ) for i in list_from_query_string( s ) ],
+        'list:str'      : lambda s: list_from_query_string( s ),
+        'list:escaped'  : lambda s: [ urllib2.unquote( e ) for e in list_from_query_string( s ) ],
+        'list:int'      : lambda s: [ int( i ) for i in list_from_query_string( s ) ],
     }
     settings = settings or {}
     # yay! yet another set of query string parsers! <-- sarcasm

lib/galaxy/datatypes/dataproviders/line.py

@@ -78,7 +78,7 @@ class FilteredLineDataProvider( base.LimitedOffsetDataProvider ):
                 line = line.strip( '\n' )
             if not self.provide_blank and line == '':
                 return None
-            elif line.startswith( self.comment_char ):
+            elif self.comment_char and line.startswith( self.comment_char ):
                 return None
 
         return super( FilteredLineDataProvider, self ).filter( line )
@@ -94,7 +94,7 @@ class RegexLineDataProvider( FilteredLineDataProvider ):
     the line it is considered valid and will be provided).
     """
     settings = {
-        'regex_list'    : 'list:str',
+        'regex_list'    : 'list:escaped',
         'invert'        : 'bool',
     }
 

lib/galaxy/datatypes/images.py

@@ -213,7 +213,6 @@ Binary.register_sniffable_binary_format("pdf", "pdf", Pdf)
 
 def create_applet_tag_peek( class_name, archive, params ):
     text = """
-<!--[if !IE]>-->
 <object classid="java:%s"
       type="application/x-java-applet"
       height="30" width="200" align="center" >
@@ -221,17 +220,14 @@ def create_applet_tag_peek( class_name, archive, params ):
     for name, value in params.iteritems():
         text += """<param name="%s" value="%s"/>""" % ( name, value )
     text += """
-<!--<![endif]-->
 <object classid="clsid:8AD9C840-044E-11D1-B3E9-00805F499D93"
         height="30" width="200" >
         <param name="code" value="%s" />
         <param name="archive" value="%s"/>""" % ( class_name, archive )
     for name, value in params.iteritems():
         text += """<param name="%s" value="%s"/>""" % ( name, value )
-    text += """</object>
-<!--[if !IE]>-->
+    text += """<div class="errormessage">You must install and enable Java in your browser in order to access this applet.<div></object>
 </object>
-<!--<![endif]-->
 """
     return """<div><p align="center">%s</p></div>""" % text
 

lib/galaxy/datatypes/interval.py

@@ -389,7 +389,7 @@ class Bed( Interval ):
     MetadataElement( name="endCol", default=3, desc="End column", param=metadata.ColumnParameter )
     MetadataElement( name="strandCol", desc="Strand column (click box & select)", param=metadata.ColumnParameter, optional=True, no_value=0 )
     MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True, visible=False )
-    MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[4], param=metadata.ColumnParameter, multiple=True )
+    MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[4], param=metadata.ColumnParameter, optional=True, multiple=True )
     ###do we need to repeat these? they are the same as should be inherited from interval type
 
     def set_meta( self, dataset, overwrite = True, **kwd ):

lib/galaxy/datatypes/metadata.py

@@ -2,15 +2,13 @@
 Galaxy Metadata
 
 """
-from galaxy import eggs
-eggs.require("simplejson")
 
 import copy
 import cPickle
+import json
 import logging
 import os
 import shutil
-import simplejson
 import sys
 import tempfile
 import weakref
@@ -130,7 +128,7 @@ class MetadataCollection( object ):
     def from_JSON_dict( self, filename ):
         dataset = self.parent
         log.debug( 'loading metadata from file for: %s %s' % ( dataset.__class__.__name__, dataset.id ) )
-        JSONified_dict = simplejson.load( open( filename ) )
+        JSONified_dict = json.load( open( filename ) )
         for name, spec in self.spec.items():
             if name in JSONified_dict:
                 dataset._metadata[ name ] = spec.param.from_external_value( JSONified_dict[ name ], dataset )
@@ -146,7 +144,7 @@ class MetadataCollection( object ):
         for name, spec in self.spec.items():
             if name in dataset_meta_dict:
                 meta_dict[ name ] = spec.param.to_external_value( dataset_meta_dict[ name ] )
-        simplejson.dump( meta_dict, open( filename, 'wb+' ) )
+        json.dump( meta_dict, open( filename, 'wb+' ) )
 
     def __getstate__( self ):
         return None #cannot pickle a weakref item (self._parent), when data._metadata_collection is None, it will be recreated on demand
@@ -456,7 +454,7 @@ class ListParameter( MetadataParameter ):
 class DictParameter( MetadataParameter ):
 
     def to_string( self, value ):
-        return  simplejson.dumps( value )
+        return  json.dumps( value )
 
 
 class PythonObjectParameter( MetadataParameter ):
@@ -594,7 +592,7 @@ class MetadataTempFile( object ):
     @classmethod
     def cleanup_from_JSON_dict_filename( cls, filename ):
         try:
-            for key, value in simplejson.load( open( filename ) ).items():
+            for key, value in json.load( open( filename ) ).items():
                 if cls.is_JSONified_value( value ):
                     value = cls.from_JSON( value )
                 if isinstance( value, cls ) and os.path.exists( value.file_name ):
@@ -686,10 +684,10 @@ class JobExternalOutputMetadataWrapper( object ):
                 #file to store a 'return code' indicating the results of the set_meta() call
                 #results code is like (True/False - if setting metadata was successful/failed , exception or string of reason of success/failure )
                 metadata_files.filename_results_code = abspath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_results_%s_" % key ).name )
-                simplejson.dump( ( False, 'External set_meta() not called' ), open( metadata_files.filename_results_code, 'wb+' ) ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
+                json.dump( ( False, 'External set_meta() not called' ), open( metadata_files.filename_results_code, 'wb+' ) ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
                 #file to store kwds passed to set_meta()
                 metadata_files.filename_kwds = abspath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_kwds_%s_" % key ).name )
-                simplejson.dump( kwds, open( metadata_files.filename_kwds, 'wb+' ), ensure_ascii=True )
+                json.dump( kwds, open( metadata_files.filename_kwds, 'wb+' ), ensure_ascii=True )
                 #existing metadata file parameters need to be overridden with cluster-writable file locations
                 metadata_files.filename_override_metadata = abspath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_override_%s_" % key ).name )
                 open( metadata_files.filename_override_metadata, 'wb+' ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
@@ -699,7 +697,7 @@ class JobExternalOutputMetadataWrapper( object ):
                         metadata_temp = MetadataTempFile()
                         shutil.copy( dataset.metadata.get( meta_key, None ).file_name, metadata_temp.file_name )
                         override_metadata.append( ( meta_key, metadata_temp.to_JSON() ) )
-                simplejson.dump( override_metadata, open( metadata_files.filename_override_metadata, 'wb+' ) )
+                json.dump( override_metadata, open( metadata_files.filename_override_metadata, 'wb+' ) )
                 #add to session and flush
                 sa_session.add( metadata_files )
                 sa_session.flush()
@@ -711,7 +709,7 @@ class JobExternalOutputMetadataWrapper( object ):
         metadata_files = self.get_output_filenames_by_dataset( dataset, sa_session )
         if not metadata_files:
             return False # this file doesn't exist
-        rval, rstring = simplejson.load( open( metadata_files.filename_results_code ) )
+        rval, rstring = json.load( open( metadata_files.filename_results_code ) )
         if not rval:
             log.debug( 'setting metadata externally failed for %s %s: %s' % ( dataset.__class__.__name__, dataset.id, rstring ) )
         return rval

lib/galaxy/datatypes/registry.py

@@ -1,16 +1,38 @@
 """
 Provides mapping between extensions and datatypes, mime-types, etc.
 """
-import os, sys, tempfile, threading, logging, imp
-import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo, binary, assembly, ngsindex, graph
+import os
+import sys
+import tempfile
+import threading
+import logging
+import imp
+import data
+import tabular
+import interval
+import images
+import sequence
+import qualityscore
+import genetics
+import xml
+import coverage
+import tracks
+import chrominfo
+import binary
+import assembly
+import ngsindex
+import graph
 import galaxy.util
 from galaxy.util.odict import odict
 from display_applications.application import DisplayApplication
 
+
 class ConfigurationError( Exception ):
     pass
 
+
 class Registry( object ):
+
     def __init__( self ):
         self.log = logging.getLogger(__name__)
         self.log.addHandler( logging.NullHandler() )
@@ -47,19 +69,29 @@ class Registry( object ):
         self.datatype_elems = []
         self.sniffer_elems = []
         self.xml_filename = None
+
     def load_datatypes( self, root_dir=None, config=None, deactivate=False, override=True ):
         """
-        Parse a datatypes XML file located at root_dir/config.  If deactivate is True, an installed tool shed
-        repository that includes proprietary datatypes is being deactivated, so appropriate loaded datatypes
-        will be removed from the registry.  The value of override will be False when a tool shed repository is
-        being installed.  Since installation is occurring after the datatypes registry has been initialized, its
-        contents cannot be overridden by new introduced conflicting data types.
+        Parse a datatypes XML file located at root_dir/config (if processing the Galaxy distributed config) or contained within
+        an installed Tool Shed repository.  If deactivate is True, an installed Tool Shed repository that includes custom datatypes
+        is being deactivated or uninstalled, so appropriate loaded datatypes will be removed from the registry.  The value of
+        override will be False when a Tool Shed repository is being installed.  Since installation is occurring after the datatypes
+        registry has been initialized at server startup, it's contents cannot be overridden by newly introduced conflicting data types.
         """
+
         def __import_module( full_path, datatype_module, datatype_class_name ):
             open_file_obj, file_name, description = imp.find_module( datatype_module, [ full_path ] )
             imported_module = imp.load_module( datatype_class_name, open_file_obj, file_name, description )
             return imported_module
+
         if root_dir and config:
+            # If handling_proprietary_datatypes is determined as True below, we'll have an elem that looks something like this:
+            # <datatype display_in_upload="true"
+            #           extension="blastxml"
+            #           mimetype="application/xml"
+            #           proprietary_datatype_module="blast"
+            #           proprietary_path="[cloned repository path]"
+            #           type="galaxy.datatypes.blast:BlastXml" />
             handling_proprietary_datatypes = False
             # Parse datatypes_conf.xml
             tree = galaxy.util.parse_xml( config )
@@ -82,308 +114,281 @@ class Registry( object ):
             # Proprietary datatype's <registration> tag may have special attributes, proprietary_converter_path and proprietary_display_path.
             proprietary_converter_path = registration.get( 'proprietary_converter_path', None )
             proprietary_display_path = registration.get( 'proprietary_display_path', None )
-            if proprietary_converter_path or proprietary_display_path and not handling_proprietary_datatypes:
+            if proprietary_converter_path is not None or proprietary_display_path is not None and not handling_proprietary_datatypes:
                 handling_proprietary_datatypes = True
             for elem in registration.findall( 'datatype' ):
-                try:
-                    extension = elem.get( 'extension', None )
-                    dtype = elem.get( 'type', None )
-                    type_extension = elem.get( 'type_extension', None )
-                    mimetype = elem.get( 'mimetype', None )
-                    display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) )
-                    make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) )
-                    # Proprietary datatypes included in installed tool shed repositories will include two special attributes
-                    # (proprietary_path and proprietary_datatype_module) if they depend on proprietary datatypes classes.
-                    proprietary_path = elem.get( 'proprietary_path', None )
-                    proprietary_datatype_module = elem.get( 'proprietary_datatype_module', None )
-                    if proprietary_path or proprietary_datatype_module and not handling_proprietary_datatypes:
-                        handling_proprietary_datatypes = True
-                    if deactivate:
-                        # We are deactivating an installed tool shed repository, so eliminate the
-                        # datatype elem from the in-memory list of datatype elems.
-                        for in_memory_elem in self.datatype_elems:
-                            in_memory_extension = in_memory_elem.get( 'extension', None )
-                            if in_memory_extension == extension:
-                                in_memory_dtype = elem.get( 'type', None )
-                                in_memory_type_extension = elem.get( 'type_extension', None )
-                                in_memory_mimetype = elem.get( 'mimetype', None )
-                                in_memory_display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) )
-                                in_memory_make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) )
-                                if in_memory_dtype == dtype and in_memory_type_extension == type_extension and in_memory_mimetype == mimetype \
-                                    and in_memory_display_in_upload == display_in_upload and in_memory_make_subclass == make_subclass:
-                                    self.datatype_elems.remove( in_memory_elem )
-                    else:
-                        # Keep an in-memory list of datatype elems to enable persistence.
-                        if extension not in self.datatypes_by_extension:
-                            self.datatype_elems.append( elem )
-                    if extension and extension in self.datatypes_by_extension and deactivate:
-                        # We are deactivating an installed tool shed repository, so eliminate the datatype from the registry.
-                        # TODO: Handle deactivating datatype converters, etc before removing from self.datatypes_by_extension.
-                        self.log.debug( "Removing datatype with extension '%s' from the registry." % extension )
+                # Keep a status of the process steps to enable stopping the process of handling the datatype if necessary.
+                ok = True
+                extension = elem.get( 'extension', None )
+                dtype = elem.get( 'type', None )
+                type_extension = elem.get( 'type_extension', None )
+                mimetype = elem.get( 'mimetype', None )
+                display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) )
+                # If make_subclass is True, it does not necessarily imply that we are subclassing a datatype that is contained
+                # in the distribution.
+                make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) )
+                # Proprietary datatypes included in installed tool shed repositories will include two special attributes
+                # (proprietary_path and proprietary_datatype_module) if they depend on proprietary datatypes classes.
+                # The value of proprietary_path is the path to the cloned location of the tool shed repository's contained
+                # datatypes_conf.xml file.
+                proprietary_path = elem.get( 'proprietary_path', None )
+                proprietary_datatype_module = elem.get( 'proprietary_datatype_module', None )
+                if proprietary_path is not None or proprietary_datatype_module is not None and not handling_proprietary_datatypes:
+                    handling_proprietary_datatypes = True
+                if deactivate:
+                    # We are deactivating or uninstalling an installed tool shed repository, so eliminate the datatype
+                    # elem from the in-memory list of datatype elems.
+                    for in_memory_elem in self.datatype_elems:
+                        in_memory_extension = in_memory_elem.get( 'extension', None )
+                        if in_memory_extension == extension:
+                            in_memory_dtype = elem.get( 'type', None )
+                            in_memory_type_extension = elem.get( 'type_extension', None )
+                            in_memory_mimetype = elem.get( 'mimetype', None )
+                            in_memory_display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) )
+                            in_memory_make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) )
+                            if in_memory_dtype == dtype and \
+                                in_memory_type_extension == type_extension and \
+                                in_memory_mimetype == mimetype and \
+                                in_memory_display_in_upload == display_in_upload and \
+                                in_memory_make_subclass == make_subclass:
+                                self.datatype_elems.remove( in_memory_elem )
+                    if extension is not None and extension in self.datatypes_by_extension:
+                        # We are deactivating or uninstalling an installed tool shed repository, so eliminate the datatype
+                        # from the registry.  TODO: Handle deactivating datatype converters, etc before removing from
+                        # self.datatypes_by_extension.
                         del self.datatypes_by_extension[ extension ]
                         if extension in self.upload_file_formats:
                             self.upload_file_formats.remove( extension )
-                        can_process_datatype = False
-                    else:
-                        can_process_datatype = ( extension and ( dtype or type_extension ) ) and ( extension not in self.datatypes_by_extension or override )
+                        self.log.debug( "Removed datatype with extension '%s' from the registry." % extension )
+                else:
+                    # We are loading new datatype, so we'll make sure it is correctly defined before proceeding.
+                    can_process_datatype = False
+                    if extension is not None:
+                        if dtype is not None or type_extension is not None:
+                            if override or extension not in self.datatypes_by_extension:
+                                can_process_datatype = True
                     if can_process_datatype:
-                        if dtype:
-                            fields = dtype.split( ':' )
-                            datatype_module = fields[0]
-                            datatype_class_name = fields[1]
-                            datatype_class = None
-                            if proprietary_path and proprietary_datatype_module and datatype_class_name:
-                                # We need to change the value of sys.path, so do it in a way that is thread-safe.
-                                lock = threading.Lock()
-                                lock.acquire( True )
-                                try:
-                                    imported_module = __import_module( proprietary_path, proprietary_datatype_module, datatype_class_name )
-                                    if imported_module not in self.imported_modules:
-                                        self.imported_modules.append( imported_module )
-                                    if hasattr( imported_module, datatype_class_name ):
-                                        datatype_class = getattr( imported_module, datatype_class_name )
-                                except Exception, e:
-                                    full_path = os.path.join( proprietary_path, proprietary_datatype_module )
-                                    self.log.debug( "Exception importing proprietary code file %s: %s" % ( str( full_path ), str( e ) ) )
-                                finally:
-                                    lock.release()
-                            if datatype_class is None:
-                                # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution.
-                                fields = datatype_module.split( '.' )
-                                module = __import__( fields.pop(0) )
-                                for mod in fields:
-                                    module = getattr( module, mod )
-                                datatype_class = getattr( module, datatype_class_name )
-                        elif type_extension:
-                            datatype_class = self.datatypes_by_extension[type_extension].__class__
-                        if make_subclass:
-                            datatype_class = type( datatype_class_name, (datatype_class,), {} )
-                        if extension in self.datatypes_by_extension:
-                            self.log.warning( "Overriding conflicting datatype with extension '%s', using datatype from %s." % ( extension, config ) )
-                        self.datatypes_by_extension[ extension ] = datatype_class()
-                        if mimetype is None:
-                            # Use default mime type as per datatype spec
-                            mimetype = self.datatypes_by_extension[ extension ].get_mime()
-                        self.mimetypes_by_extension[ extension ] = mimetype
-                        if datatype_class.track_type:
-                            self.available_tracks.append( extension )
-                        if display_in_upload and extension not in self.upload_file_formats:
-                            self.upload_file_formats.append( extension )
-                        # Max file size cut off for setting optional metadata
-                        self.datatypes_by_extension[ extension ].max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None )
-                        for converter in elem.findall( 'converter' ):
-                            # Build the list of datatype converters which will later be loaded into the calling app's toolbox.
-                            converter_config = converter.get( 'file', None )
-                            target_datatype = converter.get( 'target_datatype', None )
-                            depends_on = converter.get( 'depends_on', None )
-                            if depends_on and target_datatype:
-                                if extension not in self.converter_deps:
-                                    self.converter_deps[extension] = {}
-                                self.converter_deps[extension][target_datatype] = depends_on.split(',')
-                            if converter_config and target_datatype:
-                                #if imported_modules:
-                                if proprietary_converter_path:
-                                    self.proprietary_converters.append( ( converter_config, extension, target_datatype ) )
-                                else:
-                                    self.converters.append( ( converter_config, extension, target_datatype ) )
-                        for composite_file in elem.findall( 'composite_file' ):
-                            # add composite files
-                            name = composite_file.get( 'name', None )
-                            if name is None:
-                                self.log.warning( "You must provide a name for your composite_file (%s)." % composite_file )
-                            optional = composite_file.get( 'optional', False )
-                            mimetype = composite_file.get( 'mimetype', None )
-                            self.datatypes_by_extension[extension].add_composite_file( name, optional=optional, mimetype=mimetype )
-                        for display_app in elem.findall( 'display' ):
-                            #if imported_modules:
-                            if proprietary_display_path:
-                                if elem not in self.proprietary_display_app_containers:
-                                    self.proprietary_display_app_containers.append( elem )
-                            else:
-                                if elem not in self.display_app_containers:
-                                    self.display_app_containers.append( elem )
-                    elif not deactivate:
-                        # A new tool shed repository that contains proprietary datatypes is being installed, and since installation
-                        # is occurring after the datatypes registry has been initialized, its contents cannot be overridden by new
-                        # introduced conflicting data types.
-                        self.log.warning( "Ignoring conflicting datatype with extension '%s' from %s." % ( extension, config ) )
-                except Exception, e:
-                    if deactivate:
-                        self.log.warning( "Error deactivating datatype with extension '%s': %s" % ( extension, str( e ) ) )
+                        if dtype is not None:
+                            try:
+                                fields = dtype.split( ':' )
+                                datatype_module = fields[ 0 ]
+                                datatype_class_name = fields[ 1 ]
+                            except Exception, e:
+                                self.log.exception( 'Error parsing datatype definition for dtype %s: %s' % ( str( dtype ), str( e ) ) )
+                                ok = False
+                            if ok:
+                                datatype_class = None
+                                if proprietary_path and proprietary_datatype_module and datatype_class_name:
+                                    # We need to change the value of sys.path, so do it in a way that is thread-safe.
+                                    lock = threading.Lock()
+                                    lock.acquire( True )
+                                    try:
+                                        imported_module = __import_module( proprietary_path,
+                                                                           proprietary_datatype_module,
+                                                                           datatype_class_name )
+                                        if imported_module not in self.imported_modules:
+                                            self.imported_modules.append( imported_module )
+                                        if hasattr( imported_module, datatype_class_name ):
+                                            datatype_class = getattr( imported_module, datatype_class_name )
+                                    except Exception, e:
+                                        full_path = os.path.join( proprietary_path, proprietary_datatype_module )
+                                        self.log.debug( "Exception importing proprietary code file %s: %s" % ( str( full_path ), str( e ) ) )
+                                    finally:
+                                        lock.release()
+                                # Either the above exception was thrown because the proprietary_datatype_module is not derived from a class
+                                # in the repository, or we are loading Galaxy's datatypes. In either case we'll look in the registry.
+                                if datatype_class is None:
+                                    try:
+                                        # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution.
+                                        fields = datatype_module.split( '.' )
+                                        module = __import__( fields.pop( 0 ) )
+                                        for mod in fields:
+                                            module = getattr( module, mod )
+                                        datatype_class = getattr( module, datatype_class_name )
+                                        self.log.debug( 'Retrieved datatype module %s from the datatype registry.' % str( datatype_module ) )
+                                    except Exception, e:
+                                        self.log.exception( 'Error importing datatype module %s: %s' % ( str( datatype_module ), str( e ) ) )
+                                        ok = False
+                        elif type_extension is not None:
+                            try:
+                                datatype_class = self.datatypes_by_extension[ type_extension ].__class__
+                            except Exception, e:
+                                self.log.exception( 'Error determining datatype_class for type_extension %s: %s' % ( str( type_extension ), str( e ) ) )
+                                ok = False
+                        if ok:
+                            if not deactivate:
+                                # A new tool shed repository that contains custom datatypes is being installed, and since installation is
+                                # occurring after the datatypes registry has been initialized at server startup, its contents cannot be
+                                # overridden by new introduced conflicting data types unless the value of override is True.
+                                if extension in self.datatypes_by_extension:
+                                    # Because of the way that the value of can_process_datatype was set above, we know that the value of
+                                    # override is True.
+                                    self.log.warning( "Overriding conflicting datatype with extension '%s', using datatype from %s." % \
+                                                      ( str( extension ), str( config ) ) )
+                                if make_subclass:
+                                    datatype_class = type( datatype_class_name, ( datatype_class, ), {} )
+                                self.datatypes_by_extension[ extension ] = datatype_class()
+                                if mimetype is None:
+                                    # Use default mimetype per datatype specification.
+                                    mimetype = self.datatypes_by_extension[ extension ].get_mime()
+                                self.mimetypes_by_extension[ extension ] = mimetype
+                                if datatype_class.track_type:
+                                    self.available_tracks.append( extension )
+                                if display_in_upload and extension not in self.upload_file_formats:
+                                    self.upload_file_formats.append( extension )
+                                # Max file size cut off for setting optional metadata.
+                                self.datatypes_by_extension[ extension ].max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None )
+                                for converter in elem.findall( 'converter' ):
+                                    # Build the list of datatype converters which will later be loaded into the calling app's toolbox.
+                                    converter_config = converter.get( 'file', None )
+                                    target_datatype = converter.get( 'target_datatype', None )
+                                    depends_on = converter.get( 'depends_on', None )
+                                    if depends_on is not None and target_datatype is not None:
+                                        if extension not in self.converter_deps:
+                                            self.converter_deps[ extension ] = {}
+                                        self.converter_deps[ extension ][ target_datatype ] = depends_on.split( ',' )
+                                    if converter_config and target_datatype:
+                                        if proprietary_converter_path:
+                                            self.proprietary_converters.append( ( converter_config, extension, target_datatype ) )
+                                        else:
+                                            self.converters.append( ( converter_config, extension, target_datatype ) )
+                                # Add composite files.
+                                for composite_file in elem.findall( 'composite_file' ):
+                                    name = composite_file.get( 'name', None )
+                                    if name is None:
+                                        self.log.warning( "You must provide a name for your composite_file (%s)." % composite_file )
+                                    optional = composite_file.get( 'optional', False )
+                                    mimetype = composite_file.get( 'mimetype', None )
+                                    self.datatypes_by_extension[ extension ].add_composite_file( name, optional=optional, mimetype=mimetype )
+                                for display_app in elem.findall( 'display' ):
+                                    if proprietary_display_path:
+                                        if elem not in self.proprietary_display_app_containers:
+                                            self.proprietary_display_app_containers.append( elem )
+                                    else:
+                                        if elem not in self.display_app_containers:
+                                            self.display_app_containers.append( elem )
+                                # Processing the new datatype elem is now complete, so make sure the element defining it is retained by appending
+                                # the new datatype to the in-memory list of datatype elems to enable persistence.
+                                self.datatype_elems.append( elem )
                     else:
-                        self.log.warning( "Error loading datatype with extension '%s': %s" % ( extension, str( e ) ) )
-            # Load datatype sniffers from the config
-            sniffers = root.find( 'sniffers' )
-            if sniffers:
-                for elem in sniffers.findall( 'sniffer' ):
-                    # Keep an in-memory list of sniffer elems to enable persistence.
-                    if elem not in self.sniffer_elems:
-                        self.sniffer_elems.append( elem )
-                    dtype = elem.get( 'type', None )
-                    if dtype:
-                        try:
-                            fields = dtype.split( ":" )
-                            datatype_module = fields[0]
-                            datatype_class_name = fields[1]
-                            module = None
-                            #if imported_modules:
-                            if handling_proprietary_datatypes:
-                                # See if one of the imported modules contains the datatype class name.
-                                for imported_module in self.imported_modules:
-                                    if hasattr( imported_module, datatype_class_name ):
-                                        module = imported_module
-                                        break
-                            if module is None:
+                        if extension is not None:
+                            if dtype is not None or type_extension is not None:
+                                if extension in self.datatypes_by_extension:
+                                    if not override:
+                                        # Do not load the datatype since it conflicts with an existing datatype which we are not supposed
+                                        # to override.
+                                        self.log.warning( "Ignoring conflicting datatype with extension '%s' from %s." % ( extension, config ) )
+            # Load datatype sniffers from the config - we'll do this even if one or more datatypes were not properly processed in the config
+            # since sniffers are not tightly coupled with datatypes.
+            self.load_datatype_sniffers( root,
+                                         deactivate=deactivate,
+                                         handling_proprietary_datatypes=handling_proprietary_datatypes,
+                                         override=override )
+            self.upload_file_formats.sort()
+            # Persist the xml form of the registry into a temporary file so that it can be loaded from the command line by tools and
+            # set_metadata processing.
+            self.to_xml_file()
+        self.set_default_values()
+
+        def append_to_sniff_order():
+            # Just in case any supported data types are not included in the config's sniff_order section.
+            for ext in self.datatypes_by_extension:
+                datatype = self.datatypes_by_extension[ ext ]
+                included = False
+                for atype in self.sniff_order:
+                    if isinstance( atype, datatype.__class__ ):
+                        included = True
+                        break
+                if not included:
+                    self.sniff_order.append( datatype )
+        append_to_sniff_order()
+
+    def load_datatype_sniffers( self, root, deactivate=False, handling_proprietary_datatypes=False, override=False ):
+        """
+        Process the sniffers element from a parsed a datatypes XML file located at root_dir/config (if processing the Galaxy
+        distributed config) or contained within an installed Tool Shed repository.  If deactivate is True, an installed Tool
+        Shed repository that includes custom sniffers is being deactivated or uninstalled, so appropriate loaded sniffers will
+        be removed from the registry.  The value of override will be False when a Tool Shed repository is being installed.
+        Since installation is occurring after the datatypes registry has been initialized at server startup, it's contents
+        cannot be overridden by newly introduced conflicting sniffers.
+        """
+        sniffer_elem_classes = [ e.attrib[ 'type' ] for e in self.sniffer_elems ]
+        sniffers = root.find( 'sniffers' )
+        if sniffers:
+            for elem in sniffers.findall( 'sniffer' ):
+                # Keep a status of the process steps to enable stopping the process of handling the sniffer if necessary.
+                ok = True
+                dtype = elem.get( 'type', None )
+                if dtype is not None:
+                    try:
+                        fields = dtype.split( ":" )
+                        datatype_module = fields[ 0 ]
+                        datatype_class_name = fields[ 1 ]
+                        module = None
+                    except Exception, e:
+                        self.log.exception( 'Error determining datatype class or module for dtype %s: %s' % ( str( dtype ), str( e ) ) )
+                        ok = False
+                    if ok:
+                        if handling_proprietary_datatypes:
+                            # See if one of the imported modules contains the datatype class name.
+                            for imported_module in self.imported_modules:
+                                if hasattr( imported_module, datatype_class_name ):
+                                    module = imported_module
+                                    break
+                        if module is None:
+                            try:
                                 # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution.
                                 module = __import__( datatype_module )
                                 for comp in datatype_module.split( '.' )[ 1: ]:
                                     module = getattr( module, comp )
-                            aclass = getattr( module, datatype_class_name )()
-                            if deactivate:
-                                if elem in self.sniffer_elems:
-                                    self.sniffer_elems.remove( elem )
-                                for sniffer_class in self.sniff_order:
-                                    if sniffer_class.__class__ == aclass.__class__:
-                                        self.sniff_order.remove( sniffer_class )
-                                        break
-                                self.log.debug( "Deactivated sniffer for datatype '%s'" % dtype )
-                            else:
-                                # See if we have a conflicting sniffer already loaded.
-                                conflict = False
-                                for conflict_loc, sniffer_class in enumerate( self.sniff_order ):
-                                    if sniffer_class.__class__ == aclass.__class__:
-                                        # We have a conflicting sniffer, so replace the one previously loaded.
-                                        conflict = True
+                            except Exception, e:
+                                self.log.exception( "Error importing datatype class for '%s': %s" % ( str( dtype ), str( e ) ) )
+                                ok = False
+                        if ok:
+                            try:
+                                aclass = getattr( module, datatype_class_name )()
+                            except Exception, e:
+                                self.log.exception( 'Error calling method %s from class %s: %s', str( datatype_class_name ), str( module ), str( e ) )
+                                ok = False
+                            if ok: 
+                                if deactivate:
+                                    # We are deactivating or uninstalling an installed Tool Shed repository, so eliminate the appropriate sniffers.
+                                    sniffer_class = elem.get( 'type', None )
+                                    if sniffer_class is not None:
+                                        for index, s_e_c in enumerate( sniffer_elem_classes ):
+                                            if sniffer_class == s_e_c:
+                                                del self.sniffer_elems[ index ]
+                                                sniffer_elem_classes = [ e.attrib[ 'type' ] for e in self.sniffer_elems ]
+                                                self.log.debug( "Removed sniffer element for datatype '%s'" % str( dtype ) )
+                                                break
+                                        for sniffer_class in self.sniff_order:
+                                            if sniffer_class.__class__ == aclass.__class__:
+                                                self.sniff_order.remove( sniffer_class )
+                                                self.log.debug( "Removed sniffer class for datatype '%s' from sniff order" % str( dtype ) )
+                                                break
+                                else:
+                                    # We are loading new sniffer, so see if we have a conflicting sniffer already loaded.
+                                    conflict = False
+                                    for conflict_loc, sniffer_class in enumerate( self.sniff_order ):
+                                        if sniffer_class.__class__ == aclass.__class__:
+                                            # We have a conflicting sniffer, so replace the one previously loaded.
+                                            conflict = True
+                                            if override:
+                                                del self.sniff_order[ conflict_loc ]
+                                                self.log.debug( "Removed conflicting sniffer for datatype '%s'" % dtype )
+                                            break
+                                    if conflict:
                                         if override:
-                                            del self.sniff_order[ conflict_loc ]
-                                            self.log.debug( "Replaced conflicting sniffer for datatype '%s'" % dtype )
-                                        break
-                                if conflict:
-                                    if override:
+                                            self.sniff_order.append( aclass )
+                                            self.log.debug( "Loaded sniffer for datatype '%s'" % dtype )
+                                    else:
                                         self.sniff_order.append( aclass )
                                         self.log.debug( "Loaded sniffer for datatype '%s'" % dtype )
-                                else:
-                                    self.sniff_order.append( aclass )
-                                    self.log.debug( "Loaded sniffer for datatype '%s'" % dtype )
-                        except Exception, exc:
-                            if deactivate:
-                                self.log.warning( "Error deactivating sniffer for datatype '%s': %s" % ( dtype, str( exc ) ) )
-                            else:
-                                self.log.warning( "Error appending sniffer for datatype '%s' to sniff_order: %s" % ( dtype, str( exc ) ) )
-            self.upload_file_formats.sort()
-            # Persist the xml form of the registry into a temporary file so that it
-            # can be loaded from the command line by tools and set_metadata processing.
-            self.to_xml_file()
-        # Default values.
-        if not self.datatypes_by_extension:
-            self.datatypes_by_extension = {
-                'ab1'         : binary.Ab1(),
-                'axt'         : sequence.Axt(),
-                'bam'         : binary.Bam(),
-                'bed'         : interval.Bed(),
-                'coverage'    : coverage.LastzCoverage(),
-                'customtrack' : interval.CustomTrack(),
-                'csfasta'     : sequence.csFasta(),
-                'fasta'       : sequence.Fasta(),
-                'eland'       : tabular.Eland(),
-                'fastq'       : sequence.Fastq(),
-                'fastqsanger' : sequence.FastqSanger(),
-                'gtf'         : interval.Gtf(),
-                'gff'         : interval.Gff(),
-                'gff3'        : interval.Gff3(),
-                'genetrack'   : tracks.GeneTrack(),
-                'interval'    : interval.Interval(),
-                'laj'         : images.Laj(),
-                'lav'         : sequence.Lav(),
-                'maf'         : sequence.Maf(),
-                'pileup'      : tabular.Pileup(),
-                'qualsolid'   : qualityscore.QualityScoreSOLiD(),
-                'qualsolexa'  : qualityscore.QualityScoreSolexa(),
-                'qual454'     : qualityscore.QualityScore454(),
-                'sam'         : tabular.Sam(),
-                'scf'         : binary.Scf(),
-                'sff'         : binary.Sff(),
-                'tabular'     : tabular.Tabular(),
-                'taxonomy'    : tabular.Taxonomy(),
-                'txt'         : data.Text(),
-                'wig'         : interval.Wiggle(),
-                'xml'         : xml.GenericXml(),
-            }
-            self.mimetypes_by_extension = {
-                'ab1'         : 'application/octet-stream',
-                'axt'         : 'text/plain',
-                'bam'         : 'application/octet-stream',
-                'bed'         : 'text/plain',
-                'customtrack' : 'text/plain',
-                'csfasta'     : 'text/plain',
-                'eland'       : 'application/octet-stream',
-                'fasta'       : 'text/plain',
-                'fastq'       : 'text/plain',
-                'fastqsanger' : 'text/plain',
-                'gtf'         : 'text/plain',
-                'gff'         : 'text/plain',
-                'gff3'        : 'text/plain',
-                'interval'    : 'text/plain',
-                'laj'         : 'text/plain',
-                'lav'         : 'text/plain',
-                'maf'         : 'text/plain',
-                'memexml'     : 'application/xml',
-                'pileup'      : 'text/plain',
-                'qualsolid'   : 'text/plain',
-                'qualsolexa'  : 'text/plain',
-                'qual454'     : 'text/plain',
-                'sam'         : 'text/plain',
-                'scf'         : 'application/octet-stream',
-                'sff'         : 'application/octet-stream',
-                'tabular'     : 'text/plain',
-                'taxonomy'    : 'text/plain',
-                'txt'         : 'text/plain',
-                'wig'         : 'text/plain',
-                'xml'         : 'application/xml',
-            }
-        # super supertype fix for input steps in workflows.
-        if 'data' not in self.datatypes_by_extension:
-            self.datatypes_by_extension['data'] = data.Data()
-            self.mimetypes_by_extension['data'] = 'application/octet-stream'
-        # Default values - the order in which we attempt to determine data types is critical
-        # because some formats are much more flexibly defined than others.
-        if len(self.sniff_order) < 1:
-            self.sniff_order = [
-                binary.Bam(),
-                binary.Sff(),
-                xml.GenericXml(),
-                sequence.Maf(),
-                sequence.Lav(),
-                sequence.csFasta(),
-                qualityscore.QualityScoreSOLiD(),
-                qualityscore.QualityScore454(),
-                sequence.Fasta(),
-                sequence.Fastq(),
-                interval.Wiggle(),
-                images.Html(),
-                sequence.Axt(),
-                interval.Bed(),
-                interval.CustomTrack(),
-                interval.Gtf(),
-                interval.Gff(),
-                interval.Gff3(),
-                tabular.Pileup(),
-                interval.Interval(),
-                tabular.Sam(),
-                tabular.Eland()
-            ]
-        def append_to_sniff_order():
-            # Just in case any supported data types are not included in the config's sniff_order section.
-            for ext in self.datatypes_by_extension:
-                datatype = self.datatypes_by_extension[ext]
-                included = False
-                for atype in self.sniff_order:
-                    if isinstance(atype, datatype.__class__):
-                        included = True
-                        break
-                if not included:
-                    self.sniff_order.append(datatype)
-        append_to_sniff_order()
+                                    # Processing the new sniffer elem is now complete, so make sure the element defining it is loaded if necessary.
+                                    sniffer_class = elem.get( 'type', None )
+                                    if sniffer_class is not None:
+                                        if sniffer_class not in sniffer_elem_classes:
+                                            self.sniffer_elems.append( elem )
 
     def get_datatype_class_by_name( self, name ):
         """
@@ -414,25 +419,28 @@ class Registry( object ):
         #
         #return datatype
 
-    def get_available_tracks(self):
+    def get_available_tracks( self ):
         return self.available_tracks
-    def get_mimetype_by_extension(self, ext, default = 'application/octet-stream' ):
+
+    def get_mimetype_by_extension( self, ext, default='application/octet-stream' ):
         """Returns a mimetype based on an extension"""
         try:
-            mimetype = self.mimetypes_by_extension[ext]
+            mimetype = self.mimetypes_by_extension[ ext ]
         except KeyError:
             #datatype was never declared
             mimetype = default
-            self.log.warning('unknown mimetype in data factory %s' % ext)
+            self.log.warning( 'unknown mimetype in data factory %s' % str( ext ) )
         return mimetype
-    def get_datatype_by_extension(self, ext ):
+
+    def get_datatype_by_extension( self, ext ):
         """Returns a datatype based on an extension"""
         try:
-            builder = self.datatypes_by_extension[ext]
+            builder = self.datatypes_by_extension[ ext ]
         except KeyError:
             builder = data.Text()
         return builder
-    def change_datatype(self, data, ext):
+
+    def change_datatype( self, data, ext ):
         data.extension = ext
         # call init_meta and copy metadata from itself.  The datatype
         # being converted *to* will handle any metadata copying and
@@ -441,13 +449,15 @@ class Registry( object ):
             data.set_size()
             data.init_meta( copy_from=data )
         return data
-    def old_change_datatype(self, data, ext):
+
+    def old_change_datatype( self, data, ext ):
         """Creates and returns a new datatype based on an existing data and an extension"""
-        newdata = factory(ext)(id=data.id)
+        newdata = factory( ext )( id=data.id )
         for key, value in data.__dict__.items():
-            setattr(newdata, key, value)
+            setattr( newdata, key, value )
         newdata.ext = ext
         return newdata
+
     def load_datatype_converters( self, toolbox, installed_repository_dict=None, deactivate=False ):
         """
         If deactivate is False, add datatype converters from self.converters or self.proprietary_converters
@@ -461,9 +471,9 @@ class Registry( object ):
             # Load converters defined by local datatypes_conf.xml.
             converters = self.converters
         for elem in converters:
-            tool_config = elem[0]
-            source_datatype = elem[1]
-            target_datatype = elem[2]
+            tool_config = elem[ 0 ]
+            source_datatype = elem[ 1 ]
+            target_datatype = elem[ 2 ]
             if installed_repository_dict:
                 converter_path = installed_repository_dict[ 'converter_path' ]
             else:
@@ -490,7 +500,8 @@ class Registry( object ):
                     if converter.id in toolbox.tools_by_id:
                         del toolbox.tools_by_id[ converter.id ]
                     if source_datatype in self.datatype_converters:
-                        del self.datatype_converters[ source_datatype ][ target_datatype ]
+                        if target_datatype in self.datatype_converters[ source_datatype ]:
+                            del self.datatype_converters[ source_datatype ][ target_datatype ]
                     self.log.debug( "Deactivated converter: %s", converter.id )
                 else:
                     toolbox.tools_by_id[ converter.id ] = converter
@@ -503,6 +514,7 @@ class Registry( object ):
                     self.log.exception( "Error deactivating converter from (%s): %s" % ( converter_path, str( e ) ) )
                 else:
                     self.log.exception( "Error loading converter (%s): %s" % ( converter_path, str( e ) ) )
+
     def load_display_applications( self, installed_repository_dict=None, deactivate=False ):
         """
         If deactivate is False, add display applications from self.display_app_containers or
@@ -577,6 +589,7 @@ class Registry( object ):
                 if current_app is None and isinstance( d_type1, type( d_type2 ) ):
                     self.log.debug( "Adding inherited display application '%s' to datatype '%s'" % ( display_app.id, extension ) )
                     d_type1.add_display_application( display_app )
+
     def load_external_metadata_tool( self, toolbox ):
         """Adds a tool which is used to set external metadata"""
         # We need to be able to add a job to the queue to set metadata. The queue will currently only accept jobs with an associated
@@ -585,6 +598,9 @@ class Registry( object ):
         tool_xml_text = """
             <tool id="__SET_METADATA__" name="Set External Metadata" version="1.0.1" tool_type="set_metadata">
               <type class="SetMetadataTool" module="galaxy.tools"/>
+              <requirements>
+                  <requirement type="package">samtools</requirement>
+              </requirements>
               <action module="galaxy.tools.actions.metadata" class="SetMetadataToolAction"/>
               <command>$__SET_EXTERNAL_METADATA_COMMAND_LINE__</command>
               <inputs>
@@ -601,25 +617,128 @@ class Registry( object ):
         toolbox.tools_by_id[ set_meta_tool.id ] = set_meta_tool
         self.set_external_metadata_tool = set_meta_tool
         self.log.debug( "Loaded external metadata tool: %s", self.set_external_metadata_tool.id )
-    def get_converters_by_datatype(self, ext):
+
+    def set_default_values( self ):
+        # Default values.
+        if not self.datatypes_by_extension:
+            self.datatypes_by_extension = {
+                'ab1'         : binary.Ab1(),
+                'axt'         : sequence.Axt(),
+                'bam'         : binary.Bam(),
+                'bed'         : interval.Bed(),
+                'coverage'    : coverage.LastzCoverage(),
+                'customtrack' : interval.CustomTrack(),
+                'csfasta'     : sequence.csFasta(),
+                'fasta'       : sequence.Fasta(),
+                'eland'       : tabular.Eland(),
+                'fastq'       : sequence.Fastq(),
+                'fastqsanger' : sequence.FastqSanger(),
+                'gtf'         : interval.Gtf(),
+                'gff'         : interval.Gff(),
+                'gff3'        : interval.Gff3(),
+                'genetrack'   : tracks.GeneTrack(),
+                'interval'    : interval.Interval(),
+                'laj'         : images.Laj(),
+                'lav'         : sequence.Lav(),
+                'maf'         : sequence.Maf(),
+                'pileup'      : tabular.Pileup(),
+                'qualsolid'   : qualityscore.QualityScoreSOLiD(),
+                'qualsolexa'  : qualityscore.QualityScoreSolexa(),
+                'qual454'     : qualityscore.QualityScore454(),
+                'sam'         : tabular.Sam(),
+                'scf'         : binary.Scf(),
+                'sff'         : binary.Sff(),
+                'tabular'     : tabular.Tabular(),
+                'taxonomy'    : tabular.Taxonomy(),
+                'txt'         : data.Text(),
+                'wig'         : interval.Wiggle(),
+                'xml'         : xml.GenericXml(),
+            }
+            self.mimetypes_by_extension = {
+                'ab1'         : 'application/octet-stream',
+                'axt'         : 'text/plain',
+                'bam'         : 'application/octet-stream',
+                'bed'         : 'text/plain',
+                'customtrack' : 'text/plain',
+                'csfasta'     : 'text/plain',
+                'eland'       : 'application/octet-stream',
+                'fasta'       : 'text/plain',
+                'fastq'       : 'text/plain',
+                'fastqsanger' : 'text/plain',
+                'gtf'         : 'text/plain',
+                'gff'         : 'text/plain',
+                'gff3'        : 'text/plain',
+                'interval'    : 'text/plain',
+                'laj'         : 'text/plain',
+                'lav'         : 'text/plain',
+                'maf'         : 'text/plain',
+                'memexml'     : 'application/xml',
+                'pileup'      : 'text/plain',
+                'qualsolid'   : 'text/plain',
+                'qualsolexa'  : 'text/plain',
+                'qual454'     : 'text/plain',
+                'sam'         : 'text/plain',
+                'scf'         : 'application/octet-stream',
+                'sff'         : 'application/octet-stream',
+                'tabular'     : 'text/plain',
+                'taxonomy'    : 'text/plain',
+                'txt'         : 'text/plain',
+                'wig'         : 'text/plain',
+                'xml'         : 'application/xml',
+            }
+        # super supertype fix for input steps in workflows.
+        if 'data' not in self.datatypes_by_extension:
+            self.datatypes_by_extension[ 'data' ] = data.Data()
+            self.mimetypes_by_extension[ 'data' ] = 'application/octet-stream'
+        # Default values - the order in which we attempt to determine data types is critical
+        # because some formats are much more flexibly defined than others.
+        if len( self.sniff_order ) < 1:
+            self.sniff_order = [
+                binary.Bam(),
+                binary.Sff(),
+                xml.GenericXml(),
+                sequence.Maf(),
+                sequence.Lav(),
+                sequence.csFasta(),
+                qualityscore.QualityScoreSOLiD(),
+                qualityscore.QualityScore454(),
+                sequence.Fasta(),
+                sequence.Fastq(),
+                interval.Wiggle(),
+                images.Html(),
+                sequence.Axt(),
+                interval.Bed(),
+                interval.CustomTrack(),
+                interval.Gtf(),
+                interval.Gff(),
+                interval.Gff3(),
+                tabular.Pileup(),
+                interval.Interval(),
+                tabular.Sam(),
+                tabular.Eland()
+            ]
+
+    def get_converters_by_datatype( self, ext ):
         """Returns available converters by source type"""
         converters = odict()
-        source_datatype = type(self.get_datatype_by_extension(ext))
+        source_datatype = type( self.get_datatype_by_extension( ext ) )
         for ext2, dict in self.datatype_converters.items():
-            converter_datatype = type(self.get_datatype_by_extension(ext2))
-            if issubclass(source_datatype, converter_datatype):
-                converters.update(dict)
+            converter_datatype = type( self.get_datatype_by_extension( ext2 ) )
+            if issubclass( source_datatype, converter_datatype ):
+                converters.update( dict )
         #Ensure ext-level converters are present
         if ext in self.datatype_converters.keys():
-            converters.update(self.datatype_converters[ext])
+            converters.update( self.datatype_converters[ ext ] )
         return converters
-    def get_converter_by_target_type(self, source_ext, target_ext):
+
+    def get_converter_by_target_type( self, source_ext, target_ext ):
         """Returns a converter based on source and target datatypes"""
-        converters = self.get_converters_by_datatype(source_ext)
+        converters = self.get_converters_by_datatype( source_ext )
         if target_ext in converters.keys():
-            return converters[target_ext]
+            return converters[ target_ext ]
         return None
-    def find_conversion_destination_for_dataset_by_extensions( self, dataset, accepted_formats, converter_safe = True ):
+
+    def find_conversion_destination_for_dataset_by_extensions( self, dataset, accepted_formats, converter_safe=True ):
         """Returns ( target_ext, existing converted dataset )"""
         for convert_ext in self.get_converters_by_datatype( dataset.ext ):
             if self.get_datatype_by_extension( convert_ext ).matches_any( accepted_formats ):
@@ -632,8 +751,10 @@ class Registry( object ):
                     ret_data = None
                 return ( convert_ext, ret_data )
         return ( None, None )
+
     def get_composite_extensions( self ):
         return [ ext for ( ext, d_type ) in self.datatypes_by_extension.iteritems() if d_type.composite_type is not None ]
+
     def get_upload_metadata_params( self, context, group, tool ):
         """Returns dict of case value:inputs for metadata conditional for upload tool"""
         rval = {}
@@ -649,16 +770,17 @@ class Registry( object ):
         if 'auto' not in rval and 'txt' in rval: #need to manually add 'auto' datatype
             rval[ 'auto' ] = rval[ 'txt' ]
         return rval
+
     @property
     def integrated_datatypes_configs( self ):
         if self.xml_filename and os.path.isfile( self.xml_filename ):
             return self.xml_filename
         self.to_xml_file()
         return self.xml_filename
+
     def to_xml_file( self ):
         if self.xml_filename is not None:
-            # If persisted previously, attempt to remove
-            # the temporary file in which we were written.
+            # If persisted previously, attempt to remove the temporary file in which we were written.
             try:
                 os.unlink( self.xml_filename )
             except:

lib/galaxy/datatypes/sequence.py

@@ -2,12 +2,14 @@
 Sequence classes
 """
 
-import data
+from . import data
 import gzip
+import json
 import logging
 import os
 import re
 import string
+
 from cgi import escape
 
 from galaxy import eggs, util
@@ -16,8 +18,6 @@ from galaxy.datatypes.checkers import is_gzip
 from galaxy.datatypes.sniff import get_test_fname, get_headers
 from galaxy.datatypes.metadata import MetadataElement
 
-eggs.require("simplejson")
-import simplejson
 
 try:
     eggs.require( "bx-python" )
@@ -44,8 +44,8 @@ class SequenceSplitLocations( data.Text ):
     def set_peek( self, dataset, is_multi_byte=False ):
         if not dataset.dataset.purged:
             try:
-                parsed_data = simplejson.load(open(dataset.file_name))
-                # dataset.peek = simplejson.dumps(data, sort_keys=True, indent=4)
+                parsed_data = json.load(open(dataset.file_name))
+                # dataset.peek = json.dumps(data, sort_keys=True, indent=4)
                 dataset.peek = data.get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte )
                 dataset.blurb = '%d sections' % len(parsed_data['sections'])
             except Exception, e:
@@ -60,7 +60,7 @@ class SequenceSplitLocations( data.Text ):
     def sniff( self, filename ):
         if os.path.getsize(filename) < 50000:
             try:
-                data = simplejson.load(open(filename))
+                data = json.load(open(filename))
                 sections = data['sections']
                 for section in sections:
                     if 'start' not in section or 'end' not in section or 'sequences' not in section:
@@ -155,7 +155,7 @@ class Sequence( data.Text ):
     do_slow_split = classmethod(do_slow_split)
 
     def do_fast_split( cls, input_datasets, toc_file_datasets, subdir_generator_function, split_params):
-        data = simplejson.load(open(toc_file_datasets[0].file_name))
+        data = json.load(open(toc_file_datasets[0].file_name))
         sections = data['sections']
         total_sequences = long(0)
         for section in sections:
@@ -191,7 +191,7 @@ class Sequence( data.Text ):
                     toc = toc_file_datasets[ds_no]
                     split_data['args']['toc_file'] = toc.file_name
                 f = open(os.path.join(dir, 'split_info_%s.json' % base_name), 'w')
-                simplejson.dump(split_data, f)
+                json.dump(split_data, f)
                 f.close()
             start_sequence += sequences_per_file[part_no]
         return directories
@@ -557,7 +557,7 @@ class Fastq ( Sequence ):
         sequence_count = long(args['num_sequences'])
 
         if 'toc_file' in args:
-            toc_file = simplejson.load(open(args['toc_file'], 'r'))
+            toc_file = json.load(open(args['toc_file'], 'r'))
             commands = Sequence.get_split_commands_with_toc(input_name, output_name, toc_file, start_sequence, sequence_count)
         else:
             commands = Sequence.get_split_commands_sequential(is_gzip(input_name), input_name, output_name, start_sequence, sequence_count)

lib/galaxy/datatypes/tabular.py

@@ -275,7 +275,7 @@ class Tabular( data.Text ):
             cursor = f.read(1)
         return to_json_string( { 'ck_data': util.unicodify( ck_data ), 'ck_index': ck_index + 1 } )
 
-    def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None):
+    def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None, **kwd):
         preview = util.string_as_bool( preview )
         if chunk:
             return self.get_chunk(trans, dataset, chunk)
@@ -652,7 +652,7 @@ class Vcf( Tabular ):
 
     MetadataElement( name="columns", default=10, desc="Number of columns", readonly=True, visible=False )
     MetadataElement( name="column_types", default=['str','int','str','str','str','int','str','list','str','str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False )
-    MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, multiple=True, visible=False )
+    MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, optional=True, multiple=True, visible=False )
     MetadataElement( name="sample_names", default=[], desc="Sample names", readonly=True, visible=False, optional=True, no_value=[] )
 
     def sniff( self, filename ):

lib/galaxy/datatypes/util/gff_util.py

@@ -75,7 +75,7 @@ class GFFFeature( GFFInterval ):
         # Preference for name: GTF, GFF3, GFF.
         for attr_name in [
                            # GTF:
-                           'transcript_id', 'gene_id',
+                           'gene_id', 'transcript_id',
                            # GFF3:
                            'ID', 'id',
                            # GFF (TODO):

lib/galaxy/exceptions/__init__.py

@@ -1,34 +1,71 @@
 """
 Custom exceptions for Galaxy
 """
+
+from galaxy import eggs
+eggs.require( "Paste" )
+
 from paste import httpexceptions
+from ..exceptions import error_codes
+
 
 class MessageException( Exception ):
     """
-    Exception to make throwing errors from deep in controllers easier
+    Exception to make throwing errors from deep in controllers easier.
     """
-    def __init__( self, err_msg, type="info" ):
-        self.err_msg = err_msg
+    # status code to be set when used with API.
+    status_code = 400
+    # Error code information embedded into API json responses.
+    err_code = error_codes.UNKNOWN
+
+    def __init__( self, err_msg=None, type="info", **extra_error_info ):
+        self.err_msg = err_msg or self.err_code.default_error_message
         self.type = type
+        self.extra_error_info = extra_error_info
+
     def __str__( self ):
         return self.err_msg
 
+
 class ItemDeletionException( MessageException ):
     pass
 
+
 class ItemAccessibilityException( MessageException ):
-    pass
+    status_code = 403
+    err_code = error_codes.USER_CANNOT_ACCESS_ITEM
+
 
 class ItemOwnershipException( MessageException ):
-    pass
+    status_code = 403
+    err_code = error_codes.USER_DOES_NOT_OWN_ITEM
+
+
+class DuplicatedSlugException( MessageException ):
+    status_code = 400
+    err_code = error_codes.USER_SLUG_DUPLICATE
+
+
+class ObjectAttributeInvalidException( MessageException ):
+    status_code = 400
+    err_code = error_codes.USER_OBJECT_ATTRIBUTE_INVALID
+
+
+class ObjectAttributeMissingException( MessageException ):
+    status_code = 400
+    err_code = error_codes.USER_OBJECT_ATTRIBUTE_MISSING
+
 
 class ActionInputError( MessageException ):
     def __init__( self, err_msg, type="error" ):
         super( ActionInputError, self ).__init__( err_msg, type )
 
-class ObjectNotFound( Exception ):
+
+class ObjectNotFound( MessageException ):
     """ Accessed object was not found """
-    pass
+    status_code = 404
+    err_code = error_codes.USER_OBJECT_NOT_FOUND
+
 
 class ObjectInvalid( Exception ):
     """ Accessed object store ID is invalid """

lib/galaxy/exceptions/error_codes.json

@@ -0,0 +1,57 @@
+[
+   {
+    "name": "UNKNOWN",
+    "code": 0,
+    "message": "Unknown error occurred while processing request."
+    },
+   {
+    "name": "USER_CANNOT_RUN_AS",
+    "code": 400001,
+    "message": "User does not have permissions to run jobs as another user."
+    },
+   {
+    "name": "USER_INVALID_RUN_AS",
+    "code": 400002,
+    "message": "Invalid run_as request - run_as user does not exist."
+    },
+   {
+    "name": "USER_INVALID_JSON",
+    "code": 400003,
+    "message": "Your request did not appear to be valid JSON, please consult the API documentation."
+    },
+   {
+    "name": "USER_OBJECT_ATTRIBUTE_INVALID",
+    "code": 400004,
+    "message": "Attempted to create or update object with invalid attribute value."
+    },
+   {
+    "name": "USER_OBJECT_ATTRIBUTE_MISSING",
+    "code": 400005,
+    "message": "Attempted to create object without required attribute."
+    },
+   {
+    "name": "USER_SLUG_DUPLICATE",
+    "code": 400006,
+    "message": "Slug must be unique per user."
+    },
+   {
+    "name": "USER_NO_API_KEY",
+    "code": 403001,
+    "message": "API Authentication Required for this request"
+    },
+   {
+    "name": "USER_CANNOT_ACCESS_ITEM",
+    "code": 403002,
+    "message": "User cannot access specified item."
+    },
+   {
+    "name": "USER_DOES_NOT_OWN_ITEM",
+    "code": 403003,
+    "message": "User does not own specified item."
+   },
+   {
+    "name": "USER_OBJECT_NOT_FOUND",
+    "code": 404001,
+    "message": "No such object found."
+   }
+]

lib/galaxy/exceptions/error_codes.py

@@ -0,0 +1,32 @@
+from pkg_resources import resource_string
+from json import loads
+
+# Error codes are provided as a convience to Galaxy API clients, but at this
+# time they do represent part of the more stable interface. They can change
+# without warning between releases.
+UNKNOWN_ERROR_MESSAGE = "Unknown error occurred while processing request."
+
+
+class ErrorCode( object ):
+
+    def __init__( self, code, default_error_message ):
+        self.code = code
+        self.default_error_message = default_error_message or UNKNOWN_ERROR_MESSAGE
+
+    def __str__( self ):
+        return str( self.default_error_message )
+
+    def __int__( self ):
+        return int( self.code )
+
+    @staticmethod
+    def from_dict( entry ):
+        name = entry.get("name")
+        code = entry.get("code")
+        message = entry.get("message")
+        return ( name, ErrorCode( code, message ) )
+
+error_codes_json = resource_string( __name__, 'error_codes.json' )
+for entry in loads( error_codes_json ):
+    name, error_code_obj = ErrorCode.from_dict( entry )
+    globals()[ name ] = error_code_obj

lib/galaxy/jobs/__init__.py

@@ -14,7 +14,6 @@ import re
 import shutil
 import subprocess
 import sys
-import threading
 import traceback
 from galaxy import model, util
 from galaxy.datatypes import metadata
@@ -39,21 +38,6 @@ DATABASE_MAX_STRING_SIZE_PRETTY = util.DATABASE_MAX_STRING_SIZE_PRETTY
 # and should eventually become API'd
 TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'
 
-class Sleeper( object ):
-    """
-    Provides a 'sleep' method that sleeps for a number of seconds *unless*
-    the notify method is called (from a different thread).
-    """
-    def __init__( self ):
-        self.condition = threading.Condition()
-    def sleep( self, seconds ):
-        self.condition.acquire()
-        self.condition.wait( seconds )
-        self.condition.release()
-    def wake( self ):
-        self.condition.acquire()
-        self.condition.notify()
-        self.condition.release()
 
 class JobDestination( Bunch ):
     """
@@ -74,6 +58,7 @@ class JobDestination( Bunch ):
         if self.tags is not None:
             self['tags'] = [ x.strip() for x in self.tags.split(',') ]
 
+
 class JobToolConfiguration( Bunch ):
     """
     Provides details on what handler and destination a tool should use
@@ -87,18 +72,21 @@ class JobToolConfiguration( Bunch ):
         self['params'] = dict()
         super(JobToolConfiguration, self).__init__(**kwds)
 
+
 class JobConfiguration( object ):
     """A parser and interface to advanced job management features.
 
     These features are configured in the job configuration, by default, ``job_conf.xml``
     """
     DEFAULT_NWORKERS = 4
+
     def __init__(self, app):
         """Parse the job configuration XML.
         """
         self.app = app
         self.runner_plugins = []
         self.handlers = {}
+        self.handler_runner_plugins = {}
         self.default_handler_id = None
         self.destinations = {}
         self.destination_tags = {}
@@ -151,6 +139,10 @@ class JobConfiguration( object ):
                 else:
                     log.debug("Read definition for handler '%s'" % id)
                     self.handlers[id] = (id,)
+                    for plugin in handler.findall('plugin'):
+                        if id not in self.handler_runner_plugins:
+                            self.handler_runner_plugins[id] = []
+                        self.handler_runner_plugins[id].append( plugin.get('id') )
                     if handler.get('tags', None) is not None:
                         for tag in [ x.strip() for x in handler.get('tags').split(',') ]:
                             if tag in self.handlers:
@@ -188,17 +180,17 @@ class JobConfiguration( object ):
                 self.tools[id].append(JobToolConfiguration(**dict(tool.items())))
                 self.tools[id][-1]['params'] = self.__get_params(tool)
 
-        types = dict(registered_user_concurrent_jobs = int,
-                     anonymous_user_concurrent_jobs = int,
-                     walltime = str,
-                     output_size = int)
+        types = dict(registered_user_concurrent_jobs=int,
+                     anonymous_user_concurrent_jobs=int,
+                     walltime=str,
+                     output_size=int)
 
-        self.limits = Bunch(registered_user_concurrent_jobs = None,
-                            anonymous_user_concurrent_jobs = None,
-                            walltime = None,
-                            walltime_delta = None,
-                            output_size = None,
-                            concurrent_jobs = {})
+        self.limits = Bunch(registered_user_concurrent_jobs=None,
+                            anonymous_user_concurrent_jobs=None,
+                            walltime=None,
+                            walltime_delta=None,
+                            output_size=None,
+                            concurrent_jobs={})
 
         # Parse job limits
         limits = root.find('limits')
@@ -269,12 +261,12 @@ class JobConfiguration( object ):
         self.default_destination_id = self.app.config.default_cluster_job_runner
 
         # Set the job limits
-        self.limits = Bunch(registered_user_concurrent_jobs = self.app.config.registered_user_job_limit,
-                            anonymous_user_concurrent_jobs = self.app.config.anonymous_user_job_limit,
-                            walltime = self.app.config.job_walltime,
-                            walltime_delta = self.app.config.job_walltime_delta,
-                            output_size = self.app.config.output_size_limit,
-                            concurrent_jobs = {})
+        self.limits = Bunch(registered_user_concurrent_jobs=self.app.config.registered_user_job_limit,
+                            anonymous_user_concurrent_jobs=self.app.config.anonymous_user_job_limit,
+                            walltime=self.app.config.job_walltime,
+                            walltime_delta=self.app.config.job_walltime_delta,
+                            output_size=self.app.config.output_size_limit,
+                            concurrent_jobs={})
 
         log.debug('Done loading job configuration')
 
@@ -433,13 +425,19 @@ class JobConfiguration( object ):
         """
         return self.destinations.get(id_or_tag, None)
 
-    def get_job_runner_plugins(self):
+    def get_job_runner_plugins(self, handler_id):
         """Load all configured job runner plugins
 
         :returns: list of job runner plugins
         """
         rval = {}
-        for runner in self.runner_plugins:
+        if handler_id in self.handler_runner_plugins:
+            plugins_to_load = [ rp for rp in self.runner_plugins if rp['id'] in self.handler_runner_plugins[handler_id] ]
+            log.info( "Handler '%s' will load specified runner plugins: %s", handler_id, ', '.join( [ rp['id'] for rp in plugins_to_load ] ) )
+        else:
+            plugins_to_load = self.runner_plugins
+            log.info( "Handler '%s' will load all configured runner plugins", handler_id )
+        for runner in plugins_to_load:
             class_names = []
             module = None
             id = runner['id']
@@ -490,7 +488,7 @@ class JobConfiguration( object ):
                 try:
                     rval[id] = runner_class( self.app, runner[ 'workers' ], **runner.get( 'kwds', {} ) )
                 except TypeError:
-                    log.warning( "Job runner '%s:%s' has not been converted to a new-style runner" % ( module_name, class_name ) )
+                    log.exception( "Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load" % ( module_name, class_name ) )
                     rval[id] = runner_class( self.app )
                 log.debug( "Loaded job runner '%s:%s' as '%s'" % ( module_name, class_name, id ) )
         return rval
@@ -549,6 +547,7 @@ class JobConfiguration( object ):
                 else:
                     log.warning("Legacy destination with id '%s' could not be converted: Unknown runner plugin: %s" % (id, destination.runner))
 
+
 class JobWrapper( object ):
     """
     Wraps a 'model.Job' with convenience methods for running processes and
@@ -643,7 +642,7 @@ class JobWrapper( object ):
         Prepare the job to run by creating the working directory and the
         config files.
         """
-        self.sa_session.expunge_all() #this prevents the metadata reverting that has been seen in conjunction with the PBS job runner
+        self.sa_session.expunge_all()  # this prevents the metadata reverting that has been seen in conjunction with the PBS job runner
 
         if not os.path.exists( self.working_directory ):
             os.mkdir( self.working_directory )
@@ -702,12 +701,9 @@ class JobWrapper( object ):
         self.command_line = self.tool.build_command_line( param_dict )
         # FIXME: for now, tools get Galaxy's lib dir in their path
         if self.command_line and self.command_line.startswith( 'python' ):
-            self.galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root
+            self.galaxy_lib_dir = os.path.abspath( "lib" )  # cwd = galaxy root
         # Shell fragment to inject dependencies
-        if self.app.config.use_tool_dependencies:
-            self.dependency_shell_commands = self.tool.build_dependency_shell_commands()
-        else:
-            self.dependency_shell_commands = None
+        self.dependency_shell_commands = self.tool.build_dependency_shell_commands()
         # We need command_line persisted to the db in order for Galaxy to re-queue the job
         # if the server was stopped and restarted before the job finished
         job.command_line = self.command_line
@@ -738,7 +734,7 @@ class JobWrapper( object ):
                 job.traceback = traceback.format_exc()
                 # Get the exception and let the tool attempt to generate
                 # a better message
-                etype, evalue, tb =  sys.exc_info()
+                etype, evalue, tb = sys.exc_info()
                 m = self.tool.handle_job_failure_exception( evalue )
                 if m:
                     message = m
@@ -793,8 +789,8 @@ class JobWrapper( object ):
         # If the job was deleted, call tool specific fail actions (used for e.g. external metadata) and clean up
         if self.tool:
             self.tool.job_failed( self, message, exception )
-        if self.app.config.cleanup_job == 'always' or (self.app.config.cleanup_job == 'onsuccess' and job.state == job.states.DELETED):
-            self.cleanup()
+        delete_files = self.app.config.cleanup_job == 'always' or (self.app.config.cleanup_job == 'onsuccess' and job.state == job.states.DELETED)
+        self.cleanup( delete_files=delete_files )
 
     def pause( self, job=None, message=None ):
         if job is None:
@@ -809,7 +805,7 @@ class JobWrapper( object ):
             job.state = job.states.PAUSED
             self.sa_session.add( job )
 
-    def change_state( self, state, info = False ):
+    def change_state( self, state, info=False ):
         job = self.get_job()
         self.sa_session.refresh( job )
         for dataset_assoc in job.output_datasets + job.output_library_datasets:
@@ -835,7 +831,7 @@ class JobWrapper( object ):
         log.warning('set_runner() is deprecated, use set_job_destination()')
         self.set_job_destination(self.job_destination, external_id)
 
-    def set_job_destination(self, job_destination, external_id=None ):
+    def set_job_destination( self, job_destination, external_id=None ):
         """
         Persist job destination params in the database for recovery.
 
@@ -887,11 +883,10 @@ class JobWrapper( object ):
 
         # We set final_job_state to use for dataset management, but *don't* set
         # job.state until after dataset collection to prevent history issues
-        if job.states.ERROR != job.state:
-            if ( self.check_tool_output( stdout, stderr, tool_exit_code, job )):
-                final_job_state = job.states.OK
-            else:
-                final_job_state = job.states.ERROR
+        if ( self.check_tool_output( stdout, stderr, tool_exit_code, job ) ):
+            final_job_state = job.states.OK
+        else:
+            final_job_state = job.states.ERROR
 
         if self.version_string_cmd:
             version_filename = self.get_version_string_path()
@@ -915,11 +910,11 @@ class JobWrapper( object ):
                         job.state = final_job_state
                         return self.fail( "Job %s's output dataset(s) could not be read" % job.id )
 
-        job_context = ExpressionContext( dict( stdout = job.stdout, stderr = job.stderr ) )
+        job_context = ExpressionContext( dict( stdout=job.stdout, stderr=job.stderr ) )
         for dataset_assoc in job.output_datasets + job.output_library_datasets:
             context = self.get_dataset_finish_context( job_context, dataset_assoc.dataset.dataset )
             #should this also be checking library associations? - can a library item be added from a history before the job has ended? - lets not allow this to occur
-            for dataset in dataset_assoc.dataset.dataset.history_associations + dataset_assoc.dataset.dataset.library_associations: #need to update all associated output hdas, i.e. history was shared with job running
+            for dataset in dataset_assoc.dataset.dataset.history_associations + dataset_assoc.dataset.dataset.library_associations:  # need to update all associated output hdas, i.e. history was shared with job running
                 trynum = 0
                 while trynum < self.app.config.retry_job_output_collection:
                     try:
@@ -930,9 +925,9 @@ class JobWrapper( object ):
                     except ( OSError, ObjectNotFound ), e:
                         trynum += 1
                         log.warning( 'Error accessing %s, will retry: %s', dataset.dataset.file_name, e )
-                        time.sleep( 2 ) 
+                        time.sleep( 2 )
                 dataset.blurb = 'done'
-                dataset.peek  = 'no peek'
+                dataset.peek = 'no peek'
                 dataset.info = (dataset.info or '')
                 if context['stdout'].strip():
                     #Ensure white space between entries
@@ -960,7 +955,7 @@ class JobWrapper( object ):
                     #it would be quicker to just copy the metadata from the originating output dataset,
                     #but somewhat trickier (need to recurse up the copied_from tree), for now we'll call set_meta()
                     if ( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and self.app.config.retry_metadata_internally ):
-                        dataset.datatype.set_meta( dataset, overwrite = False ) #call datatype.set_meta directly for the initial set_meta call during dataset creation
+                        dataset.datatype.set_meta( dataset, overwrite=False )  # call datatype.set_meta directly for the initial set_meta call during dataset creation
                     elif not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and job.states.ERROR != final_job_state:
                         dataset._state = model.Dataset.states.FAILED_METADATA
                     else:
@@ -1030,19 +1025,22 @@ class JobWrapper( object ):
         out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] )
         inp_data.update( [ ( da.name, da.dataset ) for da in job.input_library_datasets ] )
         out_data.update( [ ( da.name, da.dataset ) for da in job.output_library_datasets ] )
-        param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] ) # why not re-use self.param_dict here? ##dunno...probably should, this causes tools.parameters.basic.UnvalidatedValue to be used in following methods instead of validated and transformed values during i.e. running workflows
+        param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] )  # why not re-use self.param_dict here? ##dunno...probably should, this causes tools.parameters.basic.UnvalidatedValue to be used in following methods instead of validated and transformed values during i.e. running workflows
         param_dict = self.tool.params_from_strings( param_dict, self.app )
         # Check for and move associated_files
         self.tool.collect_associated_files(out_data, self.working_directory)
         gitd = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first()
         if gitd:
-            self.tool.collect_associated_files({'' : gitd}, self.working_directory)
+            self.tool.collect_associated_files({'': gitd}, self.working_directory)
         # Create generated output children and primary datasets and add to param_dict
-        collected_datasets = {'children':self.tool.collect_child_datasets(out_data, self.working_directory),'primary':self.tool.collect_primary_datasets(out_data, self.working_directory)}
-        param_dict.update({'__collected_datasets__':collected_datasets})
+        collected_datasets = {
+            'children': self.tool.collect_child_datasets(out_data, self.working_directory),
+            'primary': self.tool.collect_primary_datasets(out_data, self.working_directory)
+        }
+        param_dict.update({'__collected_datasets__': collected_datasets})
         # Certain tools require tasks to be completed after job execution
         # ( this used to be performed in the "exec_after_process" hook, but hooks are deprecated ).
-        self.tool.exec_after_process( self.queue.app, inp_data, out_data, param_dict, job = job )
+        self.tool.exec_after_process( self.queue.app, inp_data, out_data, param_dict, job=job )
         # Call 'exec_after_process' hook
         self.tool.call_hook( 'exec_after_process', self.queue.app, inp_data=inp_data,
                              out_data=out_data, param_dict=param_dict,
@@ -1068,22 +1066,26 @@ class JobWrapper( object ):
         self.sa_session.flush()
 
         log.debug( 'job %d ended' % self.job_id )
-        if self.app.config.cleanup_job == 'always' or ( not stderr and self.app.config.cleanup_job == 'onsuccess' ):
-            self.cleanup()
+        delete_files = self.app.config.cleanup_job == 'always' or ( job.state == job.states.OK and self.app.config.cleanup_job == 'onsuccess' )
+        self.cleanup( delete_files=delete_files )
 
     def check_tool_output( self, stdout, stderr, tool_exit_code, job ):
         return check_output( self.tool, stdout, stderr, tool_exit_code, job )
 
-    def cleanup( self ):
-        # remove temporary files
+    def cleanup( self, delete_files=True ):
+        # At least one of these tool cleanup actions (job import), is needed
+        # for thetool to work properly, that is why one might want to run
+        # cleanup but not delete files.
         try:
-            for fname in self.extra_filenames:
-                os.remove( fname )
-            self.external_output_metadata.cleanup_external_metadata( self.sa_session )
+            if delete_files:
+                for fname in self.extra_filenames:
+                    os.remove( fname )
+                self.external_output_metadata.cleanup_external_metadata( self.sa_session )
             galaxy.tools.imp_exp.JobExportHistoryArchiveWrapper( self.job_id ).cleanup_after_job( self.sa_session )
             galaxy.tools.imp_exp.JobImportHistoryArchiveWrapper( self.app, self.job_id ).cleanup_after_job()
             galaxy.tools.genome_index.GenomeIndexToolWrapper( self.job_id ).postprocessing( self.sa_session, self.app )
-            self.app.object_store.delete(self.get_job(), base_dir='job_work', entire_dir=True, dir_only=True, extra_dir=str(self.job_id))
+            if delete_files:
+                self.app.object_store.delete(self.get_job(), base_dir='job_work', entire_dir=True, dir_only=True, extra_dir=str(self.job_id))
         except:
             log.exception( "Unable to cleanup job %d" % self.job_id )
 
@@ -1133,7 +1135,7 @@ class JobWrapper( object ):
     def get_input_fnames( self ):
         job = self.get_job()
         filenames = []
-        for da in job.input_datasets + job.input_library_datasets: #da is JobToInputDatasetAssociation object
+        for da in job.input_datasets + job.input_library_datasets:  # da is JobToInputDatasetAssociation object
             if da.dataset:
                 filenames.extend(self.get_input_dataset_fnames(da.dataset))
         return filenames
@@ -1155,11 +1157,12 @@ class JobWrapper( object ):
 
     def compute_outputs( self ) :
         class DatasetPath( object ):
-            def __init__( self, dataset_id, real_path, false_path = None, mutable = True ):
+            def __init__( self, dataset_id, real_path, false_path=None, mutable=True ):
                 self.dataset_id = dataset_id
                 self.real_path = real_path
                 self.false_path = false_path
                 self.mutable = mutable
+
             def __str__( self ):
                 if self.false_path is None:
                     return self.real_path
@@ -1176,13 +1179,13 @@ class JobWrapper( object ):
             self.output_hdas_and_paths = {}
             for name, hda in [ ( da.name, da.dataset ) for da in job.output_datasets + job.output_library_datasets ]:
                 false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % hda.dataset.id ) )
-                dsp = DatasetPath( hda.dataset.id, hda.dataset.file_name, false_path, mutable = hda.dataset.external_filename is None  )
+                dsp = DatasetPath( hda.dataset.id, hda.dataset.file_name, false_path, mutable=hda.dataset.external_filename is None  )
                 self.output_paths.append( dsp )
                 self.output_hdas_and_paths[name] = hda, dsp
             if special:
                 false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % special.dataset.id ) )
         else:
-            results = [ ( da.name, da.dataset, DatasetPath( da.dataset.dataset.id, da.dataset.file_name, mutable = da.dataset.dataset.external_filename is None ) ) for da in job.output_datasets + job.output_library_datasets ]
+            results = [ ( da.name, da.dataset, DatasetPath( da.dataset.dataset.id, da.dataset.file_name, mutable=da.dataset.dataset.external_filename is None ) ) for da in job.output_datasets + job.output_library_datasets ]
             self.output_paths = [t[2] for t in results]
             self.output_hdas_and_paths = dict([(t[0],  t[1:]) for t in results])
         if special:
@@ -1257,13 +1260,13 @@ class JobWrapper( object ):
             datatypes_config = self.app.datatypes_registry.integrated_datatypes_configs
         return self.external_output_metadata.setup_external_metadata( [ output_dataset_assoc.dataset for output_dataset_assoc in job.output_datasets ],
                                                                       self.sa_session,
-                                                                      exec_dir = exec_dir,
-                                                                      tmp_dir = tmp_dir,
-                                                                      dataset_files_path = dataset_files_path,
-                                                                      config_root = config_root,
-                                                                      config_file = config_file,
-                                                                      datatypes_config = datatypes_config,
-                                                                      job_metadata = os.path.join( self.working_directory, TOOL_PROVIDED_JOB_METADATA_FILE ),
+                                                                      exec_dir=exec_dir,
+                                                                      tmp_dir=tmp_dir,
+                                                                      dataset_files_path=dataset_files_path,
+                                                                      config_root=config_root,
+                                                                      config_file=config_file,
+                                                                      datatypes_config=datatypes_config,
+                                                                      job_metadata=os.path.join( self.working_directory, TOOL_PROVIDED_JOB_METADATA_FILE ),
                                                                       **kwds )
 
     @property
@@ -1337,7 +1340,7 @@ class JobWrapper( object ):
         just copy these files directly to the ulimate destination.
         """
         return output_path
-    
+
     @property
     def requires_setting_metadata( self ):
         if self.tool:
@@ -1410,7 +1413,7 @@ class TaskWrapper(JobWrapper):
         out_data.update( [ ( da.name, da.dataset ) for da in job.output_library_datasets ] )
         # DBTODO New method for generating command line for a task?
         # These can be passed on the command line if wanted as $userId $userEmail
-        if job.history and job.history.user: # check for anonymous user!
+        if job.history and job.history.user:  # check for anonymous user!
             userId = '%d' % job.history.user.id
             userEmail = str(job.history.user.email)
         else:
@@ -1449,12 +1452,9 @@ class TaskWrapper(JobWrapper):
             self.command_line = self.command_line.replace(k, v)
         # FIXME: for now, tools get Galaxy's lib dir in their path
         if self.command_line and self.command_line.startswith( 'python' ):
-            self.galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root
+            self.galaxy_lib_dir = os.path.abspath( "lib" )  # cwd = galaxy root
         # Shell fragment to inject dependencies
-        if self.app.config.use_tool_dependencies:
-            self.dependency_shell_commands = self.tool.build_dependency_shell_commands()
-        else:
-            self.dependency_shell_commands = None
+        self.dependency_shell_commands = self.tool.build_dependency_shell_commands()
         # We need command_line persisted to the db in order for Galaxy to re-queue the job
         # if the server was stopped and restarted before the job finished
         task.command_line = self.command_line
@@ -1474,7 +1474,7 @@ class TaskWrapper(JobWrapper):
         self.status = 'error'
         # How do we want to handle task failure?  Fail the job and let it clean up?
 
-    def change_state( self, state, info = False ):
+    def change_state( self, state, info=False ):
         task = self.get_task()
         self.sa_session.refresh( task )
         if info:
@@ -1523,8 +1523,8 @@ class TaskWrapper(JobWrapper):
         # if the job was deleted, don't finish it
         if task.state == task.states.DELETED:
             # Job was deleted by an administrator
-            if self.app.config.cleanup_job in ( 'always', 'onsuccess' ):
-                self.cleanup()
+            delete_files = self.app.config.cleanup_job in ( 'always', 'onsuccess' )
+            self.cleanup( delete_files=delete_files )
             return
         elif task.state == task.states.ERROR:
             self.fail( task.info )
@@ -1590,11 +1590,14 @@ class NoopQueue( object ):
     """
     def put( self, *args, **kwargs ):
         return
+
     def put_stop( self, *args ):
         return
+
     def shutdown( self ):
         return
 
+
 class ParallelismInfo(object):
     """
     Stores the information (if any) for running multiple instances of the tool in parallel
@@ -1606,7 +1609,7 @@ class ParallelismInfo(object):
             items = tag.iteritems()
         else:
             items = tag.attrib.items()
-        self.attributes = dict([item for item in items if item[0] != 'method' ])
+        self.attributes = dict( [ item for item in items if item[ 0 ] != 'method' ])
         if len(self.attributes) == 0:
             # legacy basic mode - provide compatible defaults
             self.attributes['split_size'] = 20

lib/galaxy/jobs/actions/post.py

@@ -12,7 +12,7 @@ def get_form_template(action_type, title, content, help, on_output = True ):
         form = """
             if (pja.action_type == "%s"){
                 p_str = "<div class='pjaForm toolForm'><span class='action_tag' style='display:none'>"+ pja.action_type + pja.output_name + "</span><div class='toolFormTitle'> %s <br/> on " + pja.output_name + "\
-                <div style='float: right;' class='buttons'><img src='/static/images/delete_icon.png'></div></div><div class='toolFormBody'>";
+                <div style='float: right;' class='buttons'><img src='/static/images/history-buttons/delete_icon.png'></div></div><div class='toolFormBody'>";
                 %s
                 p_str += "</div><div class='toolParamHelp'>%s</div></div>";
             }""" % (action_type, title, content, help)
@@ -20,7 +20,7 @@ def get_form_template(action_type, title, content, help, on_output = True ):
         form =  """
             if (pja.action_type == "%s"){
                 p_str = "<div class='pjaForm toolForm'><span class='action_tag' style='display:none'>"+ pja.action_type + "</span><div class='toolFormTitle'> %s \
-                <div style='float: right;' class='buttons'><img src='/static/images/delete_icon.png'></div></div><div class='toolFormBody'>";
+                <div style='float: right;' class='buttons'><img src='/static/images/history-buttons/delete_icon.png'></div></div><div class='toolFormBody'>";
                 %s
                 p_str += "</div><div class='toolParamHelp'>%s</div></div>";
             }""" % (action_type, title, content, help)

lib/galaxy/jobs/command_factory.py

@@ -1,8 +1,11 @@
 from os import getcwd
 from os.path import abspath
 
+CAPTURE_RETURN_CODE = "return_code=$?"
+YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"'
 
-def build_command( job, job_wrapper, include_metadata=False, include_work_dir_outputs=True ):
+
+def build_command( runner, job_wrapper, include_metadata=False, include_work_dir_outputs=True, remote_command_params={} ):
     """
     Compose the sequence of commands necessary to execute a job. This will
     currently include:
@@ -13,64 +16,125 @@ def build_command( job, job_wrapper, include_metadata=False, include_work_dir_ou
         - commands to set metadata (if include_metadata is True)
     """
 
-    commands = job_wrapper.get_command_line()
+    commands_builder = CommandsBuilder(job_wrapper.get_command_line())
 
     # All job runners currently handle this case which should never occur
-    if not commands:
+    if not commands_builder.commands:
         return None
 
-    # Remove trailing semi-colon so we can start hacking up this command.
-    # TODO: Refactor to compose a list and join with ';', would be more clean.
-    commands = commands.rstrip("; ")
+    __handle_version_command(commands_builder, job_wrapper)
+    __handle_task_splitting(commands_builder, job_wrapper)
+    __handle_dependency_resolution(commands_builder, job_wrapper, remote_command_params)
 
+    if include_work_dir_outputs:
+        __handle_work_dir_outputs(commands_builder, job_wrapper, runner, remote_command_params)
+
+    if include_metadata and job_wrapper.requires_setting_metadata:
+        __handle_metadata(commands_builder, job_wrapper, runner, remote_command_params)
+
+    return commands_builder.build()
+
+
+def __handle_version_command(commands_builder, job_wrapper):
     # Prepend version string
     if job_wrapper.version_string_cmd:
-        commands = "%s &> %s; " % ( job_wrapper.version_string_cmd, job_wrapper.get_version_string_path() ) + commands
+        version_command = "%s > %s 2>&1" % ( job_wrapper.version_string_cmd, job_wrapper.get_version_string_path() )
+        commands_builder.prepend_command(version_command)
 
+
+def __handle_task_splitting(commands_builder, job_wrapper):
     # prepend getting input files (if defined)
-    if hasattr(job_wrapper, 'prepare_input_files_cmds') and job_wrapper.prepare_input_files_cmds is not None:
-        commands = "; ".join( job_wrapper.prepare_input_files_cmds + [ commands ] )
+    if getattr(job_wrapper, 'prepare_input_files_cmds', None):
+        commands_builder.prepend_commands(job_wrapper.prepare_input_files_cmds)
+
+
+def __handle_dependency_resolution(commands_builder, job_wrapper, remote_command_params):
+    local_dependency_resolution = remote_command_params.get("dependency_resolution", "local") == "local"
 
     # Prepend dependency injection
-    if job_wrapper.dependency_shell_commands:
-        commands = "; ".join( job_wrapper.dependency_shell_commands + [ commands ] )
+    if job_wrapper.dependency_shell_commands and local_dependency_resolution:
+        commands_builder.prepend_commands(job_wrapper.dependency_shell_commands)
 
-    # Coping work dir outputs or setting metadata will mask return code of
-    # tool command. If these are used capture the return code and ensure
-    # the last thing that happens is an exit with return code.
-    capture_return_code_command = "; return_code=$?"
-    captured_return_code = False
 
+def __handle_work_dir_outputs(commands_builder, job_wrapper, runner, remote_command_params):
     # Append commands to copy job outputs based on from_work_dir attribute.
-    if include_work_dir_outputs:
-        work_dir_outputs = job.get_work_dir_outputs( job_wrapper )
-        if work_dir_outputs:
-            if not captured_return_code:
-                commands += capture_return_code_command
-                captured_return_code = True
+    work_dir_outputs_kwds = {}
+    if 'working_directory' in remote_command_params:
+        work_dir_outputs_kwds['job_working_directory'] = remote_command_params['working_directory']
+    work_dir_outputs = runner.get_work_dir_outputs( job_wrapper, **work_dir_outputs_kwds )
+    if work_dir_outputs:
+        commands_builder.capture_return_code()
+        copy_commands = map(__copy_if_exists_command, work_dir_outputs)
+        commands_builder.append_commands(copy_commands)
 
-            commands += "; " + "; ".join( [ "if [ -f %s ] ; then cp %s %s ; fi" %
-                ( source_file, source_file, destination ) for ( source_file, destination ) in work_dir_outputs ] )
 
+def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_params):
     # Append metadata setting commands, we don't want to overwrite metadata
     # that was copied over in init_meta(), as per established behavior
-    if include_metadata and job_wrapper.requires_setting_metadata:
-        metadata_command = job_wrapper.setup_external_metadata(
-            exec_dir=abspath( getcwd() ),
-            tmp_dir=job_wrapper.working_directory,
-            dataset_files_path=job.app.model.Dataset.file_path,
-            output_fnames=job_wrapper.get_output_fnames(),
-            set_extension=False,
-            kwds={ 'overwrite' : False }
-        ) or ''
-        metadata_command = metadata_command.strip()
-        if metadata_command:
-            if not captured_return_code:
-                commands += capture_return_code_command
-                captured_return_code = True
-            commands += "; cd %s; %s" % (abspath( getcwd() ), metadata_command)
+    metadata_kwds = remote_command_params.get('metadata_kwds', {})
+    exec_dir = metadata_kwds.get( 'exec_dir', abspath( getcwd() ) )
+    tmp_dir = metadata_kwds.get( 'tmp_dir', job_wrapper.working_directory )
+    dataset_files_path = metadata_kwds.get( 'dataset_files_path', runner.app.model.Dataset.file_path )
+    output_fnames = metadata_kwds.get( 'output_fnames', job_wrapper.get_output_fnames() )
+    config_root = metadata_kwds.get( 'config_root', None )
+    config_file = metadata_kwds.get( 'config_file', None )
+    datatypes_config = metadata_kwds.get( 'datatypes_config', None )
+    metadata_command = job_wrapper.setup_external_metadata(
+        exec_dir=exec_dir,
+        tmp_dir=tmp_dir,
+        dataset_files_path=dataset_files_path,
+        output_fnames=output_fnames,
+        set_extension=False,
+        config_root=config_root,
+        config_file=config_file,
+        datatypes_config=datatypes_config,
+        kwds={ 'overwrite' : False }
+    ) or ''
+    metadata_command = metadata_command.strip()
+    if metadata_command:
+        commands_builder.capture_return_code()
+        commands_builder.append_command("cd %s; %s" % (exec_dir, metadata_command))
 
-    if captured_return_code:
-        commands += '; sh -c "exit $return_code"'
 
-    return commands
+def __copy_if_exists_command(work_dir_output):
+    source_file, destination = work_dir_output
+    return "if [ -f %s ] ; then cp %s %s ; fi" % ( source_file, source_file, destination )
+
+
+class CommandsBuilder(object):
+
+    def __init__(self, initial_command):
+        # Remove trailing semi-colon so we can start hacking up this command.
+        # TODO: Refactor to compose a list and join with ';', would be more clean.
+        commands = initial_command.rstrip("; ")
+        self.commands = commands
+
+        # Coping work dir outputs or setting metadata will mask return code of
+        # tool command. If these are used capture the return code and ensure
+        # the last thing that happens is an exit with return code.
+        self.return_code_captured = False
+
+    def prepend_command(self, command):
+        self.commands = "%s; %s" % (command, self.commands)
+        return self
+
+    def prepend_commands(self, commands):
+        return self.prepend_command("; ".join(commands))
+
+    def append_command(self, command):
+        self.commands = "%s; %s" % (self.commands, command)
+
+    def append_commands(self, commands):
+        self.append_command("; ".join(commands))
+
+    def capture_return_code(self):
+        if not self.return_code_captured:
+            self.return_code_captured = True
+            self.append_command(CAPTURE_RETURN_CODE)
+
+    def build(self):
+        if self.return_code_captured:
+            self.append_command(YIELD_CAPTURED_CODE)
+        return self.commands
+
+__all__ = [build_command]

lib/galaxy/jobs/handler.py

@@ -11,7 +11,8 @@ from Queue import Queue, Empty
 from sqlalchemy.sql.expression import and_, or_, select, func
 
 from galaxy import model
-from galaxy.jobs import Sleeper, JobWrapper, TaskWrapper, JobDestination
+from galaxy.util.sleeper import Sleeper
+from galaxy.jobs import JobWrapper, TaskWrapper, JobDestination
 
 log = logging.getLogger( __name__ )
 
@@ -106,7 +107,7 @@ class JobHandlerQueue( object ):
             if job.tool_id not in self.app.toolbox.tools_by_id:
                 log.warning( "(%s) Tool '%s' removed from tool config, unable to recover job" % ( job.id, job.tool_id ) )
                 JobWrapper( job, self ).fail( 'This tool was disabled before the job completed.  Please contact your Galaxy administrator.' )
-            if job.job_runner_name is not None and job.job_runner_external_id is None:
+            elif job.job_runner_name is not None and job.job_runner_external_id is None:
                 # This could happen during certain revisions of Galaxy where a runner URL was persisted before the job was dispatched to a runner.
                 log.debug( "(%s) Job runner assigned but no external ID recorded, adding to the job handler queue" % job.id )
                 job.job_runner_name = None
@@ -564,7 +565,7 @@ class JobHandlerStopQueue( object ):
 class DefaultJobDispatcher( object ):
     def __init__( self, app ):
         self.app = app
-        self.job_runners = self.app.job_config.get_job_runner_plugins()
+        self.job_runners = self.app.job_config.get_job_runner_plugins( self.app.config.server_name )
         # Once plugins are loaded, all job destinations that were created from
         # URLs can have their URL params converted to the destination's param
         # dict by the plugin.

lib/galaxy/jobs/manager.py

@@ -11,7 +11,8 @@ import time
 from Queue import Empty, Queue
 
 from galaxy import model
-from galaxy.jobs import handler, JobWrapper, NoopQueue, Sleeper
+from galaxy.util.sleeper import Sleeper
+from galaxy.jobs import handler, JobWrapper, NoopQueue
 from galaxy.util.json import from_json_string
 
 log = logging.getLogger( __name__ )

lib/galaxy/jobs/runners/__init__.py

@@ -15,19 +15,48 @@ import galaxy.jobs
 from galaxy.jobs.command_factory import build_command
 from galaxy import model
 from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size
+from galaxy.util import in_directory
 from galaxy.jobs.runners.util.job_script import job_script
 
 log = logging.getLogger( __name__ )
 
 STOP_SIGNAL = object()
 
+
+class RunnerParams( object ):
+
+    def __init__( self, specs = None, params = None ):
+        self.specs = specs or dict()
+        self.params = params or dict()
+        for name, value in self.params.items():
+            assert name in self.specs, 'Invalid job runner parameter for this plugin: %s' % name
+            if 'map' in self.specs[ name ]:
+                try:
+                    self.params[ name ] = self.specs[ name ][ 'map' ]( value )
+                except Exception, e:
+                    raise Exception( 'Job runner parameter "%s" value "%s" could not be converted to the correct type: %s' % ( name, value, e ) )
+            if 'valid' in self.specs[ name ]:
+                assert self.specs[ name ][ 'valid' ]( value ), 'Job runner parameter %s failed validation' % name
+
+    def __getattr__( self, name ):
+        return self.params.get( name, self.specs[ name ][ 'default' ] )
+
+    __getitem__ = __getattr__
+
+
 class BaseJobRunner( object ):
-    def __init__( self, app, nworkers ):
+    def __init__( self, app, nworkers, **kwargs ):
         """Start the job runner
         """
         self.app = app
         self.sa_session = app.model.context
         self.nworkers = nworkers
+        runner_param_specs = dict( recheck_missing_job_retries = dict( map = int, valid = lambda x: x >= 0, default = 0 ) )
+        if 'runner_param_specs' in kwargs:
+            runner_param_specs.update( kwargs.pop( 'runner_param_specs' ) )
+        if kwargs:
+            log.debug( 'Loading %s with params: %s', self.runner_name, kwargs )
+        self.runner_params = RunnerParams( specs = runner_param_specs, params = kwargs )
 
     def _init_worker_threads(self):
         """Start ``nworkers`` worker threads.
@@ -114,7 +143,7 @@ class BaseJobRunner( object ):
                 job_wrapper.cleanup()
             return False
         elif job_state != model.Job.states.QUEUED:
-            log.info( "(%d) Job is in state %s, skipping execution"  % ( job_id, job_state ) )
+            log.info( "(%s) Job is in state %s, skipping execution"  % ( job_id, job_state ) )
             # cleanup may not be safe in all states
             return False
 
@@ -146,23 +175,13 @@ class BaseJobRunner( object ):
     def build_command_line( self, job_wrapper, include_metadata=False, include_work_dir_outputs=True ):
         return build_command( self, job_wrapper, include_metadata=include_metadata, include_work_dir_outputs=include_work_dir_outputs )
 
-    def get_work_dir_outputs( self, job_wrapper ):
+    def get_work_dir_outputs( self, job_wrapper, job_working_directory=None ):
         """
         Returns list of pairs (source_file, destination) describing path
         to work_dir output file and ultimate destination.
         """
-
-        def in_directory( file, directory ):
-            """
-            Return true, if the common prefix of both is equal to directory
-            e.g. /a/b/c/d.rst and directory is /a/b, the common prefix is /a/b
-            """
-
-            # Make both absolute.
-            directory = os.path.abspath( directory )
-            file = os.path.abspath( file )
-
-            return os.path.commonprefix( [ file, directory ] ) == directory
+        if not job_working_directory:
+            job_working_directory = os.path.abspath( job_wrapper.working_directory )
 
         # Set up dict of dataset id --> output path; output path can be real or
         # false depending on outputs_to_working_directory
@@ -186,9 +205,9 @@ class BaseJobRunner( object ):
                         if hda_tool_output and hda_tool_output.from_work_dir:
                             # Copy from working dir to HDA.
                             # TODO: move instead of copy to save time?
-                            source_file = os.path.join( os.path.abspath( job_wrapper.working_directory ), hda_tool_output.from_work_dir )
+                            source_file = os.path.join( job_working_directory, hda_tool_output.from_work_dir )
                             destination = job_wrapper.get_output_destination( output_paths[ dataset.dataset_id ] )
-                            if in_directory( source_file, job_wrapper.working_directory ):
+                            if in_directory( source_file, job_working_directory ):
                                 output_pairs.append( ( source_file, destination ) )
                                 log.debug( "Copying %s to %s as directed by from_work_dir" % ( source_file, destination ) )
                             else:
@@ -196,7 +215,7 @@ class BaseJobRunner( object ):
                                 log.exception( "from_work_dir specified a location not in the working directory: %s, %s" % ( source_file, job_wrapper.working_directory ) )
         return output_pairs
 
-    def _handle_metadata_externally(self, job_wrapper):
+    def _handle_metadata_externally( self, job_wrapper, resolve_requirements=False ):
         """
         Set metadata externally. Used by the local and lwr job runners where this
         shouldn't be attached to command-line to execute.
@@ -210,7 +229,19 @@ class BaseJobRunner( object ):
                                                                             tmp_dir=job_wrapper.working_directory,
                                                                             #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
                                                                             kwds={ 'overwrite' : False } )
+            if resolve_requirements:
+                dependency_shell_commands = self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands()
+                if dependency_shell_commands:
+                    if isinstance( dependency_shell_commands, list ):
+                        dependency_shell_commands = "&&".join( dependency_shell_commands )
+                    external_metadata_script = "%s&&%s" % ( dependency_shell_commands, external_metadata_script )
             log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) )
+            if resolve_requirements:
+                dependency_shell_commands = self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands()
+                if dependency_shell_commands:
+                    if isinstance( dependency_shell_commands, list ):
+                        dependency_shell_commands = "&&".join( dependency_shell_commands )
+                    external_metadata_script = "%s&&%s" % ( dependency_shell_commands, external_metadata_script )
             external_metadata_proc = subprocess.Popen( args=external_metadata_script,
                                                        shell=True,
                                                        env=os.environ,
@@ -229,6 +260,10 @@ class BaseJobRunner( object ):
         options.update(**kwds)
         return job_script(**options)
 
+    def _complete_terminal_job( self, ajs, **kwargs ):
+        if ajs.job_wrapper.get_state() != model.Job.states.DELETED:
+            self.work_queue.put( ( self.finish_job, ajs ) )
+
 
 class AsynchronousJobState( object ):
     """
@@ -290,8 +325,8 @@ class AsynchronousJobRunner( BaseJobRunner ):
     to the correct methods (queue, finish, cleanup) at appropriate times..
     """
 
-    def __init__( self, app, nworkers ):
-        super( AsynchronousJobRunner, self ).__init__( app, nworkers )
+    def __init__( self, app, nworkers, **kwargs ):
+        super( AsynchronousJobRunner, self ).__init__( app, nworkers, **kwargs )
         # 'watched' and 'queue' are both used to keep track of jobs to watch.
         # 'queue' is used to add new watched jobs, and can be called from
         # any thread (usually by the 'queue_job' method). 'watched' must only

lib/galaxy/jobs/runners/cli_job/torque.py

@@ -35,7 +35,8 @@ cd %s
 echo $? > %s
 """
 
-argmap = { 'Execution_Time'     : '-a',
+argmap = { 'destination'        : '-q',
+           'Execution_Time'     : '-a',
            'Account_Name'       : '-A',
            'Checkpoint'         : '-c',
            'Error_Path'         : '-e',