diff --git a/.hgignore b/.hgignore index 86a0898b2ae..a4452c6a0e3 100644 --- a/.hgignore +++ b/.hgignore @@ -60,6 +60,7 @@ shed_tool_data_table_conf.xml job_conf.xml data_manager_conf.xml shed_data_manager_conf.xml +object_store_conf.xml config/* static/welcome.html.* static/welcome.html diff --git a/.hgtags b/.hgtags index 94361505583..c7dca74e32f 100644 --- a/.hgtags +++ b/.hgtags @@ -4,3 +4,5 @@ a4113cc1cb5eaa68091c9a73375f00555b66dd11 release_2013.01.13 2cc8d10988e03257dc7b97f8bb332c7df745d1dd security_2013.04.08 524f246ca85395082719ae7a6ff72260d7ad5612 release_2013.06.03 1ae95b3aa98d1ccf15b243ac3ce6a895eb7efc53 release_2013.08.12 +26f58e05aa1068761660681583821e21e6cbf7ab release_2013.11.04 +5e605ed6069fe4c5ca9875e95e91b2713499e8ca release_2014.02.10 diff --git a/buildbot_setup.sh b/buildbot_setup.sh index 8eb2f73a372..abbf7a169b9 100644 --- a/buildbot_setup.sh +++ b/buildbot_setup.sh @@ -4,28 +4,6 @@ cd `dirname $0` : ${HOSTTYPE:=`uname -m`} -# link to HYPHY is arch-dependent -case "$OSTYPE" in - linux-gnu) - kernel=`uname -r | cut -f1,2 -d.` - HYPHY="/galaxy/software/linux$kernel-$HOSTTYPE/hyphy" - ;; - darwin*) - this_minor=`uname -r | awk -F. '{print ($1-4)}'` - machine=`machine` - for minor in `jot - 3 $this_minor 1`; do - HYPHY="/galaxy/software/macosx10.$minor-$machine/hyphy" - [ -d "$HYPHY" ] && break - done - [ ! -d "$HYPHY" ] && unset HYPHY - ;; - solaris2.10) - # For the psu-production builder which is Solaris, but jobs run on a - # Linux cluster - HYPHY="/galaxy/software/linux2.6-x86_64/hyphy" - ;; -esac - LINKS=" /galaxy/data/location/add_scores.loc /galaxy/data/location/all_fasta.loc @@ -121,12 +99,6 @@ case $type in ln -sf $link tool-data done - if [ -d "$HYPHY" ]; then - echo "Linking $HYPHY" - rm -f tool-data/HYPHY - ln -sf $HYPHY tool-data/HYPHY - fi - if [ -d "$JARS" ]; then echo "Linking $JARS" rm -f tool-data/shared/jars diff --git a/config/plugins/visualizations/additional_template_paths.xml b/config/plugins/visualizations/additional_template_paths.xml new file mode 100644 index 00000000000..d3ad2a03df1 --- /dev/null +++ b/config/plugins/visualizations/additional_template_paths.xml @@ -0,0 +1,5 @@ + + + + common/templates + diff --git a/config/plugins/visualizations/charts/config/charts.xml b/config/plugins/visualizations/charts/config/charts.xml new file mode 100644 index 00000000000..98623288dc3 --- /dev/null +++ b/config/plugins/visualizations/charts/config/charts.xml @@ -0,0 +1,15 @@ + + + + + + HistoryDatasetAssociation + tabular.Tabular + dataset_id + + + + dataset_id + + + diff --git a/config/plugins/visualizations/charts/sample.txt b/config/plugins/visualizations/charts/sample.txt new file mode 100644 index 00000000000..a75b7f2aa9c --- /dev/null +++ b/config/plugins/visualizations/charts/sample.txt @@ -0,0 +1,9 @@ +43 55 63 99 28 85 20 31 47 14 +1 63 94 83 94 74 7 73 51 7 +8 34 2 94 87 21 72 51 3 84 +5 50 40 30 59 43 19 77 30 80 +22 55 69 43 77 72 12 54 53 56 +36 34 81 28 59 64 54 29 57 69 +57 90 93 32 45 98 40 16 52 83 +6 17 39 86 10 11 38 69 19 50 +67 51 47 22 58 27 65 49 92 96 \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/app.js b/config/plugins/visualizations/charts/static/app.js new file mode 100644 index 00000000000..ac925d7f7c8 --- /dev/null +++ b/config/plugins/visualizations/charts/static/app.js @@ -0,0 +1,92 @@ +// dependencies +define(['library/portlet', 'library/ui', 'library/utils', + 'views/charts', 'views/viewport', 'views/chart', 'views/group', + 'models/config', 'models/datasets', 'models/chart', 'models/charts', 'models/group', 'models/types'], + function( Portlet, Ui, Utils, + ChartsView, ViewportView, ChartView, GroupView, + Config, Datasets, Chart, Charts, Group, Types + ) { + +// widget +return Backbone.View.extend( +{ + // initialize + initialize: function(options) + { + // link options + this.options = options; + + // link galaxy + this.modal = parent.Galaxy.modal; + + // create configuration model + this.config = new Config(); + + // create chart models + this.types = new Types(); + this.chart = new Chart(); + this.charts = new Charts(); + this.group = new Group(); + + // create dataset handler + this.datasets = new Datasets(this); + + // create views + this.charts_view = new ChartsView(this); + this.group_view = new GroupView(this); + this.chart_view = new ChartView(this); + this.viewport_view = new ViewportView(this); + + // append view port to charts viewer + this.charts_view.append(this.viewport_view.$el); + + // create portlet + if (!this.options.config.widget) { + this.portlet = new Portlet({icon : 'fa-bar-chart-o', label : 'Charts'}); + } else { + this.portlet = $('
'); + } + + // append views + this.portlet.append(this.charts_view.$el); + this.portlet.append(this.group_view.$el); + this.portlet.append(this.chart_view.$el); + + // set element + if (!this.options.config.widget) { + this.setElement(this.portlet.$el); + } else { + this.setElement(this.portlet); + } + + // hide views + this.group_view.$el.hide(); + this.charts_view.$el.hide(); + + // events + var self = this; + this.config.on('change:current_view', function() { + self._showCurrent(); + }); + }, + + // current view + _showCurrent: function() { + + }, + + // execute command + execute: function(options) { + }, + + // unload + onunload: function() { + }, + + // log + log: function(location, message) { + console.log(location + ' ' + message); + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/charts/bardiagram.js b/config/plugins/visualizations/charts/static/charts/bardiagram.js new file mode 100644 index 00000000000..06dbd7eeeea --- /dev/null +++ b/config/plugins/visualizations/charts/static/charts/bardiagram.js @@ -0,0 +1,33 @@ +// dependencies +define([], function() { + +// widget +return Backbone.View.extend( +{ + // initialize + initialize: function(app, options) { + this.app = app; + this.options = options; + }, + + // render + refresh : function(data) + { + // add graph to screen + var self = this; + nv.addGraph(function() { + self.d3_chart = nv.models.multiBarChart(); + + self.d3_chart.xAxis.tickFormat(d3.format('.2f')) + self.d3_chart.yAxis.tickFormat(d3.format('.1f')) + + d3.select(self.options.svg_id) + .datum(data) + .call(self.d3_chart); + + nv.utils.windowResize(self.d3_chart.update); + }); + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/charts/horizontal.js b/config/plugins/visualizations/charts/static/charts/horizontal.js new file mode 100644 index 00000000000..fd3698c1051 --- /dev/null +++ b/config/plugins/visualizations/charts/static/charts/horizontal.js @@ -0,0 +1,33 @@ +// dependencies +define(['library/utils'], function(Utils) { + +// widget +return Backbone.View.extend( +{ + // initialize + initialize: function(app, options) { + this.app = app; + this.options = options; + this.chart = options.chart; + }, + + // render + refresh : function(data) + { + // add graph to screen + var self = this; + nv.addGraph(function() { + self.d3_chart = nv.models.multiBarHorizontalChart(); + + self.d3_chart.xAxis.tickFormat(function() { return ''; }); + + d3.select(self.options.svg_id) + .datum(data) + .call(self.d3_chart); + + nv.utils.windowResize(self.d3_chart.update); + }); + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/charts/line.js b/config/plugins/visualizations/charts/static/charts/line.js new file mode 100644 index 00000000000..1409b21aec8 --- /dev/null +++ b/config/plugins/visualizations/charts/static/charts/line.js @@ -0,0 +1,39 @@ +// dependencies +define(['library/utils'], function(Utils) { + +// widget +return Backbone.View.extend( +{ + // initialize + initialize: function(app, options) { + this.app = app; + this.options = options; + this.chart = options.chart; + }, + + // render + refresh : function(data) + { + // add graph to screen + var self = this; + nv.addGraph(function() { + self.chart_3d = nv.models.lineChart(); + + self.chart_3d.xAxis + .tickFormat(d3.format(',f')); + + self.chart_3d.yAxis + .tickFormat(d3.format(',.2f')); + + d3.select(self.options.svg_id) + .datum(data) + .call(self.chart_3d); + + nv.utils.windowResize(self.chart_3d.update); + + return self.chart_3d; + }); + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/charts/linewithfocus.js b/config/plugins/visualizations/charts/static/charts/linewithfocus.js new file mode 100644 index 00000000000..37425565a2b --- /dev/null +++ b/config/plugins/visualizations/charts/static/charts/linewithfocus.js @@ -0,0 +1,37 @@ +// dependencies +define(['library/utils'], function(Utils) { + +// widget +return Backbone.View.extend( +{ + // initialize + initialize: function(app, options) { + this.app = app; + this.options = options; + this.chart = options.chart; + }, + + // render + refresh : function(data) + { + // add graph to screen + var self = this; + nv.addGraph(function() { + self.chart_3d = nv.models.lineWithFocusChart(); + + self.chart_3d.xAxis + .tickFormat(d3.format(',f')); + + self.chart_3d.yAxis + .tickFormat(d3.format(',.2f')); + + d3.select(self.options.svg_id) + .datum(data) + .call(self.chart_3d); + + nv.utils.windowResize(self.chart_3d.update); + }); + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/charts/piechart.js b/config/plugins/visualizations/charts/static/charts/piechart.js new file mode 100644 index 00000000000..babc004a5fb --- /dev/null +++ b/config/plugins/visualizations/charts/static/charts/piechart.js @@ -0,0 +1,51 @@ +// dependencies +define(['library/utils'], function(Utils) { + +// widget +return Backbone.View.extend( +{ + // initialize + initialize: function(app, options) { + this.app = app; + this.options = options; + this.chart = options.chart; + }, + + // render + refresh : function(data) + { + // loop through data groups + for (var key in data) { + // get group + var group = data[key]; + + // format chart data + var pie_data = []; + for (var key in group.values) { + var value = group.values[key]; + pie_data.push ({ + key : value.x, + y : value.y + }); + } + + // add graph to screen + var self = this; + nv.addGraph(function() { + self.chart_3d = nv.models.pieChart() + .donut(true) + .showLegend(false); + + d3.select(self.options.svg_id) + .datum(pie_data) + .call(self.chart_3d); + + nv.utils.windowResize(self.chart_3d.update); + + return self.chart_3d; + }); + } + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/charts/scatterplot.js b/config/plugins/visualizations/charts/static/charts/scatterplot.js new file mode 100644 index 00000000000..12f3b7988e5 --- /dev/null +++ b/config/plugins/visualizations/charts/static/charts/scatterplot.js @@ -0,0 +1,37 @@ +// dependencies +define(['library/utils'], function(Utils) { + +// widget +return Backbone.View.extend( +{ + // initialize + initialize: function(app, options) { + this.app = app; + this.options = options; + this.chart = options.chart; + }, + + // render + refresh : function(data) + { + // add graph to screen + var self = this; + nv.addGraph(function() { + self.d3_chart = nv.models.scatterChart() + .showDistX(true) + .showDistY(true) + .color(d3.scale.category10().range()); + + self.d3_chart.xAxis.tickFormat(d3.format('.02f')) + self.d3_chart.yAxis.tickFormat(d3.format('.02f')) + + d3.select(self.options.svg_id) + .datum(data) + .call(self.d3_chart); + + nv.utils.windowResize(self.d3_chart.update); + }); + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/charts/stackedarea.js b/config/plugins/visualizations/charts/static/charts/stackedarea.js new file mode 100644 index 00000000000..a6ab33b4e1e --- /dev/null +++ b/config/plugins/visualizations/charts/static/charts/stackedarea.js @@ -0,0 +1,59 @@ +// dependencies +define(['library/utils'], function(Utils) { + +// widget +return Backbone.View.extend( +{ + // initialize + initialize: function(app, options) { + this.app = app; + this.options = options; + this.chart = options.chart; + }, + + // render + refresh : function(data) + { + // add graph to screen + var self = this; + nv.addGraph(function() { + // check data + var valid = true; + var length = 0; + for (var key in data) { + // evalute length + if (length == 0) { + length = data[key].values.length; + } else { + if (length != data[key].values.length) { + valid = false; + break; + } + } + } + if (!valid) { + return; + } + + // make plot + self.d3_chart = nv.models.stackedAreaChart() + .x(function(d) { + return d.x + }) + .y(function(d) { + return d.y + }) + .clipEdge(true); + + self.d3_chart.xAxis.tickFormat(function() { return ''; }); + + d3.select(self.options.svg_id) + .datum(data) + .call(self.d3_chart); + + nv.utils.windowResize(self.d3_chart.update); + }); + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/library/portlet.js b/config/plugins/visualizations/charts/static/library/portlet.js new file mode 100644 index 00000000000..6f673b4f1e1 --- /dev/null +++ b/config/plugins/visualizations/charts/static/library/portlet.js @@ -0,0 +1,171 @@ +// dependencies +define(['library/utils'], function(Utils) { + +// return +return Backbone.View.extend( +{ + // visibility + visible: false, + + // defaults options + optionsDefault: { + label : '', + icon : 'fa-tasks', + buttons : null, + body : null, + height : null, + operations : null, + placement : 'bottom', + overflow : 'auto' + }, + + // content + $content : null, + + // initialize + initialize : function(options) { + // configure options + this.options = Utils.merge(options, this.optionsDefault); + + // create new element + this.setElement(this.template(this.options)); + + // link content + this.$content = this.$el.find('#content'); + + // set content height + if (this.options.height) { + this.$el.find('#body').css('height', this.options.height); + this.$el.find('#content').css('overflow', this.options.overflow); + } + + // append buttons + this.$buttons = $(this.el).find('#buttons'); + if (this.options.buttons) { + // link functions + var self = this; + $.each(this.options.buttons, function(name, item) { + item.$el.prop('id', name); + self.$buttons.append(item.$el); + }); + } else { + this.$buttons.remove(); + } + + // append operations + this.$operations = $(this.el).find('#operations'); + if (this.options.operations) { + // link functions + var self = this; + $.each(this.options.operations, function(name, item) { + item.$el.prop('id', name); + self.$operations.append(item.$el); + }); + } + + // add body + if(this.options.body) { + this.append(this.options.body); + } + }, + + // append + append: function($el) { + this.$content.append(Utils.wrap($el)); + }, + + // content + content: function() { + return this.$content; + }, + + // hide modal + show: function(){ + // fade in + this.$el.fadeIn('fast'); + + // set flag + this.visible = true; + }, + + // hide modal + hide: function(){ + // fade out + this.$el.fadeOut('fast'); + + // set flag + this.visible = false; + }, + + // enable buttons + enableButton: function(id) { + this.$buttons.find('#' + id).prop('disabled', false); + }, + + // disable buttons + disableButton: function(id) { + this.$buttons.find('#' + id).prop('disabled', true); + }, + + // enable operation + hideOperation: function(id) { + this.$operations.find('#' + id).hide(); + }, + + // disable operation + showOperation: function(id) { + this.$operations.find('#' + id).show(); + }, + + // set operation + setOperation: function(id, callback) { + var $el = this.$operations.find('#' + id); + $el.off('click'); + $el.on('click', callback); + }, + + // label + label: function(new_label) { + var $el = this.$el.find('#label'); + if (new_label) { + $el.html(new_label); + } + return $el.html(); + }, + + // fill regular modal template + template: function(options) { + var tmpl = '
'; + + if (options.label || options.icon) { + tmpl += '
' + + '
' + + '
'; + + if (options.icon) + tmpl += ' '; + + tmpl += '
' + options.label + '
'; + + tmpl += '
' + + '
'; + } + tmpl += '
'; + + if (options.placement == 'top') { + tmpl += '
'; + } + + tmpl += '
'; + + if (options.placement == 'bottom') { + tmpl += '
'; + } + + tmpl += '
' + + '
'; + return tmpl; + } +}); + +}); diff --git a/config/plugins/visualizations/charts/static/library/table.js b/config/plugins/visualizations/charts/static/library/table.js new file mode 100644 index 00000000000..d9cb523682a --- /dev/null +++ b/config/plugins/visualizations/charts/static/library/table.js @@ -0,0 +1,188 @@ +// dependencies +define(['library/utils'], function(Utils) { + +// return +return Backbone.View.extend( +{ + // current row + row: null, + + // count rows + row_count: 0, + + // defaults options + optionsDefault: { + content : 'No content available.', + onchange : null, + ondblclick : null, + onconfirm : null + }, + + // events + events : { + 'click' : 'onclick', + 'dblclick' : 'ondblclick' + }, + + // first + first: true, + + // initialize + initialize : function(options) { + // configure options + this.options = Utils.merge(options, this.optionsDefault); + + // create new element + this.setElement(this.template(options)); + + // initialize row + this.row = $(''); + }, + + // add header cell + addHeader: function($el) { + var wrapper = $(''); + wrapper.append($el); + this.row.append(wrapper); + }, + + // header + appendHeader: function() { + // append header row + $(this.el).find('thead').append(this.row); + + // row + this.row = $(''); + }, + + // add row cell + add: function($el) { + var wrapper = $(''); + wrapper.append($el); + this.row.append(wrapper); + }, + + // append + append: function(id) { + this.commit(id); + }, + + // prepend + prepend: function(id) { + this.commit(id, true); + }, + + // commit + commit: function(id, prepend) { + // add + this.row.attr('id', id); + + // add row + if (prepend) { + $(this.el).find('tbody').prepend(this.row); + } else { + $(this.el).find('tbody').append(this.row); + } + + // row + this.row = $(''); + + // row count + this.row_count++; + this.refresh(); + }, + + // remove + remove: function(id) { + $(this.el).find('#' + id).remove(); + this.row_count--; + this.refresh(); + }, + + // remove + removeAll: function() { + $(this.el).find('tbody').html(''); + this.row_count = 0; + this.refresh(); + }, + + // value + value: function(new_value) { + // get current id/value + this.before = this.$el.find('.current').attr('id'); + + // check if new_value is defined + if (new_value !== undefined) { + this.$el.find('tr').removeClass('current'); + if (new_value) { + this.$el.find('#' + new_value).addClass('current'); + } + } + + // get current id/value + var after = this.$el.find('.current').attr('id'); + if(after === undefined) { + return null; + } else { + // fire onchange + if (after != this.before && this.options.onchange) { + this.options.onchange(new_value); + } + + // return current value + return after; + } + }, + + // confirm new value + confirm: function(new_value) { + this.value(new_value); + }, + + // onclick + onclick: function(e) { + // get values + var old_value = this.value(); + var new_value = $(e.target).closest('tr').attr('id'); + + // check equality + if (new_value && old_value != new_value) { + if (this.options.onconfirm) { + this.options.onconfirm(new_value); + } else { + this.confirm(new_value); + } + } + }, + + // onclick + ondblclick: function(e) { + var value = this.value(); + if (value && this.options.ondblclick) { + this.options.ondblclick(value); + } + }, + + // refresh + refresh: function() { + if (this.row_count == 0) { + this.$el.find('tmessage').show(); + } else { + this.$el.find('tmessage').hide(); + } + }, + + // load html template + template: function(options) + { + return '
' + + '' + + '' + + '' + + '
' + + '' + options.content + '' + + '
'; + } +}); + +}); diff --git a/config/plugins/visualizations/charts/static/library/ui.js b/config/plugins/visualizations/charts/static/library/ui.js new file mode 100644 index 00000000000..c751fbfcce4 --- /dev/null +++ b/config/plugins/visualizations/charts/static/library/ui.js @@ -0,0 +1,472 @@ +// dependencies +define(['library/utils', 'library/ui.select'], function(Utils, Select) { + +// plugin +var Label = Backbone.View.extend( +{ + // options + optionsDefault: { + label : '' + }, + + // initialize + initialize : function(options) { + // get options + this.options = Utils.merge(options, this.optionsDefault); + + // create new element + this.setElement(this.template(this.options)); + }, + + // template + template: function(options) { + return ''; + }, + + // value + value: function() { + return options.label; + } +}); + +// plugin +var Button = Backbone.View.extend( +{ + // options + optionsDefault: { + id : null, + label : '', + float : 'right', + cls : 'btn-default', + type : 'submit', + icon : '' + }, + + // initialize + initialize : function(options) { + // get options + this.options = Utils.merge(options, this.optionsDefault); + + // create new element + this.setElement(this._template(this.options)); + + // add event + $(this.el).on('click', options.onclick); + }, + + // element + _template: function(options) { + var str = ''; + return str; + } +}); + +// plugin +var Anchor = Backbone.View.extend( +{ + // options + optionsDefault: { + label : '' + }, + + // initialize + initialize : function(options) { + // get options + this.options = Utils.merge(options, this.optionsDefault); + + // create new element + this.setElement(this._template(this.options)); + + // add event + $(this.el).on('click', options.onclick); + }, + + // element + _template: function(options) { + return '
' + options.label + '
'; + } +}); + +// plugin +var Message = Backbone.View.extend( +{ + // options + optionsDefault: { + message : '', + status : 'info', + persistent : false + }, + + // initialize + initialize : function(options) { + // configure options + this.options = Utils.merge(options, this.optionsDefault); + + // create new element + this.setElement('
'); + }, + + // update + update : function(options) { + // get options + this.options = Utils.merge(options, this.optionsDefault); + + // show message + if (options.message != '') { + this.$el.html(this.template(this.options)); + this.$el.fadeIn(); + + // check if message is persistent + if (!options.persistent) { + // set timer + var self = this; + window.setTimeout(function() { + if (self.$el.is(':visible')) { + self.$el.fadeOut(); + } else { + self.$el.hide(); + } + }, 3000); + } + } else { + this.$el.fadeOut(); + } + }, + + // element + template: function(options) { + return '
' + options.message + '
'; + } +}); + +// plugin +var Searchbox = Backbone.View.extend( +{ + // options + optionsDefault: { + onclick : null, + searchword : '' + }, + + // initialize + initialize : function(options) { + // configure options + this.options = Utils.merge(options, this.optionsDefault); + + // create new element + this.setElement(this.template(this.options)); + + // add click event + var self = this; + if (this.options.onclick) { + this.$el.on('submit', function(e) { + var search_field = self.$el.find('#search'); + self.options.onclick(search_field.val()); + }); + } + }, + + // element + template: function(options) { + return ''; + } +}); + +// plugin +var Title = Backbone.View.extend( +{ + // options + optionsDefault: { + label : 'Unlabeled', + body : null + }, + + // initialize + initialize : function(options) { + // configure options + this.options = Utils.merge(options, this.optionsDefault); + + // create new element + this.setElement(this.template(this.options)); + + if (this.options.body) { + this.$el.find('.body').append(this.options.body); + } + }, + + // element + template: function(options) { + return '
' + + options.label + ':' + + '
'; + } +}); + +// tab +var ButtonMenu = Backbone.View.extend( +{ + // main options + optionsDefault: + { + id : '', + title : '', + target : '', + href : '', + onunload : null, + onclick : null, + visible : true, + icon : null, + tag : '' + }, + + // optional sub menu + $menu: null, + + // initialize + initialize: function (options) + { + // get options + this.options = Utils.merge(options, this.optionsDefault); + + // add template for tab + this.setElement($(this._template(this.options))); + + // find root + var $root = $(this.el).find('.root'); + + // link head + var self = this; + $root.on('click', function(e) + { + // prevent default + e.preventDefault(); + + // add click event + if(self.options.onclick) { + self.options.onclick(); + } + }); + + // visiblity + if (!this.options.visible) + this.hide(); + }, + + // show + show: function() + { + $(this.el).show(); + }, + + // hide + hide: function() + { + $(this.el).hide(); + }, + + // add menu item + addMenu: function (options) + { + // menu option defaults + var menuOptions = { + title : '', + target : '', + href : '', + onclick : null, + divider : false, + icon : null + } + + // get options + menuOptions = Utils.merge(options, menuOptions); + + // check if submenu element is available + if (!this.$menu) + { + // insert submenu element into root + $(this.el).append(this._templateMenu()); + + // update element link + this.$menu = $(this.el).find('.menu'); + } + + // create + var $item = $(this._templateMenuItem(menuOptions)); + + // add events + $item.on('click', function(e) + { + // prevent default + e.preventDefault(); + + // add click event + if(menuOptions.onclick) { + menuOptions.onclick(); + } + }); + + // append menu + this.$menu.append($item); + + // append divider + if (menuOptions.divider) + this.$menu.append($(this._templateDivider())); + }, + + // fill template header + _templateMenuItem: function (options) + { + var tmpl = '
  • ' + + ''; + + if (options.icon) + tmpl += ''; + + tmpl += ' ' + options.title + + '' + + '
  • '; + return tmpl; + }, + + // fill template header + _templateMenu: function () + { + return ''; + }, + + _templateDivider: function() + { + return '
  • '; + }, + + // fill template + _template: function (options) + { + // start template + var tmpl = '
    ' + + '' + + '
    '; + + // return template + return tmpl; + } +}); + +// plugin +var ButtonIcon = Backbone.View.extend( +{ + // options + optionsDefault: { + id : null, + label : '', + float : 'right', + cls : 'icon-btn', + type : 'submit', + icon : '', + tooltip : '' + }, + + // initialize + initialize : function(options) { + // get options + this.options = Utils.merge(options, this.optionsDefault); + + // create new element + this.setElement(this.template(this.options)); + + // add event + $(this.el).on('click', options.onclick); + + // add tooltip + $(this.el).tooltip({title: options.tooltip, placement: 'bottom'}); + }, + + // element + template: function(options) { + var str = ''; + if (options.icon) { + str += ''; + } + str += options.label + + ''; + return str; + } +}); + + +// plugin +var Input = Backbone.View.extend( +{ + // options + optionsDefault: { + value : '', + type : 'text', + placeholder : '', + disabled : false, + visible : true + }, + + // initialize + initialize : function(options) { + // configure options + this.options = Utils.merge(options, this.optionsDefault); + + // create new element + this.setElement(this.template(this.options)); + + // disable input field + if (this.options.disabled) { + this.$el.prop('disabled', true); + } + + // hide input field + if (!this.options.visible) { + this.$el.hide(); + } + }, + + // value + value : function (new_val) { + if (new_val !== undefined) { + this.$el.val(new_val); + } + return this.$el.val(); + }, + + // element + template: function(options) { + return ''; + } +}); + +// return +return { + Label : Label, + Button : Button, + ButtonIcon : ButtonIcon, + Input : Input, + Anchor : Anchor, + Message : Message, + Searchbox : Searchbox, + Title : Title, + Select : Select, + ButtonMenu : ButtonMenu +} +}); diff --git a/config/plugins/visualizations/charts/static/library/ui.select.js b/config/plugins/visualizations/charts/static/library/ui.select.js new file mode 100644 index 00000000000..6fae2c5a196 --- /dev/null +++ b/config/plugins/visualizations/charts/static/library/ui.select.js @@ -0,0 +1,165 @@ +// dependencies +define(['library/utils'], function(Utils) { + +// plugin +return Backbone.View.extend( +{ + // options + optionsDefault: { + id : '', + cls : '', + empty : 'No data available' + }, + + // initialize + initialize : function(options) { + // configure options + this.options = Utils.merge(options, this.optionsDefault); + + // create new element + this.setElement(this._template(this.options)); + + // add change event + var self = this; + if (this.options.onchange) { + this.$el.on('change', function() { self.options.onchange(self.value()); }); + } + + // refresh + this._refresh(); + }, + + // value + value : function (new_value) { + // get current id/value + var before = this.$el.val(); + + // check if new_value is defined + if (new_value !== undefined) { + this.$el.val(new_value); + } + + // get current id/value + var after = this.$el.val(); + if(after === undefined) { + return null; + } else { + // fire onchange + if ((after != before && this.options.onchange) || this.$el.find('option').length == 1) { + this.options.onchange(after); + } + + // return current value + return after; + } + }, + + // label + label : function () { + return this.$el.find('option:selected').text(); + }, + + // disabled + disabled: function() { + return this.$el.is(':disabled'); + }, + + // enable + enable: function() { + this.$el.prop('disabled', false); + }, + + // disable + disable: function() { + this.$el.prop('disabled', true); + }, + + // add + add: function(options) { + // add options + $(this.el).append(this._templateOption(options)); + + // refresh + this._refresh(); + }, + + // remove + remove: function(value) { + // remove option + $(this.el).find('option[value=' + value + ']').remove(); + $(this.el).trigger('change'); + + // refresh + this._refresh(); + }, + + // render + update: function(options) { + // selected + var selected = this.$el.val(); + + // remove all options + $(this.el).find('option').remove(); + + // add new options + for (var key in options.data) { + $(this.el).append(this._templateOption(options.data[key])); + } + + // add selected value + if (this._exists(selected)) + $(this.el).val(selected); + + // refresh + this._refresh(); + }, + + // refresh + _refresh: function() { + // remove placeholder + $(this.el).find('option[value=null]').remove(); + + // count remaining entries + var remaining = $(this.el).find('option').length; + if (remaining == 0) { + // append placeholder + $(this.el).append(this._templateOption({value : 'null', label : this.options.empty})); + this.disable(); + } else { + this.enable(); + } + }, + + // exists + _exists: function(value) { + // check if selected value exists + return 0 != $(this.el).find('option[value=' + value + ']').length; + }, + + // option + _templateOption: function(options) { + return ''; + }, + + // element + _template: function(options) { + var tmpl = ''; + return tmpl; + } +}); + +}); diff --git a/config/plugins/visualizations/charts/static/library/utils.js b/config/plugins/visualizations/charts/static/library/utils.js new file mode 100644 index 00000000000..a5017dddbf9 --- /dev/null +++ b/config/plugins/visualizations/charts/static/library/utils.js @@ -0,0 +1,130 @@ +// dependencies +define([], function() { + +// get +function get (url, success, error) { + request('GET', url, {}, success, error); +}; + +// generic function to send json to url +function request (method, url, data, success, error) { + + // encode data into url + if (method == 'GET' || method == 'DELETE') { + url = url + "?" + $.param(data); + } + + // prepare request + var xhr = new XMLHttpRequest(); + xhr.open(method, url, true); + xhr.setRequestHeader('Accept', 'application/json'); + xhr.setRequestHeader('Cache-Control', 'no-cache'); + xhr.setRequestHeader('X-Requested-With', 'XMLHttpRequest'); + xhr.setRequestHeader('Content-Type', 'application/json'); + xhr.onloadend = function() { + var status = xhr.status; + if (status == 200) { + try { + response = jQuery.parseJSON(xhr.responseText); + } catch (e) { + response = xhr.responseText; + } + success && success(response); + } else { + error && error(status); + } + }; + + // make request + if (method == 'GET' || method == 'DELETE') { + xhr.send(); + } else { + xhr.send(JSON.stringify(data)); + } +}; + +// merge +function merge (options, optionsDefault) { + if (options) + return _.defaults(options, optionsDefault); + else + return optionsDefault; +}; + +// get css value +function cssGetAttribute (classnames, name) { + if (classnames.length == 0) { + console.log('cssGetAttribute() : Requires a list of classnames'); + } + + // place dummy element + var el = $('
    '); + var traverse_el = el; + for (var i in classnames) { + traverse_el.append('
    '); + traverse_el = traverse_el.find('.' + classnames[i]); + } + + // required append + el.appendTo(':eq(0)'); + + // get value + var value = el.css(name); + + // remove element + el.remove(); + + // return css value + return value; +}; + +// load css +function cssLoadFile (url) { + // check if css is already available + if (!$('link[href^="' + url + '"]').length) + $('').appendTo('head'); +}; + +// unique ide +function uuid(){ + return (new Date().getTime()).toString(36); +}; + +// wrap +function wrap($el) { + var wrapper = $('

    '); + wrapper.append($el); + return wrapper; +}; + +// time +function time() { + // get date object + var d = new Date(); + + // format items + var hours = (d.getHours() < 10 ? "0" : "") + d.getHours(); + var minutes = (d.getMinutes() < 10 ? "0" : "") + d.getMinutes() + + // format final stamp + var datetime = d.getDate() + "/" + + (d.getMonth() + 1) + "/" + + d.getFullYear() + ", " + + hours + ":" + + minutes; + return datetime; +}; + +// return +return { + cssLoadFile : cssLoadFile, + cssGetAttribute : cssGetAttribute, + get : get, + merge : merge, + request: request, + uuid: uuid, + wrap : wrap, + time : time +}; + +}); diff --git a/config/plugins/visualizations/charts/static/models/chart.js b/config/plugins/visualizations/charts/static/models/chart.js new file mode 100644 index 00000000000..67e5c2fc4d5 --- /dev/null +++ b/config/plugins/visualizations/charts/static/models/chart.js @@ -0,0 +1,43 @@ +// dependencies +define(['models/groups'], function(Groups) { + + +// model +return Backbone.Model.extend( +{ + // defaults + defaults : { + id : null, + title : '', + type : '', + date : null + }, + + // initialize + initialize: function(options) + { + this.groups = new Groups(); + }, + + // reset + reset: function() + { + this.clear({silent: true}).set(this.defaults); + this.groups.reset(); + this.trigger('reset', this); + }, + + // copy + copy: function(new_chart) { + // copy chart/groups + var current = this; + current.set(new_chart.attributes); + current.groups.reset(); + new_chart.groups.each(function(group) { + current.groups.add(group.clone()); + }); + current.trigger('change', current); + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/models/charts.js b/config/plugins/visualizations/charts/static/models/charts.js new file mode 100644 index 00000000000..99188856aa5 --- /dev/null +++ b/config/plugins/visualizations/charts/static/models/charts.js @@ -0,0 +1,10 @@ +// dependencies +define(['models/chart'], function(Chart) { + +// collection +return Backbone.Collection.extend( +{ + model: Chart +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/models/config.js b/config/plugins/visualizations/charts/static/models/config.js new file mode 100644 index 00000000000..2ac90f096ac --- /dev/null +++ b/config/plugins/visualizations/charts/static/models/config.js @@ -0,0 +1,16 @@ +// dependencies +define([], function() { + + +// model +return Backbone.Model.extend( +{ + // options + defaults : { + query_limit : 20, + query_pace : 1000, + query_max : 5 + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/models/datasets.js b/config/plugins/visualizations/charts/static/models/datasets.js new file mode 100644 index 00000000000..1d3e16de9fd --- /dev/null +++ b/config/plugins/visualizations/charts/static/models/datasets.js @@ -0,0 +1,125 @@ +// dependencies +define(['library/utils'], function(Utils) { + +// widget +return Backbone.Model.extend( +{ + // options + optionsDefault : { + limit : 500, + pace : 1000, + max : 2 + }, + + // list + list: {}, + + // initialize + initialize: function(app, options) + { + // link app + this.app = app; + + // configure options + this.options = Utils.merge(options, this.optionsDefault); + }, + + // get + get: function(options, callback) { + // check if dataset already exists + var dataset = this.list[options.id]; + if (dataset) { + if (dataset.values) { + if (dataset.values[options.column]) { + if (callback) { + callback(dataset); + } + return dataset; + } + } + } else { + this.list[options.id] = dataset = this.app.options.dataset; + } + + // initiate fetch cycle + if (options.column !== undefined) { + this._fetch(options, callback); + } else { + if (callback) { + callback(dataset); + } + return dataset; + } + + // return + return dataset; + }, + + // size + length: function() { + return this.list.length; + }, + + // request data + _fetch : function(options, callback) { + // get dataset + var dataset = this.list[options.id]; + + // initialize values for datasets + if (!dataset.values) { + dataset.values = {}; + } + if (!dataset.values[options.column]) { + dataset.values[options.column] = []; + } + + // initialize page number + if (options.page === undefined) { + options.page = 0; + } else { + options.page++; + } + + // check max page + if (options.page > this.options.max) { + return; + } + + // config limits + var limit = this.options.limit; + var offset = this.options.limit * options.page; + + // make request + var self = this; + Utils.request('GET', config.root + 'api/datasets/' + options.id, { + data_type : 'raw_data', + provider : 'dataset-column', + limit : limit, + offset : offset, + indeces : options.column + }, function(response) { + // verify if all data has been fetched + if (response.data.length > 0) { + // add values to dataset + for (var key in response.data){ + var value = response.data[key][0]; + if (value !== undefined) { + dataset.values[options.column].push(value); + } else { + return dataset; + } + } + + // update charts + if (callback) { + callback(dataset); + } + + // next call + setTimeout(function() { self._fetch(options, callback); }, self.options.pace); + } + }); + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/models/group.js b/config/plugins/visualizations/charts/static/models/group.js new file mode 100644 index 00000000000..445e59cae79 --- /dev/null +++ b/config/plugins/visualizations/charts/static/models/group.js @@ -0,0 +1,15 @@ +// dependencies +define([], function() { + +// model +return Backbone.Model.extend( +{ + // reset + reset: function() + { + this.clear({silent: true}).set(this.defaults); + this.trigger('reset', this); + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/models/groups.js b/config/plugins/visualizations/charts/static/models/groups.js new file mode 100644 index 00000000000..156ada47be4 --- /dev/null +++ b/config/plugins/visualizations/charts/static/models/groups.js @@ -0,0 +1,10 @@ +// dependencies +define(['models/group'], function(Group) { + +// collection +return Backbone.Collection.extend( +{ + model: Group +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/models/types.js b/config/plugins/visualizations/charts/static/models/types.js new file mode 100644 index 00000000000..8fcd42bcdb8 --- /dev/null +++ b/config/plugins/visualizations/charts/static/models/types.js @@ -0,0 +1,71 @@ +// dependencies +define([], function() { + +// widget +return Backbone.Model.extend( +{ + // types + defaults: { + 'bardiagram' : { + title : 'Bar diagram', + data : { + y : { + title : 'Values for y-axis' + } + } + }, + 'horizontal' : { + title : 'Bar diagram (horizontal)', + data : { + y : { + title : 'Values for y-axis' + } + } + }, + 'line' : { + title : 'Line chart', + data : { + y : { + title : 'Values for y-axis' + } + } + }, + 'linewithfocus' : { + title : 'Line chart (with focus)', + data : { + y : { + title : 'Values for y-axis' + } + } + }, + 'piechart' : { + title : 'Pie chart', + data : { + y : { + title : 'Values for y-axis' + } + } + }, + 'scatterplot' : { + title : 'Scatter plot', + data : { + x : { + title : 'Values for x-axis' + }, + y : { + title : 'Values for y-axis' + } + } + }, + 'stackedarea' : { + title : 'Stacked area', + data : { + y : { + title : 'Values for y-axis' + } + } + } + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/plugins/nv.d3.css b/config/plugins/visualizations/charts/static/plugins/nv.d3.css new file mode 100644 index 00000000000..d46f7ebce24 --- /dev/null +++ b/config/plugins/visualizations/charts/static/plugins/nv.d3.css @@ -0,0 +1 @@ +.chartWrap{margin:0;padding:0;overflow:hidden}.nvtooltip.with-3d-shadow,.with-3d-shadow .nvtooltip{-moz-box-shadow:0 5px 10px rgba(0,0,0,.2);-webkit-box-shadow:0 5px 10px rgba(0,0,0,.2);box-shadow:0 5px 10px rgba(0,0,0,.2);-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.nvtooltip{position:absolute;background-color:rgba(255,255,255,1);padding:1px;border:1px solid rgba(0,0,0,.2);z-index:10000;font-family:Arial;font-size:13px;text-align:left;pointer-events:none;white-space:nowrap;-webkit-touch-callout:none;-webkit-user-select:none;-khtml-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.nvtooltip.with-transitions,.with-transitions .nvtooltip{transition:opacity 250ms linear;-moz-transition:opacity 250ms linear;-webkit-transition:opacity 250ms linear;transition-delay:250ms;-moz-transition-delay:250ms;-webkit-transition-delay:250ms}.nvtooltip.x-nvtooltip,.nvtooltip.y-nvtooltip{padding:8px}.nvtooltip h3{margin:0;padding:4px 14px;line-height:18px;font-weight:400;background-color:rgba(247,247,247,.75);text-align:center;border-bottom:1px solid #ebebeb;-webkit-border-radius:5px 5px 0 0;-moz-border-radius:5px 5px 0 0;border-radius:5px 5px 0 0}.nvtooltip p{margin:0;padding:5px 14px;text-align:center}.nvtooltip span{display:inline-block;margin:2px 0}.nvtooltip table{margin:6px;border-spacing:0}.nvtooltip table td{padding:2px 9px 2px 0;vertical-align:middle}.nvtooltip table td.key{font-weight:400}.nvtooltip table td.value{text-align:right;font-weight:700}.nvtooltip table tr.highlight td{padding:1px 9px 1px 0;border-bottom-style:solid;border-bottom-width:1px;border-top-style:solid;border-top-width:1px}.nvtooltip table td.legend-color-guide div{width:8px;height:8px;vertical-align:middle}.nvtooltip .footer{padding:3px;text-align:center}.nvtooltip-pending-removal{position:absolute;pointer-events:none}svg{-webkit-touch-callout:none;-webkit-user-select:none;-khtml-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;display:block;width:100%;height:100%}svg text{font:400 12px Arial}svg .title{font:700 14px Arial}.nvd3 .nv-background{fill:#fff;fill-opacity:0}.nvd3.nv-noData{font-size:18px;font-weight:700}.nv-brush .extent{fill-opacity:.125;shape-rendering:crispEdges}.nvd3 .nv-legend .nv-series{cursor:pointer}.nvd3 .nv-legend .disabled circle{fill-opacity:0}.nvd3 .nv-axis{pointer-events:none}.nvd3 .nv-axis path{fill:none;stroke:#000;stroke-opacity:.75;shape-rendering:crispEdges}.nvd3 .nv-axis path.domain{stroke-opacity:.75}.nvd3 .nv-axis.nv-x path.domain{stroke-opacity:0}.nvd3 .nv-axis line{fill:none;stroke:#e5e5e5;shape-rendering:crispEdges}.nvd3 .nv-axis .zero line,.nvd3 .nv-axis line.zero{stroke-opacity:.75}.nvd3 .nv-axis .nv-axisMaxMin text{font-weight:700}.nvd3 .x .nv-axis .nv-axisMaxMin text,.nvd3 .x2 .nv-axis .nv-axisMaxMin text,.nvd3 .x3 .nv-axis .nv-axisMaxMin text{text-anchor:middle}.nv-brush .resize path{fill:#eee;stroke:#666}.nvd3 .nv-bars .negative rect{zfill:brown}.nvd3 .nv-bars rect{zfill:#4682b4;fill-opacity:.75;transition:fill-opacity 250ms linear;-moz-transition:fill-opacity 250ms linear;-webkit-transition:fill-opacity 250ms linear}.nvd3 .nv-bars rect.hover{fill-opacity:1}.nvd3 .nv-bars .hover rect{fill:#add8e6}.nvd3 .nv-bars text{fill:rgba(0,0,0,0)}.nvd3 .nv-bars .hover text{fill:rgba(0,0,0,1)}.nvd3 .nv-multibar .nv-groups rect,.nvd3 .nv-multibarHorizontal .nv-groups rect,.nvd3 .nv-discretebar .nv-groups rect{stroke-opacity:0;transition:fill-opacity 250ms linear;-moz-transition:fill-opacity 250ms linear;-webkit-transition:fill-opacity 250ms linear}.nvd3 .nv-multibar .nv-groups rect:hover,.nvd3 .nv-multibarHorizontal .nv-groups rect:hover,.nvd3 .nv-discretebar .nv-groups rect:hover{fill-opacity:1}.nvd3 .nv-discretebar .nv-groups text,.nvd3 .nv-multibarHorizontal .nv-groups text{font-weight:700;fill:rgba(0,0,0,1);stroke:rgba(0,0,0,0)}.nvd3.nv-pie path{stroke-opacity:0;transition:fill-opacity 250ms linear,stroke-width 250ms linear,stroke-opacity 250ms linear;-moz-transition:fill-opacity 250ms linear,stroke-width 250ms linear,stroke-opacity 250ms linear;-webkit-transition:fill-opacity 250ms linear,stroke-width 250ms linear,stroke-opacity 250ms linear}.nvd3.nv-pie .nv-slice text{stroke:#000;stroke-width:0}.nvd3.nv-pie path{stroke:#fff;stroke-width:1px;stroke-opacity:1}.nvd3.nv-pie .hover path{fill-opacity:.7}.nvd3.nv-pie .nv-label{pointer-events:none}.nvd3.nv-pie .nv-label rect{fill-opacity:0;stroke-opacity:0}.nvd3 .nv-groups path.nv-line{fill:none;stroke-width:1.5px}.nvd3 .nv-groups path.nv-line.nv-thin-line{stroke-width:1px}.nvd3 .nv-groups path.nv-area{stroke:none}.nvd3 .nv-line.hover path{stroke-width:6px}.nvd3.nv-line .nvd3.nv-scatter .nv-groups .nv-point{fill-opacity:0;stroke-opacity:0}.nvd3.nv-scatter.nv-single-point .nv-groups .nv-point{fill-opacity:.5!important;stroke-opacity:.5!important}.with-transitions .nvd3 .nv-groups .nv-point{transition:stroke-width 250ms linear,stroke-opacity 250ms linear;-moz-transition:stroke-width 250ms linear,stroke-opacity 250ms linear;-webkit-transition:stroke-width 250ms linear,stroke-opacity 250ms linear}.nvd3.nv-scatter .nv-groups .nv-point.hover,.nvd3 .nv-groups .nv-point.hover{stroke-width:7px;fill-opacity:.95!important;stroke-opacity:.95!important}.nvd3 .nv-point-paths path{stroke:#aaa;stroke-opacity:0;fill:#eee;fill-opacity:0}.nvd3 .nv-indexLine{cursor:ew-resize}.nvd3 .nv-distribution{pointer-events:none}.nvd3 .nv-groups .nv-point.hover{stroke-width:20px;stroke-opacity:.5}.nvd3 .nv-scatter .nv-point.hover{fill-opacity:1}.nvd3.nv-stackedarea path.nv-area{fill-opacity:.7;stroke-opacity:0;transition:fill-opacity 250ms linear,stroke-opacity 250ms linear;-moz-transition:fill-opacity 250ms linear,stroke-opacity 250ms linear;-webkit-transition:fill-opacity 250ms linear,stroke-opacity 250ms linear}.nvd3.nv-stackedarea path.nv-area.hover{fill-opacity:.9}.nvd3.nv-stackedarea .nv-groups .nv-point{stroke-opacity:0;fill-opacity:0}.nvd3.nv-linePlusBar .nv-bar rect{fill-opacity:.75}.nvd3.nv-linePlusBar .nv-bar rect:hover{fill-opacity:1}.nvd3.nv-bullet{font:10px sans-serif}.nvd3.nv-bullet .nv-measure{fill-opacity:.8}.nvd3.nv-bullet .nv-measure:hover{fill-opacity:1}.nvd3.nv-bullet .nv-marker{stroke:#000;stroke-width:2px}.nvd3.nv-bullet .nv-markerTriangle{stroke:#000;fill:#fff;stroke-width:1.5px}.nvd3.nv-bullet .nv-tick line{stroke:#666;stroke-width:.5px}.nvd3.nv-bullet .nv-range.nv-s0{fill:#eee}.nvd3.nv-bullet .nv-range.nv-s1{fill:#ddd}.nvd3.nv-bullet .nv-range.nv-s2{fill:#ccc}.nvd3.nv-bullet .nv-title{font-size:14px;font-weight:700}.nvd3.nv-bullet .nv-subtitle{fill:#999}.nvd3.nv-bullet .nv-range{fill:#bababa;fill-opacity:.4}.nvd3.nv-bullet .nv-range:hover{fill-opacity:.7}.nvd3.nv-sparkline path{fill:none}.nvd3.nv-sparklineplus g.nv-hoverValue{pointer-events:none}.nvd3.nv-sparklineplus .nv-hoverValue line{stroke:#333;stroke-width:1.5px}.nvd3.nv-sparklineplus,.nvd3.nv-sparklineplus g{pointer-events:all}.nvd3 .nv-hoverArea{fill-opacity:0;stroke-opacity:0}.nvd3.nv-sparklineplus .nv-xValue,.nvd3.nv-sparklineplus .nv-yValue{stroke-width:0;font-size:.9em;font-weight:400}.nvd3.nv-sparklineplus .nv-yValue{stroke:#f66}.nvd3.nv-sparklineplus .nv-maxValue{stroke:#2ca02c;fill:#2ca02c}.nvd3.nv-sparklineplus .nv-minValue{stroke:#d62728;fill:#d62728}.nvd3.nv-sparklineplus .nv-currentValue{font-weight:700;font-size:1.1em}.nvd3.nv-ohlcBar .nv-ticks .nv-tick{stroke-width:2px}.nvd3.nv-ohlcBar .nv-ticks .nv-tick.hover{stroke-width:4px}.nvd3.nv-ohlcBar .nv-ticks .nv-tick.positive{stroke:#2ca02c}.nvd3.nv-ohlcBar .nv-ticks .nv-tick.negative{stroke:#d62728}.nvd3.nv-historicalStockChart .nv-axis .nv-axislabel{font-weight:700}.nvd3.nv-historicalStockChart .nv-dragTarget{fill-opacity:0;stroke:none;cursor:move}.nvd3 .nv-brush .extent{fill-opacity:0!important}.nvd3 .nv-brushBackground rect{stroke:#000;stroke-width:.4;fill:#fff;fill-opacity:.7}.nvd3.nv-indentedtree .name{margin-left:5px}.nvd3.nv-indentedtree .clickable{color:#08C;cursor:pointer}.nvd3.nv-indentedtree span.clickable:hover{color:#005580;text-decoration:underline}.nvd3.nv-indentedtree .nv-childrenCount{display:inline-block;margin-left:5px}.nvd3.nv-indentedtree .nv-treeicon{cursor:pointer}.nvd3.nv-indentedtree .nv-treeicon.nv-folded{cursor:pointer}.nvd3 .background path{fill:none;stroke:#ccc;stroke-opacity:.4;shape-rendering:crispEdges}.nvd3 .foreground path{fill:none;stroke:#4682b4;stroke-opacity:.7}.nvd3 .brush .extent{fill-opacity:.3;stroke:#fff;shape-rendering:crispEdges}.nvd3 .axis line,.axis path{fill:none;stroke:#000;shape-rendering:crispEdges}.nvd3 .axis text{text-shadow:0 1px 0 #fff}.nvd3 .nv-interactiveGuideLine{pointer-events:none}.nvd3 line.nv-guideline{stroke:#ccc} \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/plugins/nv.d3.js b/config/plugins/visualizations/charts/static/plugins/nv.d3.js new file mode 100644 index 00000000000..2ab1accaf13 --- /dev/null +++ b/config/plugins/visualizations/charts/static/plugins/nv.d3.js @@ -0,0 +1,6 @@ +(function(){function t(e,t){return(new Date(t,e+1,0)).getDate()}function n(e,t,n){return function(r,i,s){var o=e(r),u=[];o1)while(op||r>d||d3.event.relatedTarget&&d3.event.relatedTarget.ownerSVGElement===undefined||a){if(l&&d3.event.relatedTarget&&d3.event.relatedTarget.ownerSVGElement===undefined&&d3.event.relatedTarget.className.match(t.nvPointerEventsClass))return;u.elementMouseout({mouseX:n,mouseY:r}),c.renderGuideLine(null);return}var f=s.invert(n);u.elementMousemove({mouseX:n,mouseY:r,pointXValue:f}),d3.event.type==="dblclick"&&u.elementDblclick({mouseX:n,mouseY:r,pointXValue:f})}var h=d3.select(this),p=n||960,d=r||400,v=h.selectAll("g.nv-wrap.nv-interactiveLineLayer").data([o]),m=v.enter().append("g").attr("class"," nv-wrap nv-interactiveLineLayer");m.append("g").attr("class","nv-interactiveGuideLine");if(!f)return;f.on("mousemove",g,!0).on("mouseout",g,!0).on("dblclick",g),c.renderGuideLine=function(t){if(!a)return;var n=v.select(".nv-interactiveGuideLine").selectAll("line").data(t!=null?[e.utils.NaNtoZero(t)]:[],String);n.enter().append("line").attr("class","nv-guideline").attr("x1",function(e){return e}).attr("x2",function(e){return e}).attr("y1",d).attr("y2",0),n.exit().remove()}})}var t=e.models.tooltip(),n=null,r=null,i={left:0,top:0},s=d3.scale.linear(),o=d3.scale.linear(),u=d3.dispatch("elementMousemove","elementMouseout","elementDblclick"),a=!0,f=null,l=navigator.userAgent.indexOf("MSIE")!==-1;return c.dispatch=u,c.tooltip=t,c.margin=function(e){return arguments.length?(i.top=typeof e.top!="undefined"?e.top:i.top,i.left=typeof e.left!="undefined"?e.left:i.left,c):i},c.width=function(e){return arguments.length?(n=e,c):n},c.height=function(e){return arguments.length?(r=e,c):r},c.xScale=function(e){return arguments.length?(s=e,c):s},c.showGuideLine=function(e){return arguments.length?(a=e,c):a},c.svgContainer=function(e){return arguments.length?(f=e,c):f},c},e.interactiveBisect=function(e,t,n){"use strict";if(!e instanceof Array)return null;typeof n!="function"&&(n=function(e,t){return e.x});var r=d3.bisector(n).left,i=d3.max([0,r(e,t)-1]),s=n(e[i],i);typeof s=="undefined"&&(s=i);if(s===t)return i;var o=d3.min([i+1,e.length-1]),u=n(e[o],o);return typeof u=="undefined"&&(u=o),Math.abs(u-t)>=Math.abs(s-t)?i:o},e.nearestValueIndex=function(e,t,n){"use strict";var r=Infinity,i=null;return e.forEach(function(e,s){var o=Math.abs(t-e);o<=r&&oT.height?0:x}v.top=Math.abs(x-S.top),v.left=Math.abs(E.left-S.left)}t+=a.offsetLeft+v.left-2*a.scrollLeft,u+=a.offsetTop+v.top-2*a.scrollTop}return s&&s>0&&(u=Math.floor(u/s)*s),e.tooltip.calcTooltipPosition([t,u],r,i,h),w}var t=null,n=null,r="w",i=50,s=25,o=null,u=null,a=null,f=null,l={left:null,top:null},c=!0,h="nvtooltip-"+Math.floor(Math.random()*1e5),p="nv-pointer-events-none",d=function(e,t){return e},v=function(e){return e},m=function(e){if(t!=null)return t;if(e==null)return"";var n=d3.select(document.createElement("table")),r=n.selectAll("thead").data([e]).enter().append("thead");r.append("tr").append("td").attr("colspan",3).append("strong").classed("x-value",!0).html(v(e.value));var i=n.selectAll("tbody").data([e]).enter().append("tbody"),s=i.selectAll("tr").data(function(e){return e.series}).enter().append("tr").classed("highlight",function(e){return e.highlight});s.append("td").classed("legend-color-guide",!0).append("div").style("background-color",function(e){return e.color}),s.append("td").classed("key",!0).html(function(e){return e.key}),s.append("td").classed("value",!0).html(function(e,t){return d(e.value,t)}),s.selectAll("td").each(function(e){if(e.highlight){var t=d3.scale.linear().domain([0,1]).range(["#fff",e.color]),n=.6;d3.select(this).style("border-bottom-color",t(n)).style("border-top-color",t(n))}});var o=n.node().outerHTML;return e.footer!==undefined&&(o+=""),o},g=function(e){return e&&e.series&&e.series.length>0?!0:!1};return w.nvPointerEventsClass=p,w.content=function(e){return arguments.length?(t=e,w):t},w.tooltipElem=function(){return f},w.contentGenerator=function(e){return arguments.length?(typeof e=="function"&&(m=e),w):m},w.data=function(e){return arguments.length?(n=e,w):n},w.gravity=function(e){return arguments.length?(r=e,w):r},w.distance=function(e){return arguments.length?(i=e,w):i},w.snapDistance=function(e){return arguments.length?(s=e,w):s},w.classes=function(e){return arguments.length?(u=e,w):u},w.chartContainer=function(e){return arguments.length?(a=e,w):a},w.position=function(e){return arguments.length?(l.left=typeof e.left!="undefined"?e.left:l.left,l.top=typeof e.top!="undefined"?e.top:l.top,w):l},w.fixedTop=function(e){return arguments.length?(o=e,w):o},w.enabled=function(e){return arguments.length?(c=e,w):c},w.valueFormatter=function(e){return arguments.length?(typeof e=="function"&&(d=e),w):d},w.headerFormatter=function(e){return arguments.length?(typeof e=="function"&&(v=e),w):v},w.id=function(){return h},w},e.tooltip.show=function(t,n,r,i,s,o){var u=document.createElement("div");u.className="nvtooltip "+(o?o:"xy-tooltip");var a=s;if(!s||s.tagName.match(/g|svg/i))a=document.getElementsByTagName("body")[0];u.style.left=0,u.style.top=0,u.style.opacity=0,u.innerHTML=n,a.appendChild(u),s&&(t[0]=t[0]-s.scrollLeft,t[1]=t[1]-s.scrollTop),e.tooltip.calcTooltipPosition(t,r,i,u)},e.tooltip.findFirstNonSVGParent=function(e){while(e.tagName.match(/^g|svg$/i)!==null)e=e.parentNode;return e},e.tooltip.findTotalOffsetTop=function(e,t){var n=t;do isNaN(e.offsetTop)||(n+=e.offsetTop);while(e=e.offsetParent);return n},e.tooltip.findTotalOffsetLeft=function(e,t){var n=t;do isNaN(e.offsetLeft)||(n+=e.offsetLeft);while(e=e.offsetParent);return n},e.tooltip.calcTooltipPosition=function(t,n,r,i){var s=parseInt(i.offsetHeight),o=parseInt(i.offsetWidth),u=e.utils.windowSize().width,a=e.utils.windowSize().height,f=window.pageYOffset,l=window.pageXOffset,c,h;a=window.innerWidth>=document.body.scrollWidth?a:a-16,u=window.innerHeight>=document.body.scrollHeight?u:u-16,n=n||"s",r=r||20;var p=function(t){return e.tooltip.findTotalOffsetTop(t,h)},d=function(t){return e.tooltip.findTotalOffsetLeft(t,c)};switch(n){case"e":c=t[0]-o-r,h=t[1]-s/2;var v=d(i),m=p(i);vl?t[0]+r:l-v+c),mf+a&&(h=f+a-m+h-s);break;case"w":c=t[0]+r,h=t[1]-s/2;var v=d(i),m=p(i);v+o>u&&(c=t[0]-o-r),mf+a&&(h=f+a-m+h-s);break;case"n":c=t[0]-o/2-5,h=t[1]+r;var v=d(i),m=p(i);vu&&(c=c-o/2+5),m+s>f+a&&(h=f+a-m+h-s);break;case"s":c=t[0]-o/2,h=t[1]-s-r;var v=d(i),m=p(i);vu&&(c=c-o/2+5),f>m&&(h=f);break;case"none":c=t[0],h=t[1]-r;var v=d(i),m=p(i)}return i.style.left=c+"px",i.style.top=h+"px",i.style.opacity=1,i.style.position="absolute",i},e.tooltip.cleanup=function(){var e=document.getElementsByClassName("nvtooltip"),t=[];while(e.length)t.push(e[0]),e[0].style.transitionDelay="0 !important",e[0].style.opacity=0,e[0].className="nvtooltip-pending-removal";setTimeout(function(){while(t.length){var e=t.pop();e.parentNode.removeChild(e)}},500)}}(),e.utils.windowSize=function(){var e={width:640,height:480};return document.body&&document.body.offsetWidth&&(e.width=document.body.offsetWidth,e.height=document.body.offsetHeight),document.compatMode=="CSS1Compat"&&document.documentElement&&document.documentElement.offsetWidth&&(e.width=document.documentElement.offsetWidth,e.height=document.documentElement.offsetHeight),window.innerWidth&&window.innerHeight&&(e.width=window.innerWidth,e.height=window.innerHeight),e},e.utils.windowResize=function(e){if(e===undefined)return;var t=window.onresize;window.onresize=function(n){typeof t=="function"&&t(n),e(n)}},e.utils.getColor=function(t){return arguments.length?Object.prototype.toString.call(t)==="[object Array]"?function(e,n){return e.color||t[n%t.length]}:t:e.utils.defaultColor()},e.utils.defaultColor=function(){var e=d3.scale.category20().range();return function(t,n){return t.color||e[n%e.length]}},e.utils.customTheme=function(e,t,n){t=t||function(e){return e.key},n=n||d3.scale.category20().range();var r=n.length;return function(i,s){var o=t(i);return r||(r=n.length),typeof e[o]!="undefined"?typeof e[o]=="function"?e[o]():e[o]:n[--r]}},e.utils.pjax=function(t,n){function r(r){d3.html(r,function(r){var i=d3.select(n).node();i.parentNode.replaceChild(d3.select(r).select(n).node(),i),e.utils.pjax(t,n)})}d3.selectAll(t).on("click",function(){history.pushState(this.href,this.textContent,this.href),r(this.href),d3.event.preventDefault()}),d3.select(window).on("popstate",function(){d3.event.state&&r(d3.event.state)})},e.utils.calcApproxTextWidth=function(e){if(typeof e.style=="function"&&typeof e.text=="function"){var t=parseInt(e.style("font-size").replace("px","")),n=e.text().length;return n*t*.5}return 0},e.utils.NaNtoZero=function(e){return typeof e!="number"||isNaN(e)||e===null||e===Infinity?0:e},e.utils.optionsFunc=function(e){return e&&d3.map(e).forEach(function(e,t){typeof this[e]=="function"&&this[e](t)}.bind(this)),this},e.models.axis=function(){"use strict";function m(e){return e.each(function(e){var i=d3.select(this),m=i.selectAll("g.nv-wrap.nv-axis").data([e]),g=m.enter().append("g").attr("class","nvd3 nv-wrap nv-axis"),y=g.append("g"),b=m.select("g");p!==null?t.ticks(p):(t.orient()=="top"||t.orient()=="bottom")&&t.ticks(Math.abs(s.range()[1]-s.range()[0])/100),b.transition().call(t),v=v||t.scale();var w=t.tickFormat();w==null&&(w=v.tickFormat());var E=b.selectAll("text.nv-axislabel").data([o||null]);E.exit().remove();switch(t.orient()){case"top":E.enter().append("text").attr("class","nv-axislabel");var S=s.range().length==2?s.range()[1]:s.range()[s.range().length-1]+(s.range()[1]-s.range()[0]);E.attr("text-anchor","middle").attr("y",0).attr("x",S/2);if(u){var x=m.selectAll("g.nv-axisMaxMin").data(s.domain());x.enter().append("g").attr("class","nv-axisMaxMin").append("text"),x.exit().remove(),x.attr("transform",function(e,t){return"translate("+s(e)+",0)"}).select("text").attr("dy","-0.5em").attr("y",-t.tickPadding()).attr("text-anchor","middle").text(function(e,t){var n=w(e);return(""+n).match("NaN")?"":n}),x.transition().attr("transform",function(e,t){return"translate("+s.range()[t]+",0)"})}break;case"bottom":var T=36,N=30,C=b.selectAll("g").select("text");if(f%360){C.each(function(e,t){var n=this.getBBox().width;n>N&&(N=n)});var k=Math.abs(Math.sin(f*Math.PI/180)),T=(k?k*N:N)+30;C.attr("transform",function(e,t,n){return"rotate("+f+" 0,0)"}).style("text-anchor",f%360>0?"start":"end")}E.enter().append("text").attr("class","nv-axislabel");var S=s.range().length==2?s.range()[1]:s.range()[s.range().length-1]+(s.range()[1]-s.range()[0]);E.attr("text-anchor","middle").attr("y",T).attr("x",S/2);if(u){var x=m.selectAll("g.nv-axisMaxMin").data([s.domain()[0],s.domain()[s.domain().length-1]]);x.enter().append("g").attr("class","nv-axisMaxMin").append("text"),x.exit().remove(),x.attr("transform",function(e,t){return"translate("+(s(e)+(h?s.rangeBand()/2:0))+",0)"}).select("text").attr("dy",".71em").attr("y",t.tickPadding()).attr("transform",function(e,t,n){return"rotate("+f+" 0,0)"}).style("text-anchor",f?f%360>0?"start":"end":"middle").text(function(e,t){var n=w(e);return(""+n).match("NaN")?"":n}),x.transition().attr("transform",function(e,t){return"translate("+(s(e)+(h?s.rangeBand()/2:0))+",0)"})}c&&C.attr("transform",function(e,t){return"translate(0,"+(t%2==0?"0":"12")+")"});break;case"right":E.enter().append("text").attr("class","nv-axislabel"),E.style("text-anchor",l?"middle":"begin").attr("transform",l?"rotate(90)":"").attr("y",l?-Math.max(n.right,r)+12:-10).attr("x",l?s.range()[0]/2:t.tickPadding());if(u){var x=m.selectAll("g.nv-axisMaxMin").data(s.domain());x.enter().append("g").attr("class","nv-axisMaxMin").append("text").style("opacity",0),x.exit().remove(),x.attr("transform",function(e,t){return"translate(0,"+s(e)+")"}).select("text").attr("dy",".32em").attr("y",0).attr("x",t.tickPadding()).style("text-anchor","start").text(function(e,t){var n=w(e);return(""+n).match("NaN")?"":n}),x.transition().attr("transform",function(e,t){return"translate(0,"+s.range()[t]+")"}).select("text").style("opacity",1)}break;case"left":E.enter().append("text").attr("class","nv-axislabel"),E.style("text-anchor",l?"middle":"end").attr("transform",l?"rotate(-90)":"").attr("y",l?-Math.max(n.left,r)+d:-10).attr("x",l?-s.range()[0]/2:-t.tickPadding());if(u){var x=m.selectAll("g.nv-axisMaxMin").data(s.domain());x.enter().append("g").attr("class","nv-axisMaxMin").append("text").style("opacity",0),x.exit().remove(),x.attr("transform",function(e,t){return"translate(0,"+v(e)+")"}).select("text").attr("dy",".32em").attr("y",0).attr("x",-t.tickPadding()).attr("text-anchor","end").text(function(e,t){var n=w(e);return(""+n).match("NaN")?"":n}),x.transition().attr("transform",function(e,t){return"translate(0,"+s.range()[t]+")"}).select("text").style("opacity",1)}}E.text(function(e){return e}),u&&(t.orient()==="left"||t.orient()==="right")&&(b.selectAll("g").each(function(e,t){d3.select(this).select("text").attr("opacity",1);if(s(e)s.range()[0]-10)(e>1e-10||e<-1e-10)&&d3.select(this).attr("opacity",0),d3.select(this).select("text").attr("opacity",0)}),s.domain()[0]==s.domain()[1]&&s.domain()[0]==0&&m.selectAll("g.nv-axisMaxMin").style("opacity",function(e,t){return t?0:1}));if(u&&(t.orient()==="top"||t.orient()==="bottom")){var L=[];m.selectAll("g.nv-axisMaxMin").each(function(e,t){try{t?L.push(s(e)-this.getBBox().width-4):L.push(s(e)+this.getBBox().width+4)}catch(n){t?L.push(s(e)-4):L.push(s(e)+4)}}),b.selectAll("g").each(function(e,t){if(s(e)L[1])e>1e-10||e<-1e-10?d3.select(this).remove():d3.select(this).select("text").remove()})}a&&b.selectAll(".tick").filter(function(e){return!parseFloat(Math.round(e.__data__*1e5)/1e6)&&e.__data__!==undefined}).classed("zero",!0),v=s.copy()}),m}var t=d3.svg.axis(),n={top:0,right:0,bottom:0,left:0},r=75,i=60,s=d3.scale.linear(),o=null,u=!0,a=!0,f=0,l=!0,c=!1,h=!1,p=null,d=12;t.scale(s).orient("bottom").tickFormat(function(e){return e});var v;return m.axis=t,d3.rebind(m,t,"orient","tickValues","tickSubdivide","tickSize","tickPadding","tickFormat"),d3.rebind(m,s,"domain","range","rangeBand","rangeBands"),m.options=e.utils.optionsFunc.bind(m),m.margin=function(e){return arguments.length?(n.top=typeof e.top!="undefined"?e.top:n.top,n.right=typeof e.right!="undefined"?e.right:n.right,n.bottom=typeof e.bottom!="undefined"?e.bottom:n.bottom,n.left=typeof e.left!="undefined"?e.left:n.left,m):n},m.width=function(e){return arguments.length?(r=e,m):r},m.ticks=function(e){return arguments.length?(p=e,m):p},m.height=function(e){return arguments.length?(i=e,m):i},m.axisLabel=function(e){return arguments.length?(o=e,m):o},m.showMaxMin=function(e){return arguments.length?(u=e,m):u},m.highlightZero=function(e){return arguments.length?(a=e,m):a},m.scale=function(e){return arguments.length?(s=e,t.scale(s),h=typeof s.rangeBands=="function",d3.rebind(m,s,"domain","range","rangeBand","rangeBands"),m):s},m.rotateYLabel=function(e){return arguments.length?(l=e,m):l},m.rotateLabels=function(e){return arguments.length?(f=e,m):f},m.staggerLabels=function(e){return arguments.length?(c=e,m):c},m.axisLabelDistance=function(e){return arguments.length?(d=e,m):d},m},e.models.bullet=function(){"use strict";function m(e){return e.each(function(e,n){var p=c-t.left-t.right,m=h-t.top-t.bottom,g=d3.select(this),y=i.call(this,e,n).slice().sort(d3.descending),b=s.call(this,e,n).slice().sort(d3.descending),w=o.call(this,e,n).slice().sort(d3.descending),E=u.call(this,e,n).slice(),S=a.call(this,e,n).slice(),x=f.call(this,e,n).slice(),T=d3.scale.linear().domain(d3.extent(d3.merge([l,y]))).range(r?[p,0]:[0,p]),N=this.__chart__||d3.scale.linear().domain([0,Infinity]).range(T.range());this.__chart__=T;var C=d3.min(y),k=d3.max(y),L=y[1],A=g.selectAll("g.nv-wrap.nv-bullet").data([e]),O=A.enter().append("g").attr("class","nvd3 nv-wrap nv-bullet"),M=O.append("g"),_=A.select("g");M.append("rect").attr("class","nv-range nv-rangeMax"),M.append("rect").attr("class","nv-range nv-rangeAvg"),M.append("rect").attr("class","nv-range nv-rangeMin"),M.append("rect").attr("class","nv-measure"),M.append("path").attr("class","nv-markerTriangle"),A.attr("transform","translate("+t.left+","+t.top+")");var D=function(e){return Math.abs(N(e)-N(0))},P=function(e){return Math.abs(T(e)-T(0))},H=function(e){return e<0?N(e):N(0)},B=function(e){return e<0?T(e):T(0)};_.select("rect.nv-rangeMax").attr("height",m).attr("width",P(k>0?k:C)).attr("x",B(k>0?k:C)).datum(k>0?k:C),_.select("rect.nv-rangeAvg").attr("height",m).attr("width",P(L)).attr("x",B(L)).datum(L),_.select("rect.nv-rangeMin").attr("height",m).attr("width",P(k)).attr("x",B(k)).attr("width",P(k>0?C:k)).attr("x",B(k>0?C:k)).datum(k>0?C:k),_.select("rect.nv-measure").style("fill",d).attr("height",m/3).attr("y",m/3).attr("width",w<0?T(0)-T(w[0]):T(w[0])-T(0)).attr("x",B(w)).on("mouseover",function(){v.elementMouseover({value:w[0],label:x[0]||"Current",pos:[T(w[0]),m/2]})}).on("mouseout",function(){v.elementMouseout({value:w[0],label:x[0]||"Current"})});var j=m/6;b[0]?_.selectAll("path.nv-markerTriangle").attr("transform",function(e){return"translate("+T(b[0])+","+m/2+")"}).attr("d","M0,"+j+"L"+j+","+ -j+" "+ -j+","+ -j+"Z").on("mouseover",function(){v.elementMouseover({value:b[0],label:S[0]||"Previous",pos:[T(b[0]),m/2]})}).on("mouseout",function(){v.elementMouseout({value:b[0],label:S[0]||"Previous"})}):_.selectAll("path.nv-markerTriangle").remove(),A.selectAll(".nv-range").on("mouseover",function(e,t){var n=E[t]||(t?t==1?"Mean":"Minimum":"Maximum");v.elementMouseover({value:e,label:n,pos:[T(e),m/2]})}).on("mouseout",function(e,t){var n=E[t]||(t?t==1?"Mean":"Minimum":"Maximum");v.elementMouseout({value:e,label:n})})}),m}var t={top:0,right:0,bottom:0,left:0},n="left",r=!1,i=function(e){return e.ranges},s=function(e){return e.markers},o=function(e){return e.measures},u=function(e){return e.rangeLabels?e.rangeLabels:[]},a=function(e){return e.markerLabels?e.markerLabels:[]},f=function(e){return e.measureLabels?e.measureLabels:[]},l=[0],c=380,h=30,p=null,d=e.utils.getColor(["#1f77b4"]),v=d3.dispatch("elementMouseover","elementMouseout");return m.dispatch=v,m.options=e.utils.optionsFunc.bind(m),m.orient=function(e){return arguments.length?(n=e,r=n=="right"||n=="bottom",m):n},m.ranges=function(e){return arguments.length?(i=e,m):i},m.markers=function(e){return arguments.length?(s=e,m):s},m.measures=function(e){return arguments.length?(o=e,m):o},m.forceX=function(e){return arguments.length?(l=e,m):l},m.width=function(e){return arguments.length?(c=e,m):c},m.height=function(e){return arguments.length?(h=e,m):h},m.margin=function(e){return arguments.length?(t.top=typeof e.top!="undefined"?e.top:t.top,t.right=typeof e.right!="undefined"?e.right:t.right,t.bottom=typeof e.bottom!="undefined"?e.bottom:t.bottom,t.left=typeof e.left!="undefined"?e.left:t.left,m):t},m.tickFormat=function(e){return arguments.length?(p=e,m):p},m.color=function(t){return arguments.length?(d=e.utils.getColor(t),m):d},m},e.models.bulletChart=function(){"use strict";function m(e){return e.each(function(n,h){var g=d3.select(this),y=(a||parseInt(g.style("width"))||960)-i.left-i.right,b=f-i.top-i.bottom,w=this;m.update=function(){m(e)},m.container=this;if(!n||!s.call(this,n,h)){var E=g.selectAll(".nv-noData").data([p]);return E.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),E.attr("x",i.left+y/2).attr("y",18+i.top+b/2).text(function(e){return e}),m}g.selectAll(".nv-noData").remove();var S=s.call(this,n,h).slice().sort(d3.descending),x=o.call(this,n,h).slice().sort(d3.descending),T=u.call(this,n,h).slice().sort(d3.descending),N=g.selectAll("g.nv-wrap.nv-bulletChart").data([n]),C=N.enter().append("g").attr("class","nvd3 nv-wrap nv-bulletChart"),k=C.append("g"),L=N.select("g");k.append("g").attr("class","nv-bulletWrap"),k.append("g").attr("class","nv-titles"),N.attr("transform","translate("+i.left+","+i.top+")");var A=d3.scale.linear().domain([0,Math.max(S[0],x[0],T[0])]).range(r?[y,0]:[0,y]),O=this.__chart__||d3.scale.linear().domain([0,Infinity]).range(A.range());this.__chart__=A;var M=function(e){return Math.abs(O(e)-O(0))},_=function(e){return Math.abs(A(e)-A(0))},D=k.select(".nv-titles").append("g").attr("text-anchor","end").attr("transform","translate(-6,"+(f-i.top-i.bottom)/2+")");D.append("text").attr("class","nv-title").text(function(e){return e.title}),D.append("text").attr("class","nv-subtitle").attr("dy","1em").text(function(e){return e.subtitle}),t.width(y).height(b);var P=L.select(".nv-bulletWrap");d3.transition(P).call(t);var H=l||A.tickFormat(y/100),B=L.selectAll("g.nv-tick").data(A.ticks(y/50),function(e){return this.textContent||H(e)}),j=B.enter().append("g").attr("class","nv-tick").attr("transform",function(e){return"translate("+O(e)+",0)"}).style("opacity",1e-6);j.append("line").attr("y1",b).attr("y2",b*7/6),j.append("text").attr("text-anchor","middle").attr("dy","1em").attr("y",b*7/6).text(H);var F=d3.transition(B).attr("transform",function(e){return"translate("+A(e)+",0)"}).style("opacity",1);F.select("line").attr("y1",b).attr("y2",b*7/6),F.select("text").attr("y",b*7/6),d3.transition(B.exit()).attr("transform",function(e){return"translate("+A(e)+",0)"}).style("opacity",1e-6).remove(),d.on("tooltipShow",function(e){e.key=n.title,c&&v(e,w.parentNode)})}),d3.timer.flush(),m}var t=e.models.bullet(),n="left",r=!1,i={top:5,right:40,bottom:20,left:120},s=function(e){return e.ranges},o=function(e){return e.markers},u=function(e){return e.measures},a=null,f=55,l=null,c=!0,h=function(e,t,n,r,i){return"

    "+t+"

    "+"

    "+n+"

    "},p="No Data Available.",d=d3.dispatch("tooltipShow","tooltipHide"),v=function(t,n){var r=t.pos[0]+(n.offsetLeft||0)+i.left,s=t.pos[1]+(n.offsetTop||0)+i.top,o=h(t.key,t.label,t.value,t,m);e.tooltip.show([r,s],o,t.value<0?"e":"w",null,n)};return t.dispatch.on("elementMouseover.tooltip",function(e){d.tooltipShow(e)}),t.dispatch.on("elementMouseout.tooltip",function(e){d.tooltipHide(e)}),d.on("tooltipHide",function(){c&&e.tooltip.cleanup()}),m.dispatch=d,m.bullet=t,d3.rebind(m,t,"color"),m.options=e.utils.optionsFunc.bind(m),m.orient=function(e){return arguments.length?(n=e,r=n=="right"||n=="bottom",m):n},m.ranges=function(e){return arguments.length?(s=e,m):s},m.markers=function(e){return arguments.length?(o=e,m):o},m.measures=function(e){return arguments.length?(u=e,m):u},m.width=function(e){return arguments.length?(a=e,m):a},m.height=function(e){return arguments.length?(f=e,m):f},m.margin=function(e){return arguments.length?(i.top=typeof e.top!="undefined"?e.top:i.top,i.right=typeof e.right!="undefined"?e.right:i.right,i.bottom=typeof e.bottom!="undefined"?e.bottom:i.bottom,i.left=typeof e.left!="undefined"?e.left:i.left,m):i},m.tickFormat=function(e){return arguments.length?(l=e,m):l},m.tooltips=function(e){return arguments.length?(c=e,m):c},m.tooltipContent=function(e){return arguments.length?(h=e,m):h},m.noData=function(e){return arguments.length?(p=e,m):p},m},e.models.cumulativeLineChart=function(){"use strict";function D(b){return b.each(function(b){function q(e,t){d3.select(D.container).style("cursor","ew-resize")}function R(e,t){M.x=d3.event.x,M.i=Math.round(O.invert(M.x)),rt()}function U(e,t){d3.select(D.container).style("cursor","auto"),x.index=M.i,k.stateChange(x)}function rt(){nt.data([M]);var e=D.transitionDuration();D.transitionDuration(0),D.update(),D.transitionDuration(e)}var A=d3.select(this).classed("nv-chart-"+S,!0),H=this,B=(f||parseInt(A.style("width"))||960)-u.left-u.right,j=(l||parseInt(A.style("height"))||400)-u.top-u.bottom;D.update=function(){A.transition().duration(L).call(D)},D.container=this,x.disabled=b.map(function(e){return!!e.disabled});if(!T){var F;T={};for(F in x)x[F]instanceof Array?T[F]=x[F].slice(0):T[F]=x[F]}var I=d3.behavior.drag().on("dragstart",q).on("drag",R).on("dragend",U);if(!b||!b.length||!b.filter(function(e){return e.values.length}).length){var z=A.selectAll(".nv-noData").data([N]);return z.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),z.attr("x",u.left+B/2).attr("y",u.top+j/2).text(function(e){return e}),D}A.selectAll(".nv-noData").remove(),w=t.xScale(),E=t.yScale();if(!y){var W=b.filter(function(e){return!e.disabled}).map(function(e,n){var r=d3.extent(e.values,t.y());return r[0]<-0.95&&(r[0]=-0.95),[(r[0]-r[1])/(1+r[1]),(r[1]-r[0])/(1+r[0])]}),X=[d3.min(W,function(e){return e[0]}),d3.max(W,function(e){return e[1]})];t.yDomain(X)}else t.yDomain(null);O.domain([0,b[0].values.length-1]).range([0,B]).clamp(!0);var b=P(M.i,b),V=g?"none":"all",$=A.selectAll("g.nv-wrap.nv-cumulativeLine").data([b]),J=$.enter().append("g").attr("class","nvd3 nv-wrap nv-cumulativeLine").append("g"),K=$.select("g");J.append("g").attr("class","nv-interactive"),J.append("g").attr("class","nv-x nv-axis").style("pointer-events","none"),J.append("g").attr("class","nv-y nv-axis"),J.append("g").attr("class","nv-background"),J.append("g").attr("class","nv-linesWrap").style("pointer-events",V),J.append("g").attr("class","nv-avgLinesWrap").style("pointer-events","none"),J.append("g").attr("class","nv-legendWrap"),J.append("g").attr("class","nv-controlsWrap"),c&&(i.width(B),K.select(".nv-legendWrap").datum(b).call(i),u.top!=i.height()&&(u.top=i.height(),j=(l||parseInt(A.style("height"))||400)-u.top-u.bottom),K.select(".nv-legendWrap").attr("transform","translate(0,"+ -u.top+")"));if(m){var Q=[{key:"Re-scale y-axis",disabled:!y}];s.width(140).color(["#444","#444","#444"]).rightAlign(!1).margin({top:5,right:0,bottom:5,left:20}),K.select(".nv-controlsWrap").datum(Q).attr("transform","translate(0,"+ -u.top+")").call(s)}$.attr("transform","translate("+u.left+","+u.top+")"),d&&K.select(".nv-y.nv-axis").attr("transform","translate("+B+",0)");var G=b.filter(function(e){return e.tempDisabled});$.select(".tempDisabled").remove(),G.length&&$.append("text").attr("class","tempDisabled").attr("x",B/2).attr("y","-.71em").style("text-anchor","end").text(G.map(function(e){return e.key}).join(", ")+" values cannot be calculated for this time period."),g&&(o.width(B).height(j).margin({left:u.left,top:u.top}).svgContainer(A).xScale(w),$.select(".nv-interactive").call(o)),J.select(".nv-background").append("rect"),K.select(".nv-background rect").attr("width",B).attr("height",j),t.y(function(e){return e.display.y}).width(B).height(j).color(b.map(function(e,t){return e.color||a(e,t)}).filter(function(e,t){return!b[t].disabled&&!b[t].tempDisabled}));var Y=K.select(".nv-linesWrap").datum(b.filter(function(e){return!e.disabled&&!e.tempDisabled}));Y.call(t),b.forEach(function(e,t){e.seriesIndex=t});var Z=b.filter(function(e){return!e.disabled&&!!C(e)}),et=K.select(".nv-avgLinesWrap").selectAll("line").data(Z,function(e){return e.key}),tt=function(e){var t=E(C(e));return t<0?0:t>j?j:t};et.enter().append("line").style("stroke-width",2).style("stroke-dasharray","10,10").style("stroke",function(e,n){return t.color()(e,e.seriesIndex)}).attr("x1",0).attr("x2",B).attr("y1",tt).attr("y2",tt),et.style("stroke-opacity",function(e){var t=E(C(e));return t<0||t>j?0:1}).attr("x1",0).attr("x2",B).attr("y1",tt).attr("y2",tt),et.exit().remove();var nt=Y.selectAll(".nv-indexLine").data([M]);nt.enter().append("rect").attr("class","nv-indexLine").attr("width",3).attr("x",-2).attr("fill","red").attr("fill-opacity",.5).style("pointer-events","all").call(I),nt.attr("transform",function(e){return"translate("+O(e.i)+",0)"}).attr("height",j),h&&(n.scale(w).ticks(Math.min(b[0].values.length,B/70)).tickSize(-j,0),K.select(".nv-x.nv-axis").attr("transform","translate(0,"+E.range()[0]+")"),d3.transition(K.select(".nv-x.nv-axis")).call(n)),p&&(r.scale(E).ticks(j/36).tickSize(-B,0),d3.transition(K.select(".nv-y.nv-axis")).call(r)),K.select(".nv-background rect").on("click",function(){M.x=d3.mouse(this)[0],M.i=Math.round(O.invert(M.x)),x.index=M.i,k.stateChange(x),rt()}),t.dispatch.on("elementClick",function(e){M.i=e.pointIndex,M.x=O(M.i),x.index=M.i,k.stateChange(x),rt()}),s.dispatch.on("legendClick",function(e,t){e.disabled=!e.disabled,y=!e.disabled,x.rescaleY=y,k.stateChange(x),D.update()}),i.dispatch.on("stateChange",function(e){x.disabled=e.disabled,k.stateChange(x),D.update()}),o.dispatch.on("elementMousemove",function(i){t.clearHighlights();var s,f,l,c=[];b.filter(function(e,t){return e.seriesIndex=t,!e.disabled}).forEach(function(n,r){f=e.interactiveBisect(n.values,i.pointXValue,D.x()),t.highlightPoint(r,f,!0);var o=n.values[f];if(typeof o=="undefined")return;typeof s=="undefined"&&(s=o),typeof l=="undefined"&&(l=D.xScale()(D.x()(o,f))),c.push({key:n.key,value:D.y()(o,f),color:a(n,n.seriesIndex)})});if(c.length>2){var h=D.yScale().invert(i.mouseY),p=Math.abs(D.yScale().domain()[0]-D.yScale().domain()[1]),d=.03*p,m=e.nearestValueIndex(c.map(function(e){return e.value}),h,d);m!==null&&(c[m].highlight=!0)}var g=n.tickFormat()(D.x()(s,f),f);o.tooltip.position({left:l+u.left,top:i.mouseY+u.top}).chartContainer(H.parentNode).enabled(v).valueFormatter(function(e,t){return r.tickFormat()(e)}).data({value:g,series:c})(),o.renderGuideLine(l)}),o.dispatch.on("elementMouseout",function(e){k.tooltipHide(),t.clearHighlights()}),k.on("tooltipShow",function(e){v&&_(e,H.parentNode)}),k.on("changeState",function(e){typeof e.disabled!="undefined"&&(b.forEach(function(t,n){t.disabled=e.disabled[n]}),x.disabled=e.disabled),typeof e.index!="undefined"&&(M.i=e.index,M.x=O(M.i),x.index=e.index,nt.data([M])),typeof e.rescaleY!="undefined"&&(y=e.rescaleY),D.update()})}),D}function P(e,n){return n.map(function(n,r){if(!n.values)return n;var i=t.y()(n.values[e],e);return i<-0.95&&!A?(n.tempDisabled=!0,n):(n.tempDisabled=!1,n.values= +n.values.map(function(e,n){return e.display={y:(t.y()(e,n)-i)/(1+i)},e}),n)})}var t=e.models.line(),n=e.models.axis(),r=e.models.axis(),i=e.models.legend(),s=e.models.legend(),o=e.interactiveGuideline(),u={top:30,right:30,bottom:50,left:60},a=e.utils.defaultColor(),f=null,l=null,c=!0,h=!0,p=!0,d=!1,v=!0,m=!0,g=!1,y=!0,b=function(e,t,n,r,i){return"

    "+e+"

    "+"

    "+n+" at "+t+"

    "},w,E,S=t.id(),x={index:0,rescaleY:y},T=null,N="No Data Available.",C=function(e){return e.average},k=d3.dispatch("tooltipShow","tooltipHide","stateChange","changeState"),L=250,A=!1;n.orient("bottom").tickPadding(7),r.orient(d?"right":"left"),s.updateState(!1);var O=d3.scale.linear(),M={i:0,x:0},_=function(i,s){var o=i.pos[0]+(s.offsetLeft||0),u=i.pos[1]+(s.offsetTop||0),a=n.tickFormat()(t.x()(i.point,i.pointIndex)),f=r.tickFormat()(t.y()(i.point,i.pointIndex)),l=b(i.series.key,a,f,i,D);e.tooltip.show([o,u],l,null,null,s)};return t.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+u.left,e.pos[1]+u.top],k.tooltipShow(e)}),t.dispatch.on("elementMouseout.tooltip",function(e){k.tooltipHide(e)}),k.on("tooltipHide",function(){v&&e.tooltip.cleanup()}),D.dispatch=k,D.lines=t,D.legend=i,D.xAxis=n,D.yAxis=r,D.interactiveLayer=o,d3.rebind(D,t,"defined","isArea","x","y","xScale","yScale","size","xDomain","yDomain","xRange","yRange","forceX","forceY","interactive","clipEdge","clipVoronoi","useVoronoi","id"),D.options=e.utils.optionsFunc.bind(D),D.margin=function(e){return arguments.length?(u.top=typeof e.top!="undefined"?e.top:u.top,u.right=typeof e.right!="undefined"?e.right:u.right,u.bottom=typeof e.bottom!="undefined"?e.bottom:u.bottom,u.left=typeof e.left!="undefined"?e.left:u.left,D):u},D.width=function(e){return arguments.length?(f=e,D):f},D.height=function(e){return arguments.length?(l=e,D):l},D.color=function(t){return arguments.length?(a=e.utils.getColor(t),i.color(a),D):a},D.rescaleY=function(e){return arguments.length?(y=e,D):y},D.showControls=function(e){return arguments.length?(m=e,D):m},D.useInteractiveGuideline=function(e){return arguments.length?(g=e,e===!0&&(D.interactive(!1),D.useVoronoi(!1)),D):g},D.showLegend=function(e){return arguments.length?(c=e,D):c},D.showXAxis=function(e){return arguments.length?(h=e,D):h},D.showYAxis=function(e){return arguments.length?(p=e,D):p},D.rightAlignYAxis=function(e){return arguments.length?(d=e,r.orient(e?"right":"left"),D):d},D.tooltips=function(e){return arguments.length?(v=e,D):v},D.tooltipContent=function(e){return arguments.length?(b=e,D):b},D.state=function(e){return arguments.length?(x=e,D):x},D.defaultState=function(e){return arguments.length?(T=e,D):T},D.noData=function(e){return arguments.length?(N=e,D):N},D.average=function(e){return arguments.length?(C=e,D):C},D.transitionDuration=function(e){return arguments.length?(L=e,D):L},D.noErrorCheck=function(e){return arguments.length?(A=e,D):A},D},e.models.discreteBar=function(){"use strict";function E(e){return e.each(function(e){var i=n-t.left-t.right,E=r-t.top-t.bottom,S=d3.select(this);e.forEach(function(e,t){e.values.forEach(function(e){e.series=t})});var T=p&&d?[]:e.map(function(e){return e.values.map(function(e,t){return{x:u(e,t),y:a(e,t),y0:e.y0}})});s.domain(p||d3.merge(T).map(function(e){return e.x})).rangeBands(v||[0,i],.1),o.domain(d||d3.extent(d3.merge(T).map(function(e){return e.y}).concat(f))),c?o.range(m||[E-(o.domain()[0]<0?12:0),o.domain()[1]>0?12:0]):o.range(m||[E,0]),b=b||s,w=w||o.copy().range([o(0),o(0)]);var N=S.selectAll("g.nv-wrap.nv-discretebar").data([e]),C=N.enter().append("g").attr("class","nvd3 nv-wrap nv-discretebar"),k=C.append("g"),L=N.select("g");k.append("g").attr("class","nv-groups"),N.attr("transform","translate("+t.left+","+t.top+")");var A=N.select(".nv-groups").selectAll(".nv-group").data(function(e){return e},function(e){return e.key});A.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6),A.exit().transition().style("stroke-opacity",1e-6).style("fill-opacity",1e-6).remove(),A.attr("class",function(e,t){return"nv-group nv-series-"+t}).classed("hover",function(e){return e.hover}),A.transition().style("stroke-opacity",1).style("fill-opacity",.75);var O=A.selectAll("g.nv-bar").data(function(e){return e.values});O.exit().remove();var M=O.enter().append("g").attr("transform",function(e,t,n){return"translate("+(s(u(e,t))+s.rangeBand()*.05)+", "+o(0)+")"}).on("mouseover",function(t,n){d3.select(this).classed("hover",!0),g.elementMouseover({value:a(t,n),point:t,series:e[t.series],pos:[s(u(t,n))+s.rangeBand()*(t.series+.5)/e.length,o(a(t,n))],pointIndex:n,seriesIndex:t.series,e:d3.event})}).on("mouseout",function(t,n){d3.select(this).classed("hover",!1),g.elementMouseout({value:a(t,n),point:t,series:e[t.series],pointIndex:n,seriesIndex:t.series,e:d3.event})}).on("click",function(t,n){g.elementClick({value:a(t,n),point:t,series:e[t.series],pos:[s(u(t,n))+s.rangeBand()*(t.series+.5)/e.length,o(a(t,n))],pointIndex:n,seriesIndex:t.series,e:d3.event}),d3.event.stopPropagation()}).on("dblclick",function(t,n){g.elementDblClick({value:a(t,n),point:t,series:e[t.series],pos:[s(u(t,n))+s.rangeBand()*(t.series+.5)/e.length,o(a(t,n))],pointIndex:n,seriesIndex:t.series,e:d3.event}),d3.event.stopPropagation()});M.append("rect").attr("height",0).attr("width",s.rangeBand()*.9/e.length),c?(M.append("text").attr("text-anchor","middle"),O.select("text").text(function(e,t){return h(a(e,t))}).transition().attr("x",s.rangeBand()*.9/2).attr("y",function(e,t){return a(e,t)<0?o(a(e,t))-o(0)+12:-4})):O.selectAll("text").remove(),O.attr("class",function(e,t){return a(e,t)<0?"nv-bar negative":"nv-bar positive"}).style("fill",function(e,t){return e.color||l(e,t)}).style("stroke",function(e,t){return e.color||l(e,t)}).select("rect").attr("class",y).transition().attr("width",s.rangeBand()*.9/e.length),O.transition().attr("transform",function(e,t){var n=s(u(e,t))+s.rangeBand()*.05,r=a(e,t)<0?o(0):o(0)-o(a(e,t))<1?o(0)-1:o(a(e,t));return"translate("+n+", "+r+")"}).select("rect").attr("height",function(e,t){return Math.max(Math.abs(o(a(e,t))-o(d&&d[0]||0))||1)}),b=s.copy(),w=o.copy()}),E}var t={top:0,right:0,bottom:0,left:0},n=960,r=500,i=Math.floor(Math.random()*1e4),s=d3.scale.ordinal(),o=d3.scale.linear(),u=function(e){return e.x},a=function(e){return e.y},f=[0],l=e.utils.defaultColor(),c=!1,h=d3.format(",.2f"),p,d,v,m,g=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout"),y="discreteBar",b,w;return E.dispatch=g,E.options=e.utils.optionsFunc.bind(E),E.x=function(e){return arguments.length?(u=e,E):u},E.y=function(e){return arguments.length?(a=e,E):a},E.margin=function(e){return arguments.length?(t.top=typeof e.top!="undefined"?e.top:t.top,t.right=typeof e.right!="undefined"?e.right:t.right,t.bottom=typeof e.bottom!="undefined"?e.bottom:t.bottom,t.left=typeof e.left!="undefined"?e.left:t.left,E):t},E.width=function(e){return arguments.length?(n=e,E):n},E.height=function(e){return arguments.length?(r=e,E):r},E.xScale=function(e){return arguments.length?(s=e,E):s},E.yScale=function(e){return arguments.length?(o=e,E):o},E.xDomain=function(e){return arguments.length?(p=e,E):p},E.yDomain=function(e){return arguments.length?(d=e,E):d},E.xRange=function(e){return arguments.length?(v=e,E):v},E.yRange=function(e){return arguments.length?(m=e,E):m},E.forceY=function(e){return arguments.length?(f=e,E):f},E.color=function(t){return arguments.length?(l=e.utils.getColor(t),E):l},E.id=function(e){return arguments.length?(i=e,E):i},E.showValues=function(e){return arguments.length?(c=e,E):c},E.valueFormat=function(e){return arguments.length?(h=e,E):h},E.rectClass=function(e){return arguments.length?(y=e,E):y},E},e.models.discreteBarChart=function(){"use strict";function w(e){return e.each(function(e){var u=d3.select(this),p=this,E=(s||parseInt(u.style("width"))||960)-i.left-i.right,S=(o||parseInt(u.style("height"))||400)-i.top-i.bottom;w.update=function(){g.beforeUpdate(),u.transition().duration(y).call(w)},w.container=this;if(!e||!e.length||!e.filter(function(e){return e.values.length}).length){var T=u.selectAll(".nv-noData").data([m]);return T.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),T.attr("x",i.left+E/2).attr("y",i.top+S/2).text(function(e){return e}),w}u.selectAll(".nv-noData").remove(),d=t.xScale(),v=t.yScale().clamp(!0);var N=u.selectAll("g.nv-wrap.nv-discreteBarWithAxes").data([e]),C=N.enter().append("g").attr("class","nvd3 nv-wrap nv-discreteBarWithAxes").append("g"),k=C.append("defs"),L=N.select("g");C.append("g").attr("class","nv-x nv-axis"),C.append("g").attr("class","nv-y nv-axis").append("g").attr("class","nv-zeroLine").append("line"),C.append("g").attr("class","nv-barsWrap"),L.attr("transform","translate("+i.left+","+i.top+")"),l&&L.select(".nv-y.nv-axis").attr("transform","translate("+E+",0)"),t.width(E).height(S);var A=L.select(".nv-barsWrap").datum(e.filter(function(e){return!e.disabled}));A.transition().call(t),k.append("clipPath").attr("id","nv-x-label-clip-"+t.id()).append("rect"),L.select("#nv-x-label-clip-"+t.id()+" rect").attr("width",d.rangeBand()*(c?2:1)).attr("height",16).attr("x",-d.rangeBand()/(c?1:2));if(a){n.scale(d).ticks(E/100).tickSize(-S,0),L.select(".nv-x.nv-axis").attr("transform","translate(0,"+(v.range()[0]+(t.showValues()&&v.domain()[0]<0?16:0))+")"),L.select(".nv-x.nv-axis").transition().call(n);var O=L.select(".nv-x.nv-axis").selectAll("g");c&&O.selectAll("text").attr("transform",function(e,t,n){return"translate(0,"+(n%2==0?"5":"17")+")"})}f&&(r.scale(v).ticks(S/36).tickSize(-E,0),L.select(".nv-y.nv-axis").transition().call(r)),L.select(".nv-zeroLine line").attr("x1",0).attr("x2",E).attr("y1",v(0)).attr("y2",v(0)),g.on("tooltipShow",function(e){h&&b(e,p.parentNode)})}),w}var t=e.models.discreteBar(),n=e.models.axis(),r=e.models.axis(),i={top:15,right:10,bottom:50,left:60},s=null,o=null,u=e.utils.getColor(),a=!0,f=!0,l=!1,c=!1,h=!0,p=function(e,t,n,r,i){return"

    "+t+"

    "+"

    "+n+"

    "},d,v,m="No Data Available.",g=d3.dispatch("tooltipShow","tooltipHide","beforeUpdate"),y=250;n.orient("bottom").highlightZero(!1).showMaxMin(!1).tickFormat(function(e){return e}),r.orient(l?"right":"left").tickFormat(d3.format(",.1f"));var b=function(i,s){var o=i.pos[0]+(s.offsetLeft||0),u=i.pos[1]+(s.offsetTop||0),a=n.tickFormat()(t.x()(i.point,i.pointIndex)),f=r.tickFormat()(t.y()(i.point,i.pointIndex)),l=p(i.series.key,a,f,i,w);e.tooltip.show([o,u],l,i.value<0?"n":"s",null,s)};return t.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+i.left,e.pos[1]+i.top],g.tooltipShow(e)}),t.dispatch.on("elementMouseout.tooltip",function(e){g.tooltipHide(e)}),g.on("tooltipHide",function(){h&&e.tooltip.cleanup()}),w.dispatch=g,w.discretebar=t,w.xAxis=n,w.yAxis=r,d3.rebind(w,t,"x","y","xDomain","yDomain","xRange","yRange","forceX","forceY","id","showValues","valueFormat"),w.options=e.utils.optionsFunc.bind(w),w.margin=function(e){return arguments.length?(i.top=typeof e.top!="undefined"?e.top:i.top,i.right=typeof e.right!="undefined"?e.right:i.right,i.bottom=typeof e.bottom!="undefined"?e.bottom:i.bottom,i.left=typeof e.left!="undefined"?e.left:i.left,w):i},w.width=function(e){return arguments.length?(s=e,w):s},w.height=function(e){return arguments.length?(o=e,w):o},w.color=function(n){return arguments.length?(u=e.utils.getColor(n),t.color(u),w):u},w.showXAxis=function(e){return arguments.length?(a=e,w):a},w.showYAxis=function(e){return arguments.length?(f=e,w):f},w.rightAlignYAxis=function(e){return arguments.length?(l=e,r.orient(e?"right":"left"),w):l},w.staggerLabels=function(e){return arguments.length?(c=e,w):c},w.tooltips=function(e){return arguments.length?(h=e,w):h},w.tooltipContent=function(e){return arguments.length?(p=e,w):p},w.noData=function(e){return arguments.length?(m=e,w):m},w.transitionDuration=function(e){return arguments.length?(y=e,w):y},w},e.models.distribution=function(){"use strict";function l(e){return e.each(function(e){var a=n-(i==="x"?t.left+t.right:t.top+t.bottom),l=i=="x"?"y":"x",c=d3.select(this);f=f||u;var h=c.selectAll("g.nv-distribution").data([e]),p=h.enter().append("g").attr("class","nvd3 nv-distribution"),d=p.append("g"),v=h.select("g");h.attr("transform","translate("+t.left+","+t.top+")");var m=v.selectAll("g.nv-dist").data(function(e){return e},function(e){return e.key});m.enter().append("g"),m.attr("class",function(e,t){return"nv-dist nv-series-"+t}).style("stroke",function(e,t){return o(e,t)});var g=m.selectAll("line.nv-dist"+i).data(function(e){return e.values});g.enter().append("line").attr(i+"1",function(e,t){return f(s(e,t))}).attr(i+"2",function(e,t){return f(s(e,t))}),m.exit().selectAll("line.nv-dist"+i).transition().attr(i+"1",function(e,t){return u(s(e,t))}).attr(i+"2",function(e,t){return u(s(e,t))}).style("stroke-opacity",0).remove(),g.attr("class",function(e,t){return"nv-dist"+i+" nv-dist"+i+"-"+t}).attr(l+"1",0).attr(l+"2",r),g.transition().attr(i+"1",function(e,t){return u(s(e,t))}).attr(i+"2",function(e,t){return u(s(e,t))}),f=u.copy()}),l}var t={top:0,right:0,bottom:0,left:0},n=400,r=8,i="x",s=function(e){return e[i]},o=e.utils.defaultColor(),u=d3.scale.linear(),a,f;return l.options=e.utils.optionsFunc.bind(l),l.margin=function(e){return arguments.length?(t.top=typeof e.top!="undefined"?e.top:t.top,t.right=typeof e.right!="undefined"?e.right:t.right,t.bottom=typeof e.bottom!="undefined"?e.bottom:t.bottom,t.left=typeof e.left!="undefined"?e.left:t.left,l):t},l.width=function(e){return arguments.length?(n=e,l):n},l.axis=function(e){return arguments.length?(i=e,l):i},l.size=function(e){return arguments.length?(r=e,l):r},l.getData=function(e){return arguments.length?(s=d3.functor(e),l):s},l.scale=function(e){return arguments.length?(u=e,l):u},l.color=function(t){return arguments.length?(o=e.utils.getColor(t),l):o},l},e.models.historicalBar=function(){"use strict";function w(E){return E.each(function(w){var E=n-t.left-t.right,S=r-t.top-t.bottom,T=d3.select(this);s.domain(d||d3.extent(w[0].values.map(u).concat(f))),c?s.range(m||[E*.5/w[0].values.length,E*(w[0].values.length-.5)/w[0].values.length]):s.range(m||[0,E]),o.domain(v||d3.extent(w[0].values.map(a).concat(l))).range(g||[S,0]),s.domain()[0]===s.domain()[1]&&(s.domain()[0]?s.domain([s.domain()[0]-s.domain()[0]*.01,s.domain()[1]+s.domain()[1]*.01]):s.domain([-1,1])),o.domain()[0]===o.domain()[1]&&(o.domain()[0]?o.domain([o.domain()[0]+o.domain()[0]*.01,o.domain()[1]-o.domain()[1]*.01]):o.domain([-1,1]));var N=T.selectAll("g.nv-wrap.nv-historicalBar-"+i).data([w[0].values]),C=N.enter().append("g").attr("class","nvd3 nv-wrap nv-historicalBar-"+i),k=C.append("defs"),L=C.append("g"),A=N.select("g");L.append("g").attr("class","nv-bars"),N.attr("transform","translate("+t.left+","+t.top+")"),T.on("click",function(e,t){y.chartClick({data:e,index:t,pos:d3.event,id:i})}),k.append("clipPath").attr("id","nv-chart-clip-path-"+i).append("rect"),N.select("#nv-chart-clip-path-"+i+" rect").attr("width",E).attr("height",S),A.attr("clip-path",h?"url(#nv-chart-clip-path-"+i+")":"");var O=N.select(".nv-bars").selectAll(".nv-bar").data(function(e){return e},function(e,t){return u(e,t)});O.exit().remove();var M=O.enter().append("rect").attr("x",0).attr("y",function(t,n){return e.utils.NaNtoZero(o(Math.max(0,a(t,n))))}).attr("height",function(t,n){return e.utils.NaNtoZero(Math.abs(o(a(t,n))-o(0)))}).attr("transform",function(e,t){return"translate("+(s(u(e,t))-E/w[0].values.length*.45)+",0)"}).on("mouseover",function(e,t){if(!b)return;d3.select(this).classed("hover",!0),y.elementMouseover({point:e,series:w[0],pos:[s(u(e,t)),o(a(e,t))],pointIndex:t,seriesIndex:0,e:d3.event})}).on("mouseout",function(e,t){if(!b)return;d3.select(this).classed("hover",!1),y.elementMouseout({point:e,series:w[0],pointIndex:t,seriesIndex:0,e:d3.event})}).on("click",function(e,t){if(!b)return;y.elementClick({value:a(e,t),data:e,index:t,pos:[s(u(e,t)),o(a(e,t))],e:d3.event,id:i}),d3.event.stopPropagation()}).on("dblclick",function(e,t){if(!b)return;y.elementDblClick({value:a(e,t),data:e,index:t,pos:[s(u(e,t)),o(a(e,t))],e:d3.event,id:i}),d3.event.stopPropagation()});O.attr("fill",function(e,t){return p(e,t)}).attr("class",function(e,t,n){return(a(e,t)<0?"nv-bar negative":"nv-bar positive")+" nv-bar-"+n+"-"+t}).transition().attr("transform",function(e,t){return"translate("+(s(u(e,t))-E/w[0].values.length*.45)+",0)"}).attr("width",E/w[0].values.length*.9),O.transition().attr("y",function(t,n){var r=a(t,n)<0?o(0):o(0)-o(a(t,n))<1?o(0)-1:o(a(t,n));return e.utils.NaNtoZero(r)}).attr("height",function(t,n){return e.utils.NaNtoZero(Math.max(Math.abs(o(a(t,n))-o(0)),1))})}),w}var t={top:0,right:0,bottom:0,left:0},n=960,r=500,i=Math.floor(Math.random()*1e4),s=d3.scale.linear(),o=d3.scale.linear(),u=function(e){return e.x},a=function(e){return e.y},f=[],l=[0],c=!1,h=!0,p=e.utils.defaultColor(),d,v,m,g,y=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout"),b=!0;return w.highlightPoint=function(e,t){d3.select(".nv-historicalBar-"+i).select(".nv-bars .nv-bar-0-"+e).classed("hover",t)},w.clearHighlights=function(){d3.select(".nv-historicalBar-"+i).select(".nv-bars .nv-bar.hover").classed("hover",!1)},w.dispatch=y,w.options=e.utils.optionsFunc.bind(w),w.x=function(e){return arguments.length?(u=e,w):u},w.y=function(e){return arguments.length?(a=e,w):a},w.margin=function(e){return arguments.length?(t.top=typeof e.top!="undefined"?e.top:t.top,t.right=typeof e.right!="undefined"?e.right:t.right,t.bottom=typeof e.bottom!="undefined"?e.bottom:t.bottom,t.left=typeof e.left!="undefined"?e.left:t.left,w):t},w.width=function(e){return arguments.length?(n=e,w):n},w.height=function(e){return arguments.length?(r=e,w):r},w.xScale=function(e){return arguments.length?(s=e,w):s},w.yScale=function(e){return arguments.length?(o=e,w):o},w.xDomain=function(e){return arguments.length?(d=e,w):d},w.yDomain=function(e){return arguments.length?(v=e,w):v},w.xRange=function(e){return arguments.length?(m=e,w):m},w.yRange=function(e){return arguments.length?(g=e,w):g},w.forceX=function(e){return arguments.length?(f=e,w):f},w.forceY=function(e){return arguments.length?(l=e,w):l},w.padData=function(e){return arguments.length?(c=e,w):c},w.clipEdge=function(e){return arguments.length?(h=e,w):h},w.color=function(t){return arguments.length?(p=e.utils.getColor(t),w):p},w.id=function(e){return arguments.length?(i=e,w):i},w.interactive=function(e){return arguments.length?(b=!1,w):b},w},e.models.historicalBarChart=function(){"use strict";function x(e){return e.each(function(d){var T=d3.select(this),N=this,C=(u||parseInt(T.style("width"))||960)-s.left-s.right,k=(a||parseInt(T.style("height"))||400)-s.top-s.bottom;x.update=function(){T.transition().duration(E).call(x)},x.container=this,g.disabled=d.map(function(e){return!!e.disabled});if(!y){var L;y={};for(L in g)g[L]instanceof Array?y[L]=g[L].slice(0):y[L]=g[L]}if(!d||!d.length||!d.filter(function(e){return e.values.length}).length){var A=T.selectAll(".nv-noData").data([b]);return A.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),A.attr("x",s.left+C/2).attr("y",s.top+k/2).text(function(e){return e}),x}T.selectAll(".nv-noData").remove(),v=t.xScale(),m=t.yScale();var O=T.selectAll("g.nv-wrap.nv-historicalBarChart").data([d]),M=O.enter().append("g").attr("class","nvd3 nv-wrap nv-historicalBarChart").append("g"),_=O.select("g");M.append("g").attr("class","nv-x nv-axis"),M.append("g").attr("class","nv-y nv-axis"),M.append("g").attr("class","nv-barsWrap"),M.append("g").attr("class","nv-legendWrap"),f&&(i.width(C),_.select(".nv-legendWrap").datum(d).call(i),s.top!=i.height()&&(s.top=i.height(),k=(a||parseInt(T.style("height"))||400)-s.top-s.bottom),O.select(".nv-legendWrap").attr("transform","translate(0,"+ -s.top+")")),O.attr("transform","translate("+s.left+","+s.top+")"),h&&_.select(".nv-y.nv-axis").attr("transform","translate("+C+",0)"),t.width(C).height(k).color(d.map(function(e,t){return e.color||o(e,t)}).filter(function(e,t){return!d[t].disabled}));var D=_.select(".nv-barsWrap").datum(d.filter(function(e){return!e.disabled}));D.transition().call(t),l&&(n.scale(v).tickSize(-k,0),_.select(".nv-x.nv-axis").attr("transform","translate(0,"+m.range()[0]+")"),_.select(".nv-x.nv-axis").transition().call(n)),c&&(r.scale(m).ticks(k/36).tickSize(-C,0),_.select(".nv-y.nv-axis").transition().call(r)),i.dispatch.on("legendClick",function(t,n){t.disabled=!t.disabled,d.filter(function(e){return!e.disabled}).length||d.map(function(e){return e.disabled=!1,O.selectAll(".nv-series").classed("disabled",!1),e}),g.disabled=d.map(function(e){return!!e.disabled}),w.stateChange(g),e.transition().call(x)}),i.dispatch.on("legendDblclick",function(e){d.forEach(function(e){e.disabled=!0}),e.disabled=!1,g.disabled=d.map(function(e){return!!e.disabled}),w.stateChange(g),x.update()}),w.on("tooltipShow",function(e){p&&S(e,N.parentNode)}),w.on("changeState",function(e){typeof e.disabled!="undefined"&&(d.forEach(function(t,n){t.disabled=e.disabled[n]}),g.disabled=e.disabled),x.update()})}),x}var t=e.models.historicalBar(),n=e.models.axis(),r=e.models.axis(),i=e.models.legend(),s={top:30,right:90,bottom:50,left:90},o=e.utils.defaultColor(),u=null,a=null,f=!1,l=!0,c=!0,h=!1,p=!0,d=function(e,t,n,r,i){return"

    "+e+"

    "+"

    "+n+" at "+t+"

    "},v,m,g={},y=null,b="No Data Available.",w=d3.dispatch("tooltipShow","tooltipHide","stateChange","changeState"),E=250;n.orient("bottom").tickPadding(7),r.orient(h?"right":"left");var S=function(i,s){if(s){var o=d3.select(s).select("svg"),u=o.node()?o.attr("viewBox"):null;if(u){u=u.split(" ");var a=parseInt(o.style("width"))/u[2];i.pos[0]=i.pos[0]*a,i.pos[1]=i.pos[1]*a}}var f=i.pos[0]+(s.offsetLeft||0),l=i.pos[1]+(s.offsetTop||0),c=n.tickFormat()(t.x()(i.point,i.pointIndex)),h=r.tickFormat()(t.y()(i.point,i.pointIndex)),p=d(i.series.key,c,h,i,x);e.tooltip.show([f,l],p,null,null,s)};return t.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+s.left,e.pos[1]+s.top],w.tooltipShow(e)}),t.dispatch.on("elementMouseout.tooltip",function(e){w.tooltipHide(e)}),w.on("tooltipHide",function(){p&&e.tooltip.cleanup()}),x.dispatch=w,x.bars=t,x.legend=i,x.xAxis=n,x.yAxis=r,d3.rebind(x,t,"defined","isArea","x","y","size","xScale","yScale","xDomain","yDomain","xRange","yRange","forceX","forceY","interactive","clipEdge","clipVoronoi","id","interpolate","highlightPoint","clearHighlights","interactive"),x.options=e.utils.optionsFunc.bind(x),x.margin=function(e){return arguments.length?(s.top=typeof e.top!="undefined"?e.top:s.top,s.right=typeof e.right!="undefined"?e.right:s.right,s.bottom=typeof e.bottom!="undefined"?e.bottom:s.bottom,s.left=typeof e.left!="undefined"?e.left:s.left,x):s},x.width=function(e){return arguments.length?(u=e,x):u},x.height=function(e){return arguments.length?(a=e,x):a},x.color=function(t){return arguments.length?(o=e.utils.getColor(t),i.color(o),x):o},x.showLegend=function(e){return arguments.length?(f=e,x):f},x.showXAxis=function(e){return arguments.length?(l=e,x):l},x.showYAxis=function(e){return arguments.length?(c=e,x):c},x.rightAlignYAxis=function(e){return arguments.length?(h=e,r.orient(e?"right":"left"),x):h},x.tooltips=function(e){return arguments.length?(p=e,x):p},x.tooltipContent=function(e){return arguments.length?(d=e,x):d},x.state=function(e){return arguments.length?(g=e,x):g},x.defaultState=function(e){return arguments.length?(y=e,x):y},x.noData=function(e){return arguments.length?(b=e,x):b},x.transitionDuration=function(e){return arguments.length?(E=e,x):E},x},e.models.indentedTree=function(){"use strict";function g(e){return e.each(function(e){function k(e,t,n){d3.event.stopPropagation();if(d3.event.shiftKey&&!n)return d3.event.shiftKey=!1,e.values&&e.values.forEach(function(e){(e.values||e._values)&&k(e,0,!0)}),!0;if(!O(e))return!0;e.values?(e._values=e.values,e.values=null):(e.values=e._values,e._values=null),g.update()}function L(e){return e._values&&e._values.length?h:e.values&&e.values.length?p:""}function A(e){return e._values&&e._values.length}function O(e){var t=e.values||e._values;return t&&t.length}var t=1,n=d3.select(this),i=d3.layout.tree().children(function(e){return e.values}).size([r,f]);g.update=function(){n.transition().duration(600).call(g)},e[0]||(e[0]={key:a});var s=i.nodes(e[0]),y=d3.select(this).selectAll("div").data([[s]]),b=y.enter().append("div").attr("class","nvd3 nv-wrap nv-indentedtree"),w=b.append("table"),E=y.select("table").attr("width","100%").attr("class",c);if(o){var S=w.append("thead"),x=S.append("tr");l.forEach(function(e){x.append("th").attr("width",e.width?e.width:"10%").style("text-align",e.type=="numeric"?"right":"left").append("span").text(e.label)})}var T=E.selectAll("tbody").data(function(e){return e});T.enter().append("tbody"),t=d3.max(s,function(e){return e.depth}),i.size([r,t*f]);var N=T.selectAll("tr").data(function(e){return e.filter(function(e){return u&&!e.children?u(e):!0})},function(e,t){return e.id||e.id||++m});N.exit().remove(),N.select("img.nv-treeicon").attr("src",L).classed("folded",A);var C=N.enter().append("tr");l.forEach(function(e,t){var n=C.append("td").style("padding-left",function(e){return(t?0:e.depth*f+12+(L(e)?0:16))+"px"},"important").style("text-align",e.type=="numeric"?"right":"left");t==0&&n.append("img").classed("nv-treeicon",!0).classed("nv-folded",A).attr("src",L).style("width","14px").style("height","14px").style("padding","0 1px").style("display",function(e){return L(e)?"inline-block":"none"}).on("click",k),n.each(function(n){!t&&v(n)?d3.select(this).append("a").attr("href",v).attr("class",d3.functor(e.classes)).append("span"):d3.select(this).append("span"),d3.select(this).select("span").attr("class",d3.functor(e.classes)).text(function(t){return e.format?e.format(t):t[e.key]||"-"})}),e.showCount&&(n.append("span").attr("class","nv-childrenCount"),N.selectAll("span.nv-childrenCount").text(function(e){return e.values&&e.values.length||e._values&&e._values.length?"("+(e.values&&e.values.filter(function(e){return u?u(e):!0}).length||e._values&&e._values.filter(function(e){return u?u(e):!0}).length||0)+")":""}))}),N.order().on("click",function(e){d.elementClick({row:this,data:e,pos:[e.x,e.y]})}).on("dblclick",function(e){d.elementDblclick({row:this,data:e,pos:[e.x,e.y]})}).on("mouseover",function(e){d.elementMouseover({row:this,data:e,pos:[e.x,e.y]})}).on("mouseout",function(e){d.elementMouseout({row:this,data:e,pos:[e.x,e.y]})})}),g}var t={top:0,right:0,bottom:0,left:0},n=960,r=500,i=e.utils.defaultColor(),s=Math.floor(Math.random()*1e4),o=!0,u=!1,a="No Data Available.",f=20,l=[{key:"key",label:"Name",type:"text"}],c=null,h="images/grey-plus.png",p="images/grey-minus.png",d=d3.dispatch("elementClick","elementDblclick","elementMouseover","elementMouseout"),v=function(e){return e.url},m=0;return g.options=e.utils.optionsFunc.bind(g),g.margin=function(e){return arguments.length?(t.top=typeof e.top!="undefined"?e.top:t.top,t.right=typeof e.right!="undefined"?e.right:t.right,t.bottom=typeof e.bottom!="undefined"?e.bottom:t.bottom,t.left=typeof e.left!="undefined"?e.left:t.left,g):t},g.width=function(e){return arguments.length?(n=e,g):n},g.height=function(e){return arguments.length?(r=e,g):r},g.color=function(t){return arguments.length?(i=e.utils.getColor(t),scatter.color(i),g):i},g.id=function(e){return arguments.length?(s=e,g):s},g.header=function(e){return arguments.length?(o=e,g):o},g.noData=function(e){return arguments.length?(a=e,g):a},g.filterZero=function(e){return arguments.length?(u=e,g):u},g.columns=function(e){return arguments.length?(l=e,g):l},g.tableClass=function(e){return arguments.length?(c=e,g):c},g.iconOpen=function(e){return arguments.length?(h=e,g):h},g.iconClose=function(e){return arguments.length?(p=e,g):p},g.getUrl=function(e){return arguments.length?(v=e,g):v},g},e.models.legend=function(){"use strict";function c(h){return h.each(function(c){var h=n-t.left-t.right,p=d3.select(this),d=p.selectAll("g.nv-legend").data([c]),v=d.enter().append("g").attr("class","nvd3 nv-legend").append("g"),m=d.select("g");d.attr("transform","translate("+t.left+","+t.top+")");var g=m.selectAll(".nv-series").data(function(e){return e}),y=g.enter().append("g").attr("class","nv-series").on("mouseover",function(e,t){l.legendMouseover(e,t)}).on("mouseout",function(e,t){l.legendMouseout(e,t)}).on("click",function(e,t){l.legendClick(e,t),a&&(f?(c.forEach(function(e){e.disabled=!0}),e.disabled=!1):(e.disabled=!e.disabled,c.every(function(e){return e.disabled})&&c.forEach(function(e){e.disabled=!1})),l.stateChange({disabled:c.map(function(e){return!!e.disabled})}))}).on("dblclick",function(e,t){l.legendDblclick(e,t),a&&(c.forEach(function(e){e.disabled=!0}),e.disabled=!1,l.stateChange({disabled:c.map(function(e){return!!e.disabled})}))});y.append("circle").style("stroke-width",2).attr("class","nv-legend-symbol").attr("r",5),y.append("text").attr("text-anchor","start").attr("class","nv-legend-text").attr("dy",".32em").attr("dx","8"),g.classed("disabled",function(e){return e.disabled}),g.exit().remove(),g.select("circle").style("fill",function(e,t){return e.color||s(e,t)}).style("stroke",function(e,t){return e.color||s(e,t)}),g.select("text").text(i);if(o){var b=[];g.each(function(t,n){var r=d3.select(this).select("text"),i;try{i=r.getComputedTextLength();if(i<=0)throw Error()}catch(s){i=e.utils.calcApproxTextWidth(r)}b.push(i+28)});var w=0,E=0,S=[];while(Eh&&w>1){S=[],w--;for(var x=0;x(S[x%w]||0)&&(S[x%w]=b[x]);E=S.reduce(function(e,t,n,r){return e+t})}var T=[];for(var N=0,C=0;NA&&(A=L),"translate("+O+","+k+")"}),m.attr("transform","translate("+(n-t.right-A)+","+t.top+")"),r=t.top+t.bottom+k+15}}),c}var t={top:5,right:0,bottom:5,left:0},n=400,r=20,i=function(e){return e.key},s=e.utils.defaultColor(),o=!0,u=!0,a=!0,f=!1,l=d3.dispatch("legendClick","legendDblclick","legendMouseover","legendMouseout","stateChange");return c.dispatch=l,c.options=e.utils.optionsFunc.bind(c),c.margin=function(e){return arguments.length?(t.top=typeof e.top!="undefined"?e.top:t.top,t.right=typeof e.right!="undefined"?e.right:t.right,t.bottom=typeof e.bottom!="undefined"?e.bottom:t.bottom,t.left=typeof e.left!="undefined"?e.left:t.left,c):t},c.width=function(e){return arguments.length?(n=e,c):n},c.height=function(e){return arguments.length?(r=e,c):r},c.key=function(e){return arguments.length?(i=e,c):i},c.color=function(t){return arguments.length?(s=e.utils.getColor(t),c):s},c.align=function(e){return arguments.length?(o=e,c):o},c.rightAlign=function(e){return arguments.length?(u=e,c):u},c.updateState=function(e){return arguments.length?(a=e,c):a},c.radioButtonMode=function(e){return arguments.length?(f=e,c):f},c},e.models.line=function(){"use strict";function m(g){return g.each(function(m){var g=r-n.left-n.right,b=i-n.top-n.bottom,w=d3.select(this);c=t.xScale(),h=t.yScale(),d=d||c,v=v||h;var E=w.selectAll("g.nv-wrap.nv-line").data([m]),S=E.enter().append("g").attr("class","nvd3 nv-wrap nv-line"),T=S.append("defs"),N=S.append("g"),C=E.select("g");N.append("g").attr("class","nv-groups"),N.append("g").attr("class","nv-scatterWrap"),E.attr("transform","translate("+n.left+","+n.top+")"),t.width(g).height(b);var k=E.select(".nv-scatterWrap");k.transition().call(t),T.append("clipPath").attr("id","nv-edge-clip-"+t.id()).append("rect"),E.select("#nv-edge-clip-"+t.id()+" rect").attr("width",g).attr("height",b),C.attr("clip-path",l?"url(#nv-edge-clip-"+t.id()+")":""),k.attr("clip-path",l?"url(#nv-edge-clip-"+t.id()+")":"");var L=E.select(".nv-groups").selectAll(".nv-group").data(function(e){return e},function(e){return e.key});L.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6),L.exit().remove(),L.attr("class",function(e,t){return"nv-group nv-series-"+t}).classed("hover",function(e){return e.hover}).style("fill",function(e,t){return s(e,t)}).style("stroke",function(e,t){return s(e,t)}),L.transition().style("stroke-opacity",1).style("fill-opacity",.5);var A=L.selectAll("path.nv-area").data(function(e){return f(e)?[e]:[]});A.enter().append("path").attr("class","nv-area").attr("d",function(t){return d3.svg.area().interpolate(p).defined(a).x(function(t,n){return e. +utils.NaNtoZero(d(o(t,n)))}).y0(function(t,n){return e.utils.NaNtoZero(v(u(t,n)))}).y1(function(e,t){return v(h.domain()[0]<=0?h.domain()[1]>=0?0:h.domain()[1]:h.domain()[0])}).apply(this,[t.values])}),L.exit().selectAll("path.nv-area").remove(),A.transition().attr("d",function(t){return d3.svg.area().interpolate(p).defined(a).x(function(t,n){return e.utils.NaNtoZero(c(o(t,n)))}).y0(function(t,n){return e.utils.NaNtoZero(h(u(t,n)))}).y1(function(e,t){return h(h.domain()[0]<=0?h.domain()[1]>=0?0:h.domain()[1]:h.domain()[0])}).apply(this,[t.values])});var O=L.selectAll("path.nv-line").data(function(e){return[e.values]});O.enter().append("path").attr("class","nv-line").attr("d",d3.svg.line().interpolate(p).defined(a).x(function(t,n){return e.utils.NaNtoZero(d(o(t,n)))}).y(function(t,n){return e.utils.NaNtoZero(v(u(t,n)))})),O.transition().attr("d",d3.svg.line().interpolate(p).defined(a).x(function(t,n){return e.utils.NaNtoZero(c(o(t,n)))}).y(function(t,n){return e.utils.NaNtoZero(h(u(t,n)))})),d=c.copy(),v=h.copy()}),m}var t=e.models.scatter(),n={top:0,right:0,bottom:0,left:0},r=960,i=500,s=e.utils.defaultColor(),o=function(e){return e.x},u=function(e){return e.y},a=function(e,t){return!isNaN(u(e,t))&&u(e,t)!==null},f=function(e){return e.area},l=!1,c,h,p="linear";t.size(16).sizeDomain([16,256]);var d,v;return m.dispatch=t.dispatch,m.scatter=t,d3.rebind(m,t,"id","interactive","size","xScale","yScale","zScale","xDomain","yDomain","xRange","yRange","sizeDomain","forceX","forceY","forceSize","clipVoronoi","useVoronoi","clipRadius","padData","highlightPoint","clearHighlights"),m.options=e.utils.optionsFunc.bind(m),m.margin=function(e){return arguments.length?(n.top=typeof e.top!="undefined"?e.top:n.top,n.right=typeof e.right!="undefined"?e.right:n.right,n.bottom=typeof e.bottom!="undefined"?e.bottom:n.bottom,n.left=typeof e.left!="undefined"?e.left:n.left,m):n},m.width=function(e){return arguments.length?(r=e,m):r},m.height=function(e){return arguments.length?(i=e,m):i},m.x=function(e){return arguments.length?(o=e,t.x(e),m):o},m.y=function(e){return arguments.length?(u=e,t.y(e),m):u},m.clipEdge=function(e){return arguments.length?(l=e,m):l},m.color=function(n){return arguments.length?(s=e.utils.getColor(n),t.color(s),m):s},m.interpolate=function(e){return arguments.length?(p=e,m):p},m.defined=function(e){return arguments.length?(a=e,m):a},m.isArea=function(e){return arguments.length?(f=d3.functor(e),m):f},m},e.models.lineChart=function(){"use strict";function N(m){return m.each(function(m){var C=d3.select(this),k=this,L=(a||parseInt(C.style("width"))||960)-o.left-o.right,A=(f||parseInt(C.style("height"))||400)-o.top-o.bottom;N.update=function(){C.transition().duration(x).call(N)},N.container=this,b.disabled=m.map(function(e){return!!e.disabled});if(!w){var O;w={};for(O in b)b[O]instanceof Array?w[O]=b[O].slice(0):w[O]=b[O]}if(!m||!m.length||!m.filter(function(e){return e.values.length}).length){var M=C.selectAll(".nv-noData").data([E]);return M.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),M.attr("x",o.left+L/2).attr("y",o.top+A/2).text(function(e){return e}),N}C.selectAll(".nv-noData").remove(),g=t.xScale(),y=t.yScale();var _=C.selectAll("g.nv-wrap.nv-lineChart").data([m]),D=_.enter().append("g").attr("class","nvd3 nv-wrap nv-lineChart").append("g"),P=_.select("g");D.append("rect").style("opacity",0),D.append("g").attr("class","nv-x nv-axis"),D.append("g").attr("class","nv-y nv-axis"),D.append("g").attr("class","nv-linesWrap"),D.append("g").attr("class","nv-legendWrap"),D.append("g").attr("class","nv-interactive"),P.select("rect").attr("width",L).attr("height",A>0?A:0),l&&(i.width(L),P.select(".nv-legendWrap").datum(m).call(i),o.top!=i.height()&&(o.top=i.height(),A=(f||parseInt(C.style("height"))||400)-o.top-o.bottom),_.select(".nv-legendWrap").attr("transform","translate(0,"+ -o.top+")")),_.attr("transform","translate("+o.left+","+o.top+")"),p&&P.select(".nv-y.nv-axis").attr("transform","translate("+L+",0)"),d&&(s.width(L).height(A).margin({left:o.left,top:o.top}).svgContainer(C).xScale(g),_.select(".nv-interactive").call(s)),t.width(L).height(A).color(m.map(function(e,t){return e.color||u(e,t)}).filter(function(e,t){return!m[t].disabled}));var H=P.select(".nv-linesWrap").datum(m.filter(function(e){return!e.disabled}));H.transition().call(t),c&&(n.scale(g).ticks(L/100).tickSize(-A,0),P.select(".nv-x.nv-axis").attr("transform","translate(0,"+y.range()[0]+")"),P.select(".nv-x.nv-axis").transition().call(n)),h&&(r.scale(y).ticks(A/36).tickSize(-L,0),P.select(".nv-y.nv-axis").transition().call(r)),i.dispatch.on("stateChange",function(e){b=e,S.stateChange(b),N.update()}),s.dispatch.on("elementMousemove",function(i){t.clearHighlights();var a,f,l,c=[];m.filter(function(e,t){return e.seriesIndex=t,!e.disabled}).forEach(function(n,r){f=e.interactiveBisect(n.values,i.pointXValue,N.x()),t.highlightPoint(r,f,!0);var s=n.values[f];if(typeof s=="undefined")return;typeof a=="undefined"&&(a=s),typeof l=="undefined"&&(l=N.xScale()(N.x()(s,f))),c.push({key:n.key,value:N.y()(s,f),color:u(n,n.seriesIndex)})});if(c.length>2){var h=N.yScale().invert(i.mouseY),p=Math.abs(N.yScale().domain()[0]-N.yScale().domain()[1]),d=.03*p,g=e.nearestValueIndex(c.map(function(e){return e.value}),h,d);g!==null&&(c[g].highlight=!0)}var y=n.tickFormat()(N.x()(a,f));s.tooltip.position({left:l+o.left,top:i.mouseY+o.top}).chartContainer(k.parentNode).enabled(v).valueFormatter(function(e,t){return r.tickFormat()(e)}).data({value:y,series:c})(),s.renderGuideLine(l)}),s.dispatch.on("elementMouseout",function(e){S.tooltipHide(),t.clearHighlights()}),S.on("tooltipShow",function(e){v&&T(e,k.parentNode)}),S.on("changeState",function(e){typeof e.disabled!="undefined"&&m.length===e.disabled.length&&(m.forEach(function(t,n){t.disabled=e.disabled[n]}),b.disabled=e.disabled),N.update()})}),N}var t=e.models.line(),n=e.models.axis(),r=e.models.axis(),i=e.models.legend(),s=e.interactiveGuideline(),o={top:30,right:20,bottom:50,left:60},u=e.utils.defaultColor(),a=null,f=null,l=!0,c=!0,h=!0,p=!1,d=!1,v=!0,m=function(e,t,n,r,i){return"

    "+e+"

    "+"

    "+n+" at "+t+"

    "},g,y,b={},w=null,E="No Data Available.",S=d3.dispatch("tooltipShow","tooltipHide","stateChange","changeState"),x=250;n.orient("bottom").tickPadding(7),r.orient(p?"right":"left");var T=function(i,s){var o=i.pos[0]+(s.offsetLeft||0),u=i.pos[1]+(s.offsetTop||0),a=n.tickFormat()(t.x()(i.point,i.pointIndex)),f=r.tickFormat()(t.y()(i.point,i.pointIndex)),l=m(i.series.key,a,f,i,N);e.tooltip.show([o,u],l,null,null,s)};return t.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+o.left,e.pos[1]+o.top],S.tooltipShow(e)}),t.dispatch.on("elementMouseout.tooltip",function(e){S.tooltipHide(e)}),S.on("tooltipHide",function(){v&&e.tooltip.cleanup()}),N.dispatch=S,N.lines=t,N.legend=i,N.xAxis=n,N.yAxis=r,N.interactiveLayer=s,d3.rebind(N,t,"defined","isArea","x","y","size","xScale","yScale","xDomain","yDomain","xRange","yRange","forceX","forceY","interactive","clipEdge","clipVoronoi","useVoronoi","id","interpolate"),N.options=e.utils.optionsFunc.bind(N),N.margin=function(e){return arguments.length?(o.top=typeof e.top!="undefined"?e.top:o.top,o.right=typeof e.right!="undefined"?e.right:o.right,o.bottom=typeof e.bottom!="undefined"?e.bottom:o.bottom,o.left=typeof e.left!="undefined"?e.left:o.left,N):o},N.width=function(e){return arguments.length?(a=e,N):a},N.height=function(e){return arguments.length?(f=e,N):f},N.color=function(t){return arguments.length?(u=e.utils.getColor(t),i.color(u),N):u},N.showLegend=function(e){return arguments.length?(l=e,N):l},N.showXAxis=function(e){return arguments.length?(c=e,N):c},N.showYAxis=function(e){return arguments.length?(h=e,N):h},N.rightAlignYAxis=function(e){return arguments.length?(p=e,r.orient(e?"right":"left"),N):p},N.useInteractiveGuideline=function(e){return arguments.length?(d=e,e===!0&&(N.interactive(!1),N.useVoronoi(!1)),N):d},N.tooltips=function(e){return arguments.length?(v=e,N):v},N.tooltipContent=function(e){return arguments.length?(m=e,N):m},N.state=function(e){return arguments.length?(b=e,N):b},N.defaultState=function(e){return arguments.length?(w=e,N):w},N.noData=function(e){return arguments.length?(E=e,N):E},N.transitionDuration=function(e){return arguments.length?(x=e,N):x},N},e.models.linePlusBarChart=function(){"use strict";function T(e){return e.each(function(e){var l=d3.select(this),c=this,v=(a||parseInt(l.style("width"))||960)-u.left-u.right,N=(f||parseInt(l.style("height"))||400)-u.top-u.bottom;T.update=function(){l.transition().call(T)},b.disabled=e.map(function(e){return!!e.disabled});if(!w){var C;w={};for(C in b)b[C]instanceof Array?w[C]=b[C].slice(0):w[C]=b[C]}if(!e||!e.length||!e.filter(function(e){return e.values.length}).length){var k=l.selectAll(".nv-noData").data([E]);return k.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),k.attr("x",u.left+v/2).attr("y",u.top+N/2).text(function(e){return e}),T}l.selectAll(".nv-noData").remove();var L=e.filter(function(e){return!e.disabled&&e.bar}),A=e.filter(function(e){return!e.bar});m=A.filter(function(e){return!e.disabled}).length&&A.filter(function(e){return!e.disabled})[0].values.length?t.xScale():n.xScale(),g=n.yScale(),y=t.yScale();var O=d3.select(this).selectAll("g.nv-wrap.nv-linePlusBar").data([e]),M=O.enter().append("g").attr("class","nvd3 nv-wrap nv-linePlusBar").append("g"),_=O.select("g");M.append("g").attr("class","nv-x nv-axis"),M.append("g").attr("class","nv-y1 nv-axis"),M.append("g").attr("class","nv-y2 nv-axis"),M.append("g").attr("class","nv-barsWrap"),M.append("g").attr("class","nv-linesWrap"),M.append("g").attr("class","nv-legendWrap"),p&&(o.width(v/2),_.select(".nv-legendWrap").datum(e.map(function(e){return e.originalKey=e.originalKey===undefined?e.key:e.originalKey,e.key=e.originalKey+(e.bar?" (left axis)":" (right axis)"),e})).call(o),u.top!=o.height()&&(u.top=o.height(),N=(f||parseInt(l.style("height"))||400)-u.top-u.bottom),_.select(".nv-legendWrap").attr("transform","translate("+v/2+","+ -u.top+")")),O.attr("transform","translate("+u.left+","+u.top+")"),t.width(v).height(N).color(e.map(function(e,t){return e.color||h(e,t)}).filter(function(t,n){return!e[n].disabled&&!e[n].bar})),n.width(v).height(N).color(e.map(function(e,t){return e.color||h(e,t)}).filter(function(t,n){return!e[n].disabled&&e[n].bar}));var D=_.select(".nv-barsWrap").datum(L.length?L:[{values:[]}]),P=_.select(".nv-linesWrap").datum(A[0]&&!A[0].disabled?A:[{values:[]}]);d3.transition(D).call(n),d3.transition(P).call(t),r.scale(m).ticks(v/100).tickSize(-N,0),_.select(".nv-x.nv-axis").attr("transform","translate(0,"+g.range()[0]+")"),d3.transition(_.select(".nv-x.nv-axis")).call(r),i.scale(g).ticks(N/36).tickSize(-v,0),d3.transition(_.select(".nv-y1.nv-axis")).style("opacity",L.length?1:0).call(i),s.scale(y).ticks(N/36).tickSize(L.length?0:-v,0),_.select(".nv-y2.nv-axis").style("opacity",A.length?1:0).attr("transform","translate("+v+",0)"),d3.transition(_.select(".nv-y2.nv-axis")).call(s),o.dispatch.on("stateChange",function(e){b=e,S.stateChange(b),T.update()}),S.on("tooltipShow",function(e){d&&x(e,c.parentNode)}),S.on("changeState",function(t){typeof t.disabled!="undefined"&&(e.forEach(function(e,n){e.disabled=t.disabled[n]}),b.disabled=t.disabled),T.update()})}),T}var t=e.models.line(),n=e.models.historicalBar(),r=e.models.axis(),i=e.models.axis(),s=e.models.axis(),o=e.models.legend(),u={top:30,right:60,bottom:50,left:60},a=null,f=null,l=function(e){return e.x},c=function(e){return e.y},h=e.utils.defaultColor(),p=!0,d=!0,v=function(e,t,n,r,i){return"

    "+e+"

    "+"

    "+n+" at "+t+"

    "},m,g,y,b={},w=null,E="No Data Available.",S=d3.dispatch("tooltipShow","tooltipHide","stateChange","changeState");n.padData(!0),t.clipEdge(!1).padData(!0),r.orient("bottom").tickPadding(7).highlightZero(!1),i.orient("left"),s.orient("right");var x=function(n,o){var u=n.pos[0]+(o.offsetLeft||0),a=n.pos[1]+(o.offsetTop||0),f=r.tickFormat()(t.x()(n.point,n.pointIndex)),l=(n.series.bar?i:s).tickFormat()(t.y()(n.point,n.pointIndex)),c=v(n.series.key,f,l,n,T);e.tooltip.show([u,a],c,n.value<0?"n":"s",null,o)};return t.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+u.left,e.pos[1]+u.top],S.tooltipShow(e)}),t.dispatch.on("elementMouseout.tooltip",function(e){S.tooltipHide(e)}),n.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+u.left,e.pos[1]+u.top],S.tooltipShow(e)}),n.dispatch.on("elementMouseout.tooltip",function(e){S.tooltipHide(e)}),S.on("tooltipHide",function(){d&&e.tooltip.cleanup()}),T.dispatch=S,T.legend=o,T.lines=t,T.bars=n,T.xAxis=r,T.y1Axis=i,T.y2Axis=s,d3.rebind(T,t,"defined","size","clipVoronoi","interpolate"),T.options=e.utils.optionsFunc.bind(T),T.x=function(e){return arguments.length?(l=e,t.x(e),n.x(e),T):l},T.y=function(e){return arguments.length?(c=e,t.y(e),n.y(e),T):c},T.margin=function(e){return arguments.length?(u.top=typeof e.top!="undefined"?e.top:u.top,u.right=typeof e.right!="undefined"?e.right:u.right,u.bottom=typeof e.bottom!="undefined"?e.bottom:u.bottom,u.left=typeof e.left!="undefined"?e.left:u.left,T):u},T.width=function(e){return arguments.length?(a=e,T):a},T.height=function(e){return arguments.length?(f=e,T):f},T.color=function(t){return arguments.length?(h=e.utils.getColor(t),o.color(h),T):h},T.showLegend=function(e){return arguments.length?(p=e,T):p},T.tooltips=function(e){return arguments.length?(d=e,T):d},T.tooltipContent=function(e){return arguments.length?(v=e,T):v},T.state=function(e){return arguments.length?(b=e,T):b},T.defaultState=function(e){return arguments.length?(w=e,T):w},T.noData=function(e){return arguments.length?(E=e,T):E},T},e.models.lineWithFocusChart=function(){"use strict";function k(e){return e.each(function(e){function U(e){var t=+(e=="e"),n=t?1:-1,r=M/3;return"M"+.5*n+","+r+"A6,6 0 0 "+t+" "+6.5*n+","+(r+6)+"V"+(2*r-6)+"A6,6 0 0 "+t+" "+.5*n+","+2*r+"Z"+"M"+2.5*n+","+(r+8)+"V"+(2*r-8)+"M"+4.5*n+","+(r+8)+"V"+(2*r-8)}function z(){a.empty()||a.extent(w),I.data([a.empty()?g.domain():w]).each(function(e,t){var n=g(e[0])-v.range()[0],r=v.range()[1]-g(e[1]);d3.select(this).select(".left").attr("width",n<0?0:n),d3.select(this).select(".right").attr("x",g(e[1])).attr("width",r<0?0:r)})}function W(){w=a.empty()?null:a.extent();var n=a.empty()?g.domain():a.extent();if(Math.abs(n[0]-n[1])<=1)return;T.brush({extent:n,brush:a}),z();var s=H.select(".nv-focus .nv-linesWrap").datum(e.filter(function(e){return!e.disabled}).map(function(e,r){return{key:e.key,values:e.values.filter(function(e,r){return t.x()(e,r)>=n[0]&&t.x()(e,r)<=n[1]})}}));s.transition().duration(N).call(t),H.select(".nv-focus .nv-x.nv-axis").transition().duration(N).call(r),H.select(".nv-focus .nv-y.nv-axis").transition().duration(N).call(i)}var S=d3.select(this),L=this,A=(h||parseInt(S.style("width"))||960)-f.left-f.right,O=(p||parseInt(S.style("height"))||400)-f.top-f.bottom-d,M=d-l.top-l.bottom;k.update=function(){S.transition().duration(N).call(k)},k.container=this;if(!e||!e.length||!e.filter(function(e){return e.values.length}).length){var _=S.selectAll(".nv-noData").data([x]);return _.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),_.attr("x",f.left+A/2).attr("y",f.top+O/2).text(function(e){return e}),k}S.selectAll(".nv-noData").remove(),v=t.xScale(),m=t.yScale(),g=n.xScale(),y=n.yScale();var D=S.selectAll("g.nv-wrap.nv-lineWithFocusChart").data([e]),P=D.enter().append("g").attr("class","nvd3 nv-wrap nv-lineWithFocusChart").append("g"),H=D.select("g");P.append("g").attr("class","nv-legendWrap");var B=P.append("g").attr("class","nv-focus");B.append("g").attr("class","nv-x nv-axis"),B.append("g").attr("class","nv-y nv-axis"),B.append("g").attr("class","nv-linesWrap");var j=P.append("g").attr("class","nv-context");j.append("g").attr("class","nv-x nv-axis"),j.append("g").attr("class","nv-y nv-axis"),j.append("g").attr("class","nv-linesWrap"),j.append("g").attr("class","nv-brushBackground"),j.append("g").attr("class","nv-x nv-brush"),b&&(u.width(A),H.select(".nv-legendWrap").datum(e).call(u),f.top!=u.height()&&(f.top=u.height(),O=(p||parseInt(S.style("height"))||400)-f.top-f.bottom-d),H.select(".nv-legendWrap").attr("transform","translate(0,"+ -f.top+")")),D.attr("transform","translate("+f.left+","+f.top+")"),t.width(A).height(O).color(e.map(function(e,t){return e.color||c(e,t)}).filter(function(t,n){return!e[n].disabled})),n.defined(t.defined()).width(A).height(M).color(e.map(function(e,t){return e.color||c(e,t)}).filter(function(t,n){return!e[n].disabled})),H.select(".nv-context").attr("transform","translate(0,"+(O+f.bottom+l.top)+")");var F=H.select(".nv-context .nv-linesWrap").datum(e.filter(function(e){return!e.disabled}));d3.transition(F).call(n),r.scale(v).ticks(A/100).tickSize(-O,0),i.scale(m).ticks(O/36).tickSize(-A,0),H.select(".nv-focus .nv-x.nv-axis").attr("transform","translate(0,"+O+")"),a.x(g).on("brush",function(){var e=k.transitionDuration();k.transitionDuration(0),W(),k.transitionDuration(e)}),w&&a.extent(w);var I=H.select(".nv-brushBackground").selectAll("g").data([w||a.extent()]),q=I.enter().append("g");q.append("rect").attr("class","left").attr("x",0).attr("y",0).attr("height",M),q.append("rect").attr("class","right").attr("x",0).attr("y",0).attr("height",M);var R=H.select(".nv-x.nv-brush").call(a);R.selectAll("rect").attr("height",M),R.selectAll(".resize").append("path").attr("d",U),W(),s.scale(g).ticks(A/100).tickSize(-M,0),H.select(".nv-context .nv-x.nv-axis").attr("transform","translate(0,"+y.range()[0]+")"),d3.transition(H.select(".nv-context .nv-x.nv-axis")).call(s),o.scale(y).ticks(M/36).tickSize(-A,0),d3.transition(H.select(".nv-context .nv-y.nv-axis")).call(o),H.select(".nv-context .nv-x.nv-axis").attr("transform","translate(0,"+y.range()[0]+")"),u.dispatch.on("stateChange",function(e){k.update()}),T.on("tooltipShow",function(e){E&&C(e,L.parentNode)})}),k}var t=e.models.line(),n=e.models.line(),r=e.models.axis(),i=e.models.axis(),s=e.models.axis(),o=e.models.axis(),u=e.models.legend(),a=d3.svg.brush(),f={top:30,right:30,bottom:30,left:60},l={top:0,right:30,bottom:20,left:60},c=e.utils.defaultColor(),h=null,p=null,d=100,v,m,g,y,b=!0,w=null,E=!0,S=function(e,t,n,r,i){return"

    "+e+"

    "+"

    "+n+" at "+t+"

    "},x="No Data Available.",T=d3.dispatch("tooltipShow","tooltipHide","brush"),N=250;t.clipEdge(!0),n.interactive(!1),r.orient("bottom").tickPadding(5),i.orient("left"),s.orient("bottom").tickPadding(5),o.orient("left");var C=function(n,s){var o=n.pos[0]+(s.offsetLeft||0),u=n.pos[1]+(s.offsetTop||0),a=r.tickFormat()(t.x()(n.point,n.pointIndex)),f=i.tickFormat()(t.y()(n.point,n.pointIndex)),l=S(n.series.key,a,f,n,k);e.tooltip.show([o,u],l,null,null,s)};return t.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+f.left,e.pos[1]+f.top],T.tooltipShow(e)}),t.dispatch.on("elementMouseout.tooltip",function(e){T.tooltipHide(e)}),T.on("tooltipHide",function(){E&&e.tooltip.cleanup()}),k.dispatch=T,k.legend=u,k.lines=t,k.lines2=n,k.xAxis=r,k.yAxis=i,k.x2Axis=s,k.y2Axis=o,d3.rebind(k,t,"defined","isArea","size","xDomain","yDomain","xRange","yRange","forceX","forceY","interactive","clipEdge","clipVoronoi","id"),k.options=e.utils.optionsFunc.bind(k),k.x=function(e){return arguments.length?(t.x(e),n.x(e),k):t.x},k.y=function(e){return arguments.length?(t.y(e),n.y(e),k):t.y},k.margin=function(e){return arguments.length?(f.top=typeof e.top!="undefined"?e.top:f.top,f.right=typeof e.right!="undefined"?e.right:f.right,f.bottom=typeof e.bottom!="undefined"?e.bottom:f.bottom,f.left=typeof e.left!="undefined"?e.left:f.left,k):f},k.margin2=function(e){return arguments.length?(l=e,k):l},k.width=function(e){return arguments.length?(h=e,k):h},k.height=function(e){return arguments.length?(p=e,k):p},k.height2=function(e){return arguments.length?(d=e,k):d},k.color=function(t){return arguments.length?(c=e.utils.getColor(t),u.color(c),k):c},k.showLegend=function(e){return arguments.length?(b=e,k):b},k.tooltips=function(e){return arguments.length?(E=e,k):E},k.tooltipContent=function(e){return arguments.length?(S=e,k):S},k.interpolate=function(e){return arguments.length?(t.interpolate(e),n.interpolate(e),k):t.interpolate()},k.noData=function(e){return arguments.length?(x=e,k):x},k.xTickFormat=function(e){return arguments.length?(r.tickFormat(e),s.tickFormat(e),k):r.tickFormat()},k.yTickFormat=function(e){return arguments.length?(i.tickFormat(e),o.tickFormat(e),k):i.tickFormat()},k.brushExtent=function(e){return arguments.length?(w=e,k):w},k.transitionDuration=function(e){return arguments.length?(N=e,k):N},k},e.models.linePlusBarWithFocusChart=function(){"use strict";function B(e){return e.each(function(e){function nt(e){var t=+(e=="e"),n=t?1:-1,r=q/3;return"M"+.5*n+","+r+"A6,6 0 0 "+t+" "+6.5*n+","+(r+6)+"V"+(2*r-6)+"A6,6 0 0 "+t+" "+.5*n+","+2*r+"Z"+"M"+2.5*n+","+(r+8)+"V"+(2*r-8)+"M"+4.5*n+","+(r+8)+"V"+(2*r-8)}function rt(){h.empty()||h.extent(x),Z.data([h.empty()?k.domain():x]).each(function(e,t){var n=k(e[0])-k.range()[0],r=k.range()[1]-k(e[1]);d3.select(this).select(".left").attr("width",n<0?0:n),d3.select(this).select(".right").attr("x",k(e[1])).attr("width",r<0?0:r)})}function it(){x=h.empty()?null:h.extent(),S=h.empty()?k.domain():h.extent(),D.brush({extent:S,brush:h}),rt(),r.width(F).height(I).color(e.map(function(e,t){return e.color||w(e,t)}).filter(function(t,n){return!e[n].disabled&&e[n].bar})),t.width(F).height(I).color(e.map(function(e,t){return e.color||w(e,t)}).filter(function(t,n){return!e[n].disabled&&!e[n].bar}));var n=J.select(".nv-focus .nv-barsWrap").datum(U.length?U.map(function(e,t){return{key:e.key,values:e.values.filter(function(e,t){return r.x()(e,t)>=S[0]&&r.x()(e,t)<=S[1]})}}):[{values:[]}]),i=J.select(".nv-focus .nv-linesWrap").datum(z[0].disabled?[{values:[]}]:z.map(function(e,n){return{key:e.key,values:e.values.filter(function(e,n){return t.x()(e,n)>=S[0]&&t.x()(e,n)<=S[1]})}}));U.length?C=r.xScale():C=t.xScale(),s.scale(C).ticks(F/100).tickSize(-I,0),s.domain([Math.ceil(S[0]),Math.floor(S[1])]),J.select(".nv-x.nv-axis").transition().duration(P).call(s),n.transition().duration(P).call(r),i.transition().duration(P).call(t),J.select(".nv-focus .nv-x.nv-axis").attr("transform","translate(0,"+L.range()[0]+")"),u.scale(L).ticks(I/36).tickSize(-F,0),J.select(".nv-focus .nv-y1.nv-axis").style("opacity",U.length?1:0),a.scale(A).ticks(I/36).tickSize(U.length?0:-F,0),J.select(".nv-focus .nv-y2.nv-axis").style("opacity",z.length?1:0).attr("transform","translate("+C.range()[1]+",0)"),J.select(".nv-focus .nv-y1.nv-axis").transition().duration(P).call(u),J.select(".nv-focus .nv-y2.nv-axis").transition().duration(P).call(a)}var N=d3.select(this),j=this,F=(v||parseInt(N.style("width"))||960)-p.left-p.right,I=(m||parseInt(N.style("height"))||400)-p.top-p.bottom-g,q=g-d.top-d.bottom;B.update=function(){N.transition().duration(P).call(B)},B.container=this;if(!e||!e.length||!e.filter(function(e){return e.values.length}).length){var R=N.selectAll(".nv-noData").data([_]);return R.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),R.attr("x",p.left+F/2).attr("y",p.top+I/2).text(function(e){return e}),B}N.selectAll(".nv-noData").remove();var U=e.filter(function(e){return!e.disabled&&e.bar}),z=e.filter(function(e){return!e.bar});C=r.xScale(),k=o.scale(),L=r.yScale(),A=t.yScale(),O=i.yScale(),M=n.yScale();var W=e.filter(function(e){return!e.disabled&&e.bar}).map(function(e){return e.values.map(function(e,t){return{x:y(e,t),y:b(e,t)}})}),X=e.filter(function(e){return!e.disabled&&!e.bar}).map(function(e){return e.values.map(function(e,t){return{x:y(e,t),y:b(e,t)}})});C.range([0,F]),k.domain(d3.extent(d3.merge(W.concat(X)),function(e){return e.x})).range([0,F]);var V=N.selectAll("g.nv-wrap.nv-linePlusBar").data([e]),$=V.enter().append("g").attr("class","nvd3 nv-wrap nv-linePlusBar").append("g"),J=V.select("g");$.append("g").attr("class","nv-legendWrap");var K=$.append("g").attr("class","nv-focus");K.append("g").attr("class","nv-x nv-axis"),K.append("g").attr("class","nv-y1 nv-axis"),K.append("g").attr("class","nv-y2 nv-axis"),K.append("g").attr("class","nv-barsWrap"),K.append("g").attr("class","nv-linesWrap");var Q=$.append("g").attr("class","nv-context");Q.append("g").attr("class","nv-x nv-axis"),Q.append("g").attr("class","nv-y1 nv-axis"),Q.append("g").attr("class","nv-y2 nv-axis"),Q.append("g").attr("class","nv-barsWrap"),Q.append("g").attr("class","nv-linesWrap"),Q.append("g").attr("class","nv-brushBackground"),Q.append("g").attr("class","nv-x nv-brush"),E&&(c.width(F/2),J.select(".nv-legendWrap").datum(e.map(function(e){return e.originalKey=e.originalKey===undefined?e.key:e.originalKey,e.key=e.originalKey+(e.bar?" (left axis)":" (right axis)"),e})).call(c),p.top!=c.height()&&(p.top=c.height(),I=(m||parseInt(N.style("height"))||400)-p.top-p.bottom-g),J.select(".nv-legendWrap").attr("transform","translate("+F/2+","+ -p.top+")")),V.attr("transform","translate("+p.left+","+p.top+")"),i.width(F).height(q).color(e.map(function(e,t){return e.color||w(e,t)}).filter(function(t,n){return!e[n].disabled&&e[n].bar})),n.width(F).height(q).color(e.map(function(e,t){return e.color||w(e,t)}).filter(function(t,n){return!e[n].disabled&&!e[n].bar}));var G=J.select(".nv-context .nv-barsWrap").datum(U.length?U:[{values:[]}]),Y=J.select(".nv-context .nv-linesWrap").datum(z[0].disabled?[{values:[]}]:z);J.select(".nv-context").attr("transform","translate(0,"+(I+p.bottom+d.top)+")"),G.transition().call(i),Y.transition().call(n),h.x(k).on("brush",it),x&&h.extent(x);var Z=J.select(".nv-brushBackground").selectAll("g").data([x||h.extent()]),et=Z.enter().append("g");et.append("rect").attr("class","left").attr("x",0).attr("y",0).attr("height",q),et.append("rect").attr("class","right").attr("x",0).attr("y",0).attr("height",q);var tt=J.select(".nv-x.nv-brush").call(h);tt.selectAll("rect").attr("height",q),tt.selectAll(".resize").append("path").attr("d",nt),o.ticks(F/100).tickSize(-q,0),J.select(".nv-context .nv-x.nv-axis").attr("transform","translate(0,"+O.range()[0]+")"),J.select(".nv-context .nv-x.nv-axis").transition().call(o),f.scale(O).ticks(q/36).tickSize(-F,0),J.select(".nv-context .nv-y1.nv-axis").style("opacity",U.length?1:0).attr("transform","translate(0,"+k.range()[0]+")"),J.select(".nv-context .nv-y1.nv-axis").transition().call(f),l.scale(M).ticks(q/36).tickSize(U.length?0:-F,0),J.select(".nv-context .nv-y2.nv-axis").style("opacity",z.length?1:0).attr("transform","translate("+k.range()[1]+",0)"),J.select(".nv-context .nv-y2.nv-axis").transition().call(l),c.dispatch.on("stateChange",function(e){B.update()}),D.on("tooltipShow",function(e){T&&H(e,j.parentNode)}),it()}),B}var t=e.models.line(),n=e.models.line(),r=e.models.historicalBar(),i=e.models.historicalBar(),s=e.models.axis(),o=e.models.axis(),u=e.models.axis(),a=e.models.axis(),f=e.models.axis(),l=e.models.axis(),c=e.models.legend(),h=d3.svg.brush(),p={top:30,right:30,bottom:30,left:60},d={top:0,right:30,bottom:20,left:60},v=null,m=null,g=100,y=function(e){return e.x},b=function(e){return e.y},w=e.utils.defaultColor(),E=!0,S,x=null,T=!0,N=function(e,t,n,r,i){return"

    "+e+"

    "+"

    "+n+" at "+t+"

    "},C,k,L,A,O,M,_="No Data Available.",D=d3.dispatch("tooltipShow","tooltipHide","brush"),P=0;t.clipEdge(!0),n.interactive(!1),s.orient("bottom").tickPadding(5),u.orient("left"),a.orient("right"),o.orient("bottom").tickPadding(5),f.orient("left"),l.orient("right");var H=function(n,r){S&&(n.pointIndex+=Math.ceil(S[0]));var i=n.pos[0]+(r.offsetLeft||0),o=n.pos[1]+(r.offsetTop||0),f=s.tickFormat()(t.x()(n.point,n.pointIndex)),l=(n.series.bar?u:a).tickFormat()(t.y()(n.point,n.pointIndex)),c=N(n.series.key,f,l,n,B);e.tooltip.show([i,o],c,n.value<0?"n":"s",null,r)};return t.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+p.left,e.pos[1]+p.top],D.tooltipShow(e)}),t.dispatch.on("elementMouseout.tooltip",function(e){D.tooltipHide(e)}),r.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+p.left,e.pos[1]+p.top],D.tooltipShow(e)}),r.dispatch.on("elementMouseout.tooltip",function(e){D.tooltipHide(e)}),D.on("tooltipHide",function(){T&&e.tooltip.cleanup()}),B.dispatch=D,B.legend=c,B.lines=t,B.lines2=n,B.bars=r,B.bars2=i,B.xAxis=s,B.x2Axis=o,B.y1Axis=u,B.y2Axis=a,B.y3Axis=f,B.y4Axis=l,d3.rebind(B,t,"defined","size","clipVoronoi","interpolate"),B.options=e.utils.optionsFunc.bind(B),B.x=function(e){return arguments.length?(y=e,t.x(e),r.x(e),B):y},B.y=function(e){return arguments.length?(b=e,t.y(e),r.y(e),B):b},B.margin=function(e){return arguments.length?(p.top=typeof e.top!="undefined"?e.top:p.top,p.right=typeof e.right!="undefined"?e.right:p.right,p.bottom=typeof e.bottom!="undefined"?e.bottom:p.bottom,p.left=typeof e.left!="undefined"?e.left:p.left,B):p},B.width=function(e){return arguments.length?(v=e,B):v},B.height=function(e){return arguments.length?(m=e,B):m},B.color=function(t){return arguments.length?(w=e.utils.getColor(t),c.color(w),B):w},B.showLegend=function(e){return arguments.length?(E=e,B):E},B.tooltips=function(e){return arguments.length?(T=e,B):T},B.tooltipContent=function(e){return arguments.length?(N=e,B):N},B.noData=function(e){return arguments.length?(_=e,B):_},B.brushExtent=function(e){return arguments.length?(x=e,B):x},B},e.models.multiBar=function(){"use strict";function C(e){return e.each(function(e){var C=n-t.left-t.right,k=r-t.top-t.bottom,L=d3.select(this);d&&e.length&&(d=[{values:e[0].values.map(function(e){return{x:e.x,y:0,series:e.series,size:.01}})}]),c&&(e=d3.layout.stack().offset(h).values(function(e){return e.values}).y(a)(!e.length&&d?d:e)),e.forEach(function(e,t){e.values.forEach(function(e){e.series=t})}),c&&e[0].values.map(function(t,n){var r=0,i=0;e.map(function(e){var t=e.values[n];t.size=Math.abs(t.y),t.y<0?(t.y1=i,i-=t.size):(t.y1=t.size+r,r+=t.size)})});var A=y&&b?[]:e.map(function(e){return e.values.map(function(e,t){return{x:u(e,t),y:a(e,t),y0:e.y0,y1:e.y1}})});i.domain(y||d3.merge(A).map(function(e){return e.x})).rangeBands(w||[0,C],S),s.domain(b||d3.extent(d3.merge(A).map(function(e){return c?e.y>0?e.y1:e.y1+e.y:e.y}).concat(f))).range(E||[k,0]),i.domain()[0]===i.domain()[1]&&(i.domain()[0]?i.domain([i.domain()[0]-i.domain()[0]*.01,i.domain()[1]+i.domain()[1]*.01]):i.domain([-1,1])),s.domain()[0]===s.domain()[1]&&(s.domain()[0]?s.domain([s.domain()[0]+s.domain()[0]*.01,s.domain()[1]-s.domain()[1]*.01]):s.domain([-1,1])),T=T||i,N=N||s;var O=L.selectAll("g.nv-wrap.nv-multibar").data([e]),M=O.enter().append("g").attr("class","nvd3 nv-wrap nv-multibar"),_=M.append("defs"),D=M.append("g"),P=O.select("g");D.append("g").attr("class","nv-groups"),O.attr("transform","translate("+t.left+","+t.top+")"),_.append("clipPath").attr("id","nv-edge-clip-"+o).append("rect"),O.select("#nv-edge-clip-"+o+" rect").attr("width",C).attr("height",k),P.attr("clip-path",l?"url(#nv-edge-clip-"+o+")":"");var H=O.select(".nv-groups").selectAll(".nv-group").data(function(e){return e},function(e,t){return t});H.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6),H.exit().transition().selectAll("rect.nv-bar").delay(function(t,n){return n*g/e[0].values.length}).attr("y",function(e){return c?N(e.y0):N(0)}).attr("height",0).remove(),H.attr("class",function(e,t){return"nv-group nv-series-"+t}).classed("hover",function(e){return e.hover}).style("fill",function(e,t){return p(e,t)}).style("stroke",function(e,t){return p(e,t)}),H.transition().style("stroke-opacity",1).style("fill-opacity",.75);var B=H.selectAll("rect.nv-bar").data(function(t){return d&&!e.length?d.values:t.values});B.exit().remove();var j=B.enter().append("rect").attr("class",function(e,t){return a(e,t)<0?"nv-bar negative":"nv-bar positive"}).attr("x",function(t,n,r){return c?0:r*i.rangeBand()/e.length}).attr("y",function(e){return N(c?e.y0:0)}).attr("height",0).attr("width",i.rangeBand()/(c?1:e.length)).attr("transform",function(e,t){return"translate("+i(u(e,t))+",0)"});B.style("fill",function(e,t,n){return p(e,n,t)}).style("stroke",function(e,t,n){return p(e,n,t)}).on("mouseover",function(t,n){d3.select(this).classed("hover",!0),x.elementMouseover({value:a(t,n),point:t,series:e[t.series],pos:[i(u(t,n))+i.rangeBand()*(c?e.length/2:t.series+.5)/e.length,s(a(t,n)+(c?t.y0:0))],pointIndex:n,seriesIndex:t.series,e:d3.event})}).on("mouseout",function(t,n){d3.select(this).classed("hover",!1),x.elementMouseout({value:a(t,n),point:t,series:e[t.series],pointIndex:n,seriesIndex:t.series,e:d3.event})}).on("click",function(t,n){x.elementClick({value:a(t,n),point:t,series:e[t.series],pos:[i(u(t,n))+i.rangeBand()*(c?e.length/2:t.series+.5)/e.length +,s(a(t,n)+(c?t.y0:0))],pointIndex:n,seriesIndex:t.series,e:d3.event}),d3.event.stopPropagation()}).on("dblclick",function(t,n){x.elementDblClick({value:a(t,n),point:t,series:e[t.series],pos:[i(u(t,n))+i.rangeBand()*(c?e.length/2:t.series+.5)/e.length,s(a(t,n)+(c?t.y0:0))],pointIndex:n,seriesIndex:t.series,e:d3.event}),d3.event.stopPropagation()}),B.attr("class",function(e,t){return a(e,t)<0?"nv-bar negative":"nv-bar positive"}).transition().attr("transform",function(e,t){return"translate("+i(u(e,t))+",0)"}),v&&(m||(m=e.map(function(){return!0})),B.style("fill",function(e,t,n){return d3.rgb(v(e,t)).darker(m.map(function(e,t){return t}).filter(function(e,t){return!m[t]})[n]).toString()}).style("stroke",function(e,t,n){return d3.rgb(v(e,t)).darker(m.map(function(e,t){return t}).filter(function(e,t){return!m[t]})[n]).toString()})),c?B.transition().delay(function(t,n){return n*g/e[0].values.length}).attr("y",function(e,t){return s(c?e.y1:0)}).attr("height",function(e,t){return Math.max(Math.abs(s(e.y+(c?e.y0:0))-s(c?e.y0:0)),1)}).attr("x",function(t,n){return c?0:t.series*i.rangeBand()/e.length}).attr("width",i.rangeBand()/(c?1:e.length)):B.transition().delay(function(t,n){return n*g/e[0].values.length}).attr("x",function(t,n){return t.series*i.rangeBand()/e.length}).attr("width",i.rangeBand()/e.length).attr("y",function(e,t){return a(e,t)<0?s(0):s(0)-s(a(e,t))<1?s(0)-1:s(a(e,t))||0}).attr("height",function(e,t){return Math.max(Math.abs(s(a(e,t))-s(0)),1)||0}),T=i.copy(),N=s.copy()}),C}var t={top:0,right:0,bottom:0,left:0},n=960,r=500,i=d3.scale.ordinal(),s=d3.scale.linear(),o=Math.floor(Math.random()*1e4),u=function(e){return e.x},a=function(e){return e.y},f=[0],l=!0,c=!1,h="zero",p=e.utils.defaultColor(),d=!1,v=null,m,g=1200,y,b,w,E,S=.1,x=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout"),T,N;return C.dispatch=x,C.options=e.utils.optionsFunc.bind(C),C.x=function(e){return arguments.length?(u=e,C):u},C.y=function(e){return arguments.length?(a=e,C):a},C.margin=function(e){return arguments.length?(t.top=typeof e.top!="undefined"?e.top:t.top,t.right=typeof e.right!="undefined"?e.right:t.right,t.bottom=typeof e.bottom!="undefined"?e.bottom:t.bottom,t.left=typeof e.left!="undefined"?e.left:t.left,C):t},C.width=function(e){return arguments.length?(n=e,C):n},C.height=function(e){return arguments.length?(r=e,C):r},C.xScale=function(e){return arguments.length?(i=e,C):i},C.yScale=function(e){return arguments.length?(s=e,C):s},C.xDomain=function(e){return arguments.length?(y=e,C):y},C.yDomain=function(e){return arguments.length?(b=e,C):b},C.xRange=function(e){return arguments.length?(w=e,C):w},C.yRange=function(e){return arguments.length?(E=e,C):E},C.forceY=function(e){return arguments.length?(f=e,C):f},C.stacked=function(e){return arguments.length?(c=e,C):c},C.stackOffset=function(e){return arguments.length?(h=e,C):h},C.clipEdge=function(e){return arguments.length?(l=e,C):l},C.color=function(t){return arguments.length?(p=e.utils.getColor(t),C):p},C.barColor=function(t){return arguments.length?(v=e.utils.getColor(t),C):v},C.disabled=function(e){return arguments.length?(m=e,C):m},C.id=function(e){return arguments.length?(o=e,C):o},C.hideable=function(e){return arguments.length?(d=e,C):d},C.delay=function(e){return arguments.length?(g=e,C):g},C.groupSpacing=function(e){return arguments.length?(S=e,C):S},C},e.models.multiBarChart=function(){"use strict";function A(e){return e.each(function(e){var b=d3.select(this),O=this,M=(u||parseInt(b.style("width"))||960)-o.left-o.right,_=(a||parseInt(b.style("height"))||400)-o.top-o.bottom;A.update=function(){b.transition().duration(k).call(A)},A.container=this,S.disabled=e.map(function(e){return!!e.disabled});if(!x){var D;x={};for(D in S)S[D]instanceof Array?x[D]=S[D].slice(0):x[D]=S[D]}if(!e||!e.length||!e.filter(function(e){return e.values.length}).length){var P=b.selectAll(".nv-noData").data([T]);return P.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),P.attr("x",o.left+M/2).attr("y",o.top+_/2).text(function(e){return e}),A}b.selectAll(".nv-noData").remove(),w=t.xScale(),E=t.yScale();var H=b.selectAll("g.nv-wrap.nv-multiBarWithLegend").data([e]),B=H.enter().append("g").attr("class","nvd3 nv-wrap nv-multiBarWithLegend").append("g"),j=H.select("g");B.append("g").attr("class","nv-x nv-axis"),B.append("g").attr("class","nv-y nv-axis"),B.append("g").attr("class","nv-barsWrap"),B.append("g").attr("class","nv-legendWrap"),B.append("g").attr("class","nv-controlsWrap"),c&&(i.width(M-C()),t.barColor()&&e.forEach(function(e,t){e.color=d3.rgb("#ccc").darker(t*1.5).toString()}),j.select(".nv-legendWrap").datum(e).call(i),o.top!=i.height()&&(o.top=i.height(),_=(a||parseInt(b.style("height"))||400)-o.top-o.bottom),j.select(".nv-legendWrap").attr("transform","translate("+C()+","+ -o.top+")"));if(l){var F=[{key:"Grouped",disabled:t.stacked()},{key:"Stacked",disabled:!t.stacked()}];s.width(C()).color(["#444","#444","#444"]),j.select(".nv-controlsWrap").datum(F).attr("transform","translate(0,"+ -o.top+")").call(s)}H.attr("transform","translate("+o.left+","+o.top+")"),d&&j.select(".nv-y.nv-axis").attr("transform","translate("+M+",0)"),t.disabled(e.map(function(e){return e.disabled})).width(M).height(_).color(e.map(function(e,t){return e.color||f(e,t)}).filter(function(t,n){return!e[n].disabled}));var I=j.select(".nv-barsWrap").datum(e.filter(function(e){return!e.disabled}));I.transition().call(t);if(h){n.scale(w).ticks(M/100).tickSize(-_,0),j.select(".nv-x.nv-axis").attr("transform","translate(0,"+E.range()[0]+")"),j.select(".nv-x.nv-axis").transition().call(n);var q=j.select(".nv-x.nv-axis > g").selectAll("g");q.selectAll("line, text").style("opacity",1);if(m){var R=function(e,t){return"translate("+e+","+t+")"},U=5,z=17;q.selectAll("text").attr("transform",function(e,t,n){return R(0,n%2==0?U:z)});var W=d3.selectAll(".nv-x.nv-axis .nv-wrap g g text")[0].length;j.selectAll(".nv-x.nv-axis .nv-axisMaxMin text").attr("transform",function(e,t){return R(0,t===0||W%2!==0?z:U)})}v&&q.filter(function(t,n){return n%Math.ceil(e[0].values.length/(M/100))!==0}).selectAll("text, line").style("opacity",0),g&&q.selectAll(".tick text").attr("transform","rotate("+g+" 0,0)").style("text-anchor",g>0?"start":"end"),j.select(".nv-x.nv-axis").selectAll("g.nv-axisMaxMin text").style("opacity",1)}p&&(r.scale(E).ticks(_/36).tickSize(-M,0),j.select(".nv-y.nv-axis").transition().call(r)),i.dispatch.on("stateChange",function(e){S=e,N.stateChange(S),A.update()}),s.dispatch.on("legendClick",function(e,n){if(!e.disabled)return;F=F.map(function(e){return e.disabled=!0,e}),e.disabled=!1;switch(e.key){case"Grouped":t.stacked(!1);break;case"Stacked":t.stacked(!0)}S.stacked=t.stacked(),N.stateChange(S),A.update()}),N.on("tooltipShow",function(e){y&&L(e,O.parentNode)}),N.on("changeState",function(n){typeof n.disabled!="undefined"&&(e.forEach(function(e,t){e.disabled=n.disabled[t]}),S.disabled=n.disabled),typeof n.stacked!="undefined"&&(t.stacked(n.stacked),S.stacked=n.stacked),A.update()})}),A}var t=e.models.multiBar(),n=e.models.axis(),r=e.models.axis(),i=e.models.legend(),s=e.models.legend(),o={top:30,right:20,bottom:50,left:60},u=null,a=null,f=e.utils.defaultColor(),l=!0,c=!0,h=!0,p=!0,d=!1,v=!0,m=!1,g=0,y=!0,b=function(e,t,n,r,i){return"

    "+e+"

    "+"

    "+n+" on "+t+"

    "},w,E,S={stacked:!1},x=null,T="No Data Available.",N=d3.dispatch("tooltipShow","tooltipHide","stateChange","changeState"),C=function(){return l?180:0},k=250;t.stacked(!1),n.orient("bottom").tickPadding(7).highlightZero(!0).showMaxMin(!1).tickFormat(function(e){return e}),r.orient(d?"right":"left").tickFormat(d3.format(",.1f")),s.updateState(!1);var L=function(i,s){var o=i.pos[0]+(s.offsetLeft||0),u=i.pos[1]+(s.offsetTop||0),a=n.tickFormat()(t.x()(i.point,i.pointIndex)),f=r.tickFormat()(t.y()(i.point,i.pointIndex)),l=b(i.series.key,a,f,i,A);e.tooltip.show([o,u],l,i.value<0?"n":"s",null,s)};return t.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+o.left,e.pos[1]+o.top],N.tooltipShow(e)}),t.dispatch.on("elementMouseout.tooltip",function(e){N.tooltipHide(e)}),N.on("tooltipHide",function(){y&&e.tooltip.cleanup()}),A.dispatch=N,A.multibar=t,A.legend=i,A.xAxis=n,A.yAxis=r,d3.rebind(A,t,"x","y","xDomain","yDomain","xRange","yRange","forceX","forceY","clipEdge","id","stacked","stackOffset","delay","barColor","groupSpacing"),A.options=e.utils.optionsFunc.bind(A),A.margin=function(e){return arguments.length?(o.top=typeof e.top!="undefined"?e.top:o.top,o.right=typeof e.right!="undefined"?e.right:o.right,o.bottom=typeof e.bottom!="undefined"?e.bottom:o.bottom,o.left=typeof e.left!="undefined"?e.left:o.left,A):o},A.width=function(e){return arguments.length?(u=e,A):u},A.height=function(e){return arguments.length?(a=e,A):a},A.color=function(t){return arguments.length?(f=e.utils.getColor(t),i.color(f),A):f},A.showControls=function(e){return arguments.length?(l=e,A):l},A.showLegend=function(e){return arguments.length?(c=e,A):c},A.showXAxis=function(e){return arguments.length?(h=e,A):h},A.showYAxis=function(e){return arguments.length?(p=e,A):p},A.rightAlignYAxis=function(e){return arguments.length?(d=e,r.orient(e?"right":"left"),A):d},A.reduceXTicks=function(e){return arguments.length?(v=e,A):v},A.rotateLabels=function(e){return arguments.length?(g=e,A):g},A.staggerLabels=function(e){return arguments.length?(m=e,A):m},A.tooltip=function(e){return arguments.length?(b=e,A):b},A.tooltips=function(e){return arguments.length?(y=e,A):y},A.tooltipContent=function(e){return arguments.length?(b=e,A):b},A.state=function(e){return arguments.length?(S=e,A):S},A.defaultState=function(e){return arguments.length?(x=e,A):x},A.noData=function(e){return arguments.length?(T=e,A):T},A.transitionDuration=function(e){return arguments.length?(k=e,A):k},A},e.models.multiBarHorizontal=function(){"use strict";function C(e){return e.each(function(e){var i=n-t.left-t.right,y=r-t.top-t.bottom,C=d3.select(this);p&&(e=d3.layout.stack().offset("zero").values(function(e){return e.values}).y(a)(e)),e.forEach(function(e,t){e.values.forEach(function(e){e.series=t})}),p&&e[0].values.map(function(t,n){var r=0,i=0;e.map(function(e){var t=e.values[n];t.size=Math.abs(t.y),t.y<0?(t.y1=i-t.size,i-=t.size):(t.y1=r,r+=t.size)})});var k=b&&w?[]:e.map(function(e){return e.values.map(function(e,t){return{x:u(e,t),y:a(e,t),y0:e.y0,y1:e.y1}})});s.domain(b||d3.merge(k).map(function(e){return e.x})).rangeBands(E||[0,y],.1),o.domain(w||d3.extent(d3.merge(k).map(function(e){return p?e.y>0?e.y1+e.y:e.y1:e.y}).concat(f))),d&&!p?o.range(S||[o.domain()[0]<0?m:0,i-(o.domain()[1]>0?m:0)]):o.range(S||[0,i]),T=T||s,N=N||d3.scale.linear().domain(o.domain()).range([o(0),o(0)]);var L=d3.select(this).selectAll("g.nv-wrap.nv-multibarHorizontal").data([e]),A=L.enter().append("g").attr("class","nvd3 nv-wrap nv-multibarHorizontal"),O=A.append("defs"),M=A.append("g"),_=L.select("g");M.append("g").attr("class","nv-groups"),L.attr("transform","translate("+t.left+","+t.top+")");var D=L.select(".nv-groups").selectAll(".nv-group").data(function(e){return e},function(e,t){return t});D.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6),D.exit().transition().style("stroke-opacity",1e-6).style("fill-opacity",1e-6).remove(),D.attr("class",function(e,t){return"nv-group nv-series-"+t}).classed("hover",function(e){return e.hover}).style("fill",function(e,t){return l(e,t)}).style("stroke",function(e,t){return l(e,t)}),D.transition().style("stroke-opacity",1).style("fill-opacity",.75);var P=D.selectAll("g.nv-bar").data(function(e){return e.values});P.exit().remove();var H=P.enter().append("g").attr("transform",function(t,n,r){return"translate("+N(p?t.y0:0)+","+(p?0:r*s.rangeBand()/e.length+s(u(t,n)))+")"});H.append("rect").attr("width",0).attr("height",s.rangeBand()/(p?1:e.length)),P.on("mouseover",function(t,n){d3.select(this).classed("hover",!0),x.elementMouseover({value:a(t,n),point:t,series:e[t.series],pos:[o(a(t,n)+(p?t.y0:0)),s(u(t,n))+s.rangeBand()*(p?e.length/2:t.series+.5)/e.length],pointIndex:n,seriesIndex:t.series,e:d3.event})}).on("mouseout",function(t,n){d3.select(this).classed("hover",!1),x.elementMouseout({value:a(t,n),point:t,series:e[t.series],pointIndex:n,seriesIndex:t.series,e:d3.event})}).on("click",function(t,n){x.elementClick({value:a(t,n),point:t,series:e[t.series],pos:[s(u(t,n))+s.rangeBand()*(p?e.length/2:t.series+.5)/e.length,o(a(t,n)+(p?t.y0:0))],pointIndex:n,seriesIndex:t.series,e:d3.event}),d3.event.stopPropagation()}).on("dblclick",function(t,n){x.elementDblClick({value:a(t,n),point:t,series:e[t.series],pos:[s(u(t,n))+s.rangeBand()*(p?e.length/2:t.series+.5)/e.length,o(a(t,n)+(p?t.y0:0))],pointIndex:n,seriesIndex:t.series,e:d3.event}),d3.event.stopPropagation()}),H.append("text"),d&&!p?(P.select("text").attr("text-anchor",function(e,t){return a(e,t)<0?"end":"start"}).attr("y",s.rangeBand()/(e.length*2)).attr("dy",".32em").text(function(e,t){return g(a(e,t))}),P.transition().select("text").attr("x",function(e,t){return a(e,t)<0?-4:o(a(e,t))-o(0)+4})):P.selectAll("text").text(""),v&&!p?(H.append("text").classed("nv-bar-label",!0),P.select("text.nv-bar-label").attr("text-anchor",function(e,t){return a(e,t)<0?"start":"end"}).attr("y",s.rangeBand()/(e.length*2)).attr("dy",".32em").text(function(e,t){return u(e,t)}),P.transition().select("text.nv-bar-label").attr("x",function(e,t){return a(e,t)<0?o(0)-o(a(e,t))+4:-4})):P.selectAll("text.nv-bar-label").text(""),P.attr("class",function(e,t){return a(e,t)<0?"nv-bar negative":"nv-bar positive"}),c&&(h||(h=e.map(function(){return!0})),P.style("fill",function(e,t,n){return d3.rgb(c(e,t)).darker(h.map(function(e,t){return t}).filter(function(e,t){return!h[t]})[n]).toString()}).style("stroke",function(e,t,n){return d3.rgb(c(e,t)).darker(h.map(function(e,t){return t}).filter(function(e,t){return!h[t]})[n]).toString()})),p?P.transition().attr("transform",function(e,t){return"translate("+o(e.y1)+","+s(u(e,t))+")"}).select("rect").attr("width",function(e,t){return Math.abs(o(a(e,t)+e.y0)-o(e.y0))}).attr("height",s.rangeBand()):P.transition().attr("transform",function(t,n){return"translate("+(a(t,n)<0?o(a(t,n)):o(0))+","+(t.series*s.rangeBand()/e.length+s(u(t,n)))+")"}).select("rect").attr("height",s.rangeBand()/e.length).attr("width",function(e,t){return Math.max(Math.abs(o(a(e,t))-o(0)),1)}),T=s.copy(),N=o.copy()}),C}var t={top:0,right:0,bottom:0,left:0},n=960,r=500,i=Math.floor(Math.random()*1e4),s=d3.scale.ordinal(),o=d3.scale.linear(),u=function(e){return e.x},a=function(e){return e.y},f=[0],l=e.utils.defaultColor(),c=null,h,p=!1,d=!1,v=!1,m=60,g=d3.format(",.2f"),y=1200,b,w,E,S,x=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout"),T,N;return C.dispatch=x,C.options=e.utils.optionsFunc.bind(C),C.x=function(e){return arguments.length?(u=e,C):u},C.y=function(e){return arguments.length?(a=e,C):a},C.margin=function(e){return arguments.length?(t.top=typeof e.top!="undefined"?e.top:t.top,t.right=typeof e.right!="undefined"?e.right:t.right,t.bottom=typeof e.bottom!="undefined"?e.bottom:t.bottom,t.left=typeof e.left!="undefined"?e.left:t.left,C):t},C.width=function(e){return arguments.length?(n=e,C):n},C.height=function(e){return arguments.length?(r=e,C):r},C.xScale=function(e){return arguments.length?(s=e,C):s},C.yScale=function(e){return arguments.length?(o=e,C):o},C.xDomain=function(e){return arguments.length?(b=e,C):b},C.yDomain=function(e){return arguments.length?(w=e,C):w},C.xRange=function(e){return arguments.length?(E=e,C):E},C.yRange=function(e){return arguments.length?(S=e,C):S},C.forceY=function(e){return arguments.length?(f=e,C):f},C.stacked=function(e){return arguments.length?(p=e,C):p},C.color=function(t){return arguments.length?(l=e.utils.getColor(t),C):l},C.barColor=function(t){return arguments.length?(c=e.utils.getColor(t),C):c},C.disabled=function(e){return arguments.length?(h=e,C):h},C.id=function(e){return arguments.length?(i=e,C):i},C.delay=function(e){return arguments.length?(y=e,C):y},C.showValues=function(e){return arguments.length?(d=e,C):d},C.showBarLabels=function(e){return arguments.length?(v=e,C):v},C.valueFormat=function(e){return arguments.length?(g=e,C):g},C.valuePadding=function(e){return arguments.length?(m=e,C):m},C},e.models.multiBarHorizontalChart=function(){"use strict";function C(e){return e.each(function(e){var d=d3.select(this),m=this,k=(u||parseInt(d.style("width"))||960)-o.left-o.right,L=(a||parseInt(d.style("height"))||400)-o.top-o.bottom;C.update=function(){d.transition().duration(T).call(C)},C.container=this,b.disabled=e.map(function(e){return!!e.disabled});if(!w){var A;w={};for(A in b)b[A]instanceof Array?w[A]=b[A].slice(0):w[A]=b[A]}if(!e||!e.length||!e.filter(function(e){return e.values.length}).length){var O=d.selectAll(".nv-noData").data([E]);return O.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),O.attr("x",o.left+k/2).attr("y",o.top+L/2).text(function(e){return e}),C}d.selectAll(".nv-noData").remove(),g=t.xScale(),y=t.yScale();var M=d.selectAll("g.nv-wrap.nv-multiBarHorizontalChart").data([e]),_=M.enter().append("g").attr("class","nvd3 nv-wrap nv-multiBarHorizontalChart").append("g"),D=M.select("g");_.append("g").attr("class","nv-x nv-axis"),_.append("g").attr("class","nv-y nv-axis").append("g").attr("class","nv-zeroLine").append("line"),_.append("g").attr("class","nv-barsWrap"),_.append("g").attr("class","nv-legendWrap"),_.append("g").attr("class","nv-controlsWrap"),c&&(i.width(k-x()),t.barColor()&&e.forEach(function(e,t){e.color=d3.rgb("#ccc").darker(t*1.5).toString()}),D.select(".nv-legendWrap").datum(e).call(i),o.top!=i.height()&&(o.top=i.height(),L=(a||parseInt(d.style("height"))||400)-o.top-o.bottom),D.select(".nv-legendWrap").attr("transform","translate("+x()+","+ -o.top+")"));if(l){var P=[{key:"Grouped",disabled:t.stacked()},{key:"Stacked",disabled:!t.stacked()}];s.width(x()).color(["#444","#444","#444"]),D.select(".nv-controlsWrap").datum(P).attr("transform","translate(0,"+ -o.top+")").call(s)}M.attr("transform","translate("+o.left+","+o.top+")"),t.disabled(e.map(function(e){return e.disabled})).width(k).height(L).color(e.map(function(e,t){return e.color||f(e,t)}).filter(function(t,n){return!e[n].disabled}));var H=D.select(".nv-barsWrap").datum(e.filter(function(e){return!e.disabled}));H.transition().call(t);if(h){n.scale(g).ticks(L/24).tickSize(-k,0),D.select(".nv-x.nv-axis").transition().call(n);var B=D.select(".nv-x.nv-axis").selectAll("g");B.selectAll("line, text")}p&&(r.scale(y).ticks(k/100).tickSize(-L,0),D.select(".nv-y.nv-axis").attr("transform","translate(0,"+L+")"),D.select(".nv-y.nv-axis").transition().call(r)),D.select(".nv-zeroLine line").attr("x1",y(0)).attr("x2",y(0)).attr("y1",0).attr("y2",-L),i.dispatch.on("stateChange",function(e){b=e,S.stateChange(b),C.update()}),s.dispatch.on("legendClick",function(e,n){if(!e.disabled)return;P=P.map(function(e){return e.disabled=!0,e}),e.disabled=!1;switch(e.key){case"Grouped":t.stacked(!1);break;case"Stacked":t.stacked(!0)}b.stacked=t.stacked(),S.stateChange(b),C.update()}),S.on("tooltipShow",function(e){v&&N(e,m.parentNode)}),S.on("changeState",function(n){typeof n.disabled!="undefined"&&(e.forEach(function(e,t){e.disabled=n.disabled[t]}),b.disabled=n.disabled),typeof n.stacked!="undefined"&&(t.stacked(n.stacked),b.stacked=n.stacked),C.update()})}),C}var t=e.models.multiBarHorizontal(),n=e.models.axis(),r=e.models.axis(),i=e.models.legend().height(30),s=e.models.legend().height(30),o={top:30,right:20,bottom:50,left:60},u=null,a=null,f=e.utils.defaultColor(),l=!0,c=!0,h=!0,p=!0,d=!1,v=!0,m=function(e,t,n,r,i){return"

    "+e+" - "+t+"

    "+"

    "+n+"

    "},g,y,b={stacked:d},w=null,E="No Data Available.",S=d3.dispatch("tooltipShow","tooltipHide","stateChange","changeState"),x=function(){return l?180:0},T=250;t.stacked(d),n.orient("left").tickPadding(5).highlightZero(!1).showMaxMin(!1).tickFormat(function(e){return e}),r.orient("bottom").tickFormat(d3.format(",.1f")),s.updateState(!1);var N=function(i,s){var o=i.pos[0]+(s.offsetLeft||0),u=i.pos[1]+(s.offsetTop||0),a=n.tickFormat()(t.x()(i.point,i.pointIndex)),f=r.tickFormat()(t.y()(i.point,i.pointIndex)),l=m(i.series.key,a,f,i,C);e.tooltip.show([o,u],l,i.value<0?"e":"w",null,s)};return t.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+o.left,e.pos[1]+o.top],S.tooltipShow(e)}),t.dispatch.on("elementMouseout.tooltip",function(e){S.tooltipHide(e)}),S.on("tooltipHide",function(){v&&e.tooltip.cleanup()}),C.dispatch=S,C.multibar=t,C.legend=i,C.xAxis=n,C.yAxis=r,d3.rebind(C,t,"x","y","xDomain","yDomain","xRange","yRange","forceX","forceY","clipEdge","id","delay","showValues","showBarLabels","valueFormat","stacked","barColor"),C.options=e.utils.optionsFunc.bind(C),C.margin=function(e){return arguments.length?(o.top=typeof e.top!="undefined"?e.top:o.top,o.right=typeof e.right!="undefined"?e.right:o.right,o.bottom=typeof e.bottom!="undefined"?e.bottom:o.bottom,o.left=typeof e.left!="undefined"?e.left:o.left,C):o},C.width=function(e){return arguments.length?(u=e,C):u},C.height=function(e){return arguments.length?(a=e,C):a},C.color=function(t){return arguments.length?(f=e.utils.getColor(t),i.color(f),C):f},C.showControls=function(e){return arguments.length?(l=e,C):l},C.showLegend=function(e){return arguments.length?(c=e,C):c},C.showXAxis=function(e){return arguments.length?(h=e,C):h},C.showYAxis=function(e){return arguments.length?(p=e,C):p},C.tooltip=function(e){return arguments.length?(m=e,C):m},C.tooltips=function(e){return arguments.length?(v=e,C):v},C.tooltipContent=function(e){return arguments.length?(m=e,C):m},C.state=function(e){return arguments.length?(b=e,C):b},C.defaultState=function(e){return arguments.length?(w=e,C):w},C.noData=function(e){return arguments.length?(E=e,C):E},C.transitionDuration=function(e){return arguments.length?(T=e,C):T},C},e.models.multiChart=function(){"use strict";function C(e){return e.each(function(e){var u=d3.select(this),f=this;C.update=function(){u.transition().call(C)},C.container=this;var k=(r||parseInt(u.style("width"))||960)-t.left-t.right,L=(i||parseInt(u.style("height"))||400)-t.top-t.bottom,A=e.filter(function(e){return!e.disabled&&e.type=="line"&&e.yAxis==1}),O=e.filter(function(e){return!e.disabled&&e.type=="line"&&e.yAxis==2}),M=e.filter(function(e){return!e.disabled&&e.type=="bar"&&e.yAxis==1}),_=e.filter(function(e){return!e.disabled&&e.type=="bar"&&e.yAxis==2}),D=e.filter(function(e){return!e.disabled&&e.type=="area"&&e.yAxis==1}),P=e.filter(function(e){return!e.disabled&&e.type=="area"&&e.yAxis==2}),H=e.filter(function(e){return!e.disabled&&e.yAxis==1}).map(function(e){return e.values.map(function(e,t){return{x:e.x,y:e.y}})}),B=e.filter(function(e){return!e.disabled&&e.yAxis==2}).map(function(e){return e.values.map(function(e,t){return{x:e.x,y:e.y}})});a.domain(d3.extent(d3.merge(H.concat(B)),function(e){return e.x})).range([0,k]);var j=u.selectAll("g.wrap.multiChart").data([e]),F=j.enter().append("g").attr("class","wrap nvd3 multiChart").append("g");F.append("g").attr("class","x axis"),F.append("g").attr("class","y1 axis"),F.append("g").attr("class","y2 axis"),F.append("g").attr("class","lines1Wrap"),F.append("g").attr("class","lines2Wrap"),F.append("g").attr("class","bars1Wrap"),F.append("g").attr("class","bars2Wrap"),F.append("g").attr("class","stack1Wrap"),F.append("g").attr("class","stack2Wrap"),F.append("g").attr("class","legendWrap");var I=j.select("g");s&&(x.width(k/2),I.select(".legendWrap").datum(e.map(function(e){return e.originalKey=e.originalKey===undefined?e.key:e.originalKey,e.key=e.originalKey+(e.yAxis==1?"":" (right axis)"),e})).call(x),t.top!=x.height()&&(t.top=x.height(),L=(i||parseInt(u.style("height"))||400)-t.top-t.bottom),I.select(".legendWrap").attr("transform","translate("+k/2+","+ -t.top+")")),d.width(k).height(L).interpolate("monotone").color(e.map(function(e,t){return e.color||n[t%n.length]}).filter(function(t,n){return!e[n].disabled&&e[n].yAxis==1&&e[n].type=="line"})),v.width(k).height(L).interpolate("monotone").color(e.map(function(e,t){return e.color||n[t%n.length]}).filter(function(t,n){return!e[n].disabled&&e[n].yAxis==2&&e[n].type=="line"})),m.width(k).height(L).color(e.map(function(e,t){return e.color||n[t%n.length]}).filter(function(t,n){return!e[n].disabled&&e[n].yAxis==1&&e[n].type=="bar"})),g.width(k).height(L).color(e.map(function(e,t){return e.color||n[t%n.length]}).filter(function(t,n){return!e[n].disabled&&e[n].yAxis==2&&e[n].type=="bar"})),y.width(k).height(L).color(e.map(function(e,t){return e.color||n[t%n.length]}).filter(function(t,n){return!e[n].disabled&&e[n].yAxis==1&&e[n].type=="area"})),b.width(k).height(L).color(e.map(function(e,t){return e.color||n[t%n.length]}).filter(function(t,n){return!e[n].disabled&&e[n].yAxis==2&&e[n].type=="area"})),I.attr("transform","translate("+t.left+","+t.top+")");var q=I.select(".lines1Wrap").datum(A),R=I.select(".bars1Wrap").datum(M),U=I.select(".stack1Wrap").datum(D),z=I.select(".lines2Wrap").datum(O),W=I.select(".bars2Wrap").datum(_),X=I.select(".stack2Wrap").datum(P),V=D.length?D.map(function(e){return e.values}).reduce(function(e,t){return e.map(function(e,n){return{x:e.x,y:e.y+t[n].y}})}).concat([{x:0,y:0}]):[],$=P.length?P.map(function(e){return e.values}).reduce(function(e,t){return e.map(function(e,n){return{x:e.x,y:e.y+t[n].y}})}).concat([{x:0,y:0}]):[];h.domain(l||d3.extent(d3.merge(H).concat(V),function(e){return e.y})).range([0,L]),p.domain(c||d3.extent(d3.merge(B).concat($),function(e){return e.y})).range([0,L]),d.yDomain(h.domain()),m.yDomain(h.domain()),y.yDomain(h.domain()),v.yDomain(p.domain()),g.yDomain(p.domain()),b.yDomain(p.domain()),D.length&&d3.transition(U).call(y),P.length&&d3.transition(X).call(b),M.length&&d3.transition(R).call(m),_.length&&d3.transition(W).call(g),A.length&&d3.transition(q).call(d),O.length&&d3.transition(z).call(v),w.ticks(k/100).tickSize(-L,0),I.select(".x.axis").attr("transform","translate(0,"+L+")"),d3.transition(I.select(".x.axis")).call(w),E.ticks(L/36).tickSize(-k,0),d3.transition(I.select(".y1.axis")).call(E),S.ticks(L/36).tickSize(-k,0),d3.transition(I.select(".y2.axis")).call(S),I.select(".y2.axis").style("opacity",B.length?1:0).attr("transform","translate("+a.range()[1]+",0)"),x.dispatch.on("stateChange",function(e){C.update()}),T.on("tooltipShow",function(e){o&&N(e,f.parentNode)})}),C}var t={top:30,right:20,bottom:50,left:60},n=d3.scale.category20().range(),r=null,i=null,s=!0,o=!0,u=function(e,t,n,r,i){return"

    "+e+"

    "+"

    "+n+" at "+t+"

    "},a,f,l,c,a=d3.scale.linear(),h=d3.scale.linear(),p=d3.scale.linear(),d=e.models.line().yScale(h),v=e.models.line().yScale(p),m=e.models.multiBar().stacked(!1).yScale(h),g=e.models.multiBar().stacked(!1).yScale(p),y=e.models.stackedArea().yScale(h),b=e.models.stackedArea().yScale(p),w=e.models.axis().scale(a).orient("bottom").tickPadding(5),E=e.models.axis().scale(h).orient("left"),S=e.models.axis().scale(p).orient("right"),x=e.models.legend().height(30),T=d3.dispatch("tooltipShow","tooltipHide"),N=function(t,n){var r=t.pos[0]+(n.offsetLeft||0),i=t.pos[1]+(n.offsetTop||0),s=w.tickFormat()(d.x()(t.point,t.pointIndex)),o=(t.series.yAxis==2?S:E).tickFormat()(d.y()(t.point,t.pointIndex)),a=u(t.series.key,s,o,t,C);e.tooltip.show([r,i],a,undefined,undefined,n.offsetParent)};return d.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+t.left,e.pos[1]+t.top],T.tooltipShow(e)}),d.dispatch.on("elementMouseout.tooltip",function(e){T.tooltipHide(e)}),v.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+t.left,e.pos[1]+t.top],T.tooltipShow(e)}),v.dispatch.on("elementMouseout.tooltip",function(e){T.tooltipHide(e)}),m.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+t.left,e.pos[1]+t.top],T.tooltipShow(e)}),m.dispatch.on("elementMouseout.tooltip",function(e){T.tooltipHide(e)}),g.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+t.left,e.pos[1]+t.top],T.tooltipShow(e)}),g.dispatch.on("elementMouseout.tooltip",function(e){T.tooltipHide(e)}),y.dispatch.on("tooltipShow",function(e){if(!Math.round(y.y()(e.point)*100))return setTimeout(function(){d3.selectAll(".point.hover").classed("hover",!1)},0),!1;e.pos=[e.pos[0]+t.left,e.pos[1]+t.top],T.tooltipShow(e)}),y.dispatch.on("tooltipHide",function(e){T.tooltipHide(e)}),b.dispatch.on("tooltipShow",function(e){if(!Math.round(b.y()(e.point)*100))return setTimeout(function(){d3.selectAll(".point.hover").classed("hover",!1)},0),!1;e.pos=[e.pos[0]+t.left,e.pos[1]+t.top],T.tooltipShow(e)}),b.dispatch.on("tooltipHide",function(e){T.tooltipHide(e)}),d.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+t.left,e.pos[1]+t.top],T.tooltipShow(e)}),d.dispatch.on("elementMouseout.tooltip",function(e){T.tooltipHide(e)}),v.dispatch.on("elementMouseover.tooltip",function(e){e.pos=[e.pos[0]+t.left,e.pos[1]+t.top],T.tooltipShow(e)}),v.dispatch.on("elementMouseout.tooltip",function(e){T.tooltipHide(e)}),T.on("tooltipHide",function(){o&&e.tooltip.cleanup()}),C.dispatch=T,C.lines1=d,C.lines2=v,C.bars1=m,C.bars2=g,C.stack1=y,C.stack2=b,C.xAxis=w,C.yAxis1=E,C.yAxis2=S,C.options=e.utils.optionsFunc.bind(C),C.x=function(e){return arguments.length?(getX=e,d.x(e),m.x(e),C):getX},C.y=function(e){return arguments.length?(getY=e,d.y(e),m.y(e),C):getY},C.yDomain1=function(e){return arguments.length?(l=e,C):l},C.yDomain2=function(e){return arguments.length?(c=e,C):c},C.margin=function(e){return arguments.length?(t=e,C):t},C.width=function(e){return arguments.length?(r=e,C):r},C.height=function(e){return arguments.length?(i=e,C):i},C.color=function(e){return arguments.length?(n=e,x.color(e),C):n},C.showLegend=function(e){return arguments.length?(s=e,C):s},C.tooltips=function(e){return arguments.length?(o=e,C):o},C.tooltipContent=function(e){return arguments.length?(u=e,C):u},C},e.models.ohlcBar=function(){"use strict";function x(e){return e.each(function(e){var g=n-t.left-t.right,x=r-t.top-t.bottom,T=d3.select(this);s.domain(y||d3.extent(e[0].values.map(u).concat(p))),v?s.range(w||[g*.5/e[0].values.length,g*(e[0].values.length-.5)/e[0].values.length]):s.range(w||[0,g]),o.domain(b||[d3.min(e[0].values.map(h).concat(d)),d3.max(e[0].values.map(c).concat(d))]).range(E||[x,0]),s.domain()[0]===s.domain()[1]&&(s.domain()[0]?s.domain([s.domain()[0]-s.domain()[0]*.01,s.domain()[1]+s.domain()[1]*.01]):s.domain([-1,1])),o.domain()[0]===o.domain()[1]&&(o.domain()[0]?o.domain([o.domain()[0]+o.domain()[0]*.01,o.domain()[1]-o.domain()[1]*.01]):o.domain([-1,1]));var N=d3.select(this).selectAll("g.nv-wrap.nv-ohlcBar").data([e[0].values]),C=N.enter().append("g").attr("class","nvd3 nv-wrap nv-ohlcBar"),k=C.append("defs"),L=C.append("g"),A=N.select("g");L.append("g").attr("class","nv-ticks"),N.attr("transform","translate("+t.left+","+t.top+")"),T.on("click",function(e,t){S.chartClick({data:e,index:t,pos:d3.event,id:i})}),k.append("clipPath").attr("id","nv-chart-clip-path-"+i).append("rect"),N.select("#nv-chart-clip-path-"+i+" rect").attr("width",g).attr("height",x),A.attr("clip-path",m?"url(#nv-chart-clip-path-"+i+")":"");var O=N.select(".nv-ticks").selectAll(".nv-tick").data(function(e){return e});O.exit().remove();var M=O.enter().append("path").attr("class",function(e,t,n){return(f(e,t)>l(e,t)?"nv-tick negative":"nv-tick positive")+" nv-tick-"+n+"-"+t}).attr("d",function(t,n){var r=g/e[0].values.length*.9;return"m0,0l0,"+(o(f(t,n))-o(c(t,n)))+"l"+ -r/2+",0l"+r/2+",0l0,"+(o(h(t,n))-o(f(t,n)))+"l0,"+(o(l(t,n))-o(h(t,n)))+"l"+r/2+",0l"+ -r/2+",0z"}).attr("transform",function(e,t){return"translate("+s(u(e,t))+","+o(c(e,t))+")"}).on("mouseover",function(t,n){d3.select(this).classed("hover",!0),S.elementMouseover({point:t,series:e[0],pos:[s(u(t,n)),o(a(t,n))],pointIndex:n,seriesIndex:0,e:d3.event})}).on("mouseout",function(t,n){d3.select(this).classed("hover",!1),S.elementMouseout({point:t,series:e[0],pointIndex:n,seriesIndex:0,e:d3.event})}).on("click",function(e,t){S.elementClick({value:a(e,t),data:e,index:t,pos:[s(u(e,t)),o(a(e,t))],e:d3.event,id:i}),d3.event.stopPropagation()}).on("dblclick",function(e,t){S.elementDblClick({value:a(e,t),data:e,index:t,pos:[s(u(e,t)),o(a(e,t))],e:d3.event,id:i}),d3.event.stopPropagation()});O.attr("class",function(e,t,n){return(f(e,t)>l(e,t)?"nv-tick negative":"nv-tick positive")+" nv-tick-"+n+"-"+t}),d3.transition(O).attr("transform",function(e,t){return"translate("+s(u(e,t))+","+o(c(e,t))+")"}).attr("d",function(t,n){var r=g/e[0].values.length*.9;return"m0,0l0,"+(o(f(t,n))-o(c(t,n)))+"l"+ -r/2+",0l"+r/2+",0l0,"+(o(h(t,n))-o(f(t,n)))+"l0,"+(o(l(t,n))-o(h(t,n)))+"l"+r/2+",0l"+ -r/2+",0z"})}),x}var t={top:0 +,right:0,bottom:0,left:0},n=960,r=500,i=Math.floor(Math.random()*1e4),s=d3.scale.linear(),o=d3.scale.linear(),u=function(e){return e.x},a=function(e){return e.y},f=function(e){return e.open},l=function(e){return e.close},c=function(e){return e.high},h=function(e){return e.low},p=[],d=[],v=!1,m=!0,g=e.utils.defaultColor(),y,b,w,E,S=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout");return x.dispatch=S,x.options=e.utils.optionsFunc.bind(x),x.x=function(e){return arguments.length?(u=e,x):u},x.y=function(e){return arguments.length?(a=e,x):a},x.open=function(e){return arguments.length?(f=e,x):f},x.close=function(e){return arguments.length?(l=e,x):l},x.high=function(e){return arguments.length?(c=e,x):c},x.low=function(e){return arguments.length?(h=e,x):h},x.margin=function(e){return arguments.length?(t.top=typeof e.top!="undefined"?e.top:t.top,t.right=typeof e.right!="undefined"?e.right:t.right,t.bottom=typeof e.bottom!="undefined"?e.bottom:t.bottom,t.left=typeof e.left!="undefined"?e.left:t.left,x):t},x.width=function(e){return arguments.length?(n=e,x):n},x.height=function(e){return arguments.length?(r=e,x):r},x.xScale=function(e){return arguments.length?(s=e,x):s},x.yScale=function(e){return arguments.length?(o=e,x):o},x.xDomain=function(e){return arguments.length?(y=e,x):y},x.yDomain=function(e){return arguments.length?(b=e,x):b},x.xRange=function(e){return arguments.length?(w=e,x):w},x.yRange=function(e){return arguments.length?(E=e,x):E},x.forceX=function(e){return arguments.length?(p=e,x):p},x.forceY=function(e){return arguments.length?(d=e,x):d},x.padData=function(e){return arguments.length?(v=e,x):v},x.clipEdge=function(e){return arguments.length?(m=e,x):m},x.color=function(t){return arguments.length?(g=e.utils.getColor(t),x):g},x.id=function(e){return arguments.length?(i=e,x):i},x},e.models.pie=function(){"use strict";function S(e){return e.each(function(e){function q(e){var t=(e.startAngle+e.endAngle)*90/Math.PI-90;return t>90?t-180:t}function R(e){e.endAngle=isNaN(e.endAngle)?0:e.endAngle,e.startAngle=isNaN(e.startAngle)?0:e.startAngle,m||(e.innerRadius=0);var t=d3.interpolate(this._current,e);return this._current=t(0),function(e){return A(t(e))}}function U(e){e.innerRadius=0;var t=d3.interpolate({startAngle:0,endAngle:0},e);return function(e){return A(t(e))}}var o=n-t.left-t.right,f=r-t.top-t.bottom,S=Math.min(o,f)/2,x=S-S/5,T=d3.select(this),N=T.selectAll(".nv-wrap.nv-pie").data(e),C=N.enter().append("g").attr("class","nvd3 nv-wrap nv-pie nv-chart-"+u),k=C.append("g"),L=N.select("g");k.append("g").attr("class","nv-pie"),k.append("g").attr("class","nv-pieLabels"),N.attr("transform","translate("+t.left+","+t.top+")"),L.select(".nv-pie").attr("transform","translate("+o/2+","+f/2+")"),L.select(".nv-pieLabels").attr("transform","translate("+o/2+","+f/2+")"),T.on("click",function(e,t){E.chartClick({data:e,index:t,pos:d3.event,id:u})});var A=d3.svg.arc().outerRadius(x);y&&A.startAngle(y),b&&A.endAngle(b),m&&A.innerRadius(S*w);var O=d3.layout.pie().sort(null).value(function(e){return e.disabled?0:s(e)}),M=N.select(".nv-pie").selectAll(".nv-slice").data(O),_=N.select(".nv-pieLabels").selectAll(".nv-label").data(O);M.exit().remove(),_.exit().remove();var D=M.enter().append("g").attr("class","nv-slice").on("mouseover",function(e,t){d3.select(this).classed("hover",!0),E.elementMouseover({label:i(e.data),value:s(e.data),point:e.data,pointIndex:t,pos:[d3.event.pageX,d3.event.pageY],id:u})}).on("mouseout",function(e,t){d3.select(this).classed("hover",!1),E.elementMouseout({label:i(e.data),value:s(e.data),point:e.data,index:t,id:u})}).on("click",function(e,t){E.elementClick({label:i(e.data),value:s(e.data),point:e.data,index:t,pos:d3.event,id:u}),d3.event.stopPropagation()}).on("dblclick",function(e,t){E.elementDblClick({label:i(e.data),value:s(e.data),point:e.data,index:t,pos:d3.event,id:u}),d3.event.stopPropagation()});M.attr("fill",function(e,t){return a(e,t)}).attr("stroke",function(e,t){return a(e,t)});var P=D.append("path").each(function(e){this._current=e});M.select("path").transition().attr("d",A).attrTween("d",R);if(l){var H=d3.svg.arc().innerRadius(0);c&&(H=A),h&&(H=d3.svg.arc().outerRadius(A.outerRadius())),_.enter().append("g").classed("nv-label",!0).each(function(e,t){var n=d3.select(this);n.attr("transform",function(e){if(g){e.outerRadius=x+10,e.innerRadius=x+15;var t=(e.startAngle+e.endAngle)/2*(180/Math.PI);return(e.startAngle+e.endAngle)/2v?r[p]:""})}}),S}var t={top:0,right:0,bottom:0,left:0},n=500,r=500,i=function(e){return e.x},s=function(e){return e.y},o=function(e){return e.description},u=Math.floor(Math.random()*1e4),a=e.utils.defaultColor(),f=d3.format(",.2f"),l=!0,c=!0,h=!1,p="key",v=.02,m=!1,g=!1,y=!1,b=!1,w=.5,E=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout");return S.dispatch=E,S.options=e.utils.optionsFunc.bind(S),S.margin=function(e){return arguments.length?(t.top=typeof e.top!="undefined"?e.top:t.top,t.right=typeof e.right!="undefined"?e.right:t.right,t.bottom=typeof e.bottom!="undefined"?e.bottom:t.bottom,t.left=typeof e.left!="undefined"?e.left:t.left,S):t},S.width=function(e){return arguments.length?(n=e,S):n},S.height=function(e){return arguments.length?(r=e,S):r},S.values=function(t){return e.log("pie.values() is no longer supported."),S},S.x=function(e){return arguments.length?(i=e,S):i},S.y=function(e){return arguments.length?(s=d3.functor(e),S):s},S.description=function(e){return arguments.length?(o=e,S):o},S.showLabels=function(e){return arguments.length?(l=e,S):l},S.labelSunbeamLayout=function(e){return arguments.length?(g=e,S):g},S.donutLabelsOutside=function(e){return arguments.length?(h=e,S):h},S.pieLabelsOutside=function(e){return arguments.length?(c=e,S):c},S.labelType=function(e){return arguments.length?(p=e,p=p||"key",S):p},S.donut=function(e){return arguments.length?(m=e,S):m},S.donutRatio=function(e){return arguments.length?(w=e,S):w},S.startAngle=function(e){return arguments.length?(y=e,S):y},S.endAngle=function(e){return arguments.length?(b=e,S):b},S.id=function(e){return arguments.length?(u=e,S):u},S.color=function(t){return arguments.length?(a=e.utils.getColor(t),S):a},S.valueFormat=function(e){return arguments.length?(f=e,S):f},S.labelThreshold=function(e){return arguments.length?(v=e,S):v},S},e.models.pieChart=function(){"use strict";function v(e){return e.each(function(e){var u=d3.select(this),a=this,f=(i||parseInt(u.style("width"))||960)-r.left-r.right,d=(s||parseInt(u.style("height"))||400)-r.top-r.bottom;v.update=function(){u.transition().call(v)},v.container=this,l.disabled=e.map(function(e){return!!e.disabled});if(!c){var m;c={};for(m in l)l[m]instanceof Array?c[m]=l[m].slice(0):c[m]=l[m]}if(!e||!e.length){var g=u.selectAll(".nv-noData").data([h]);return g.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),g.attr("x",r.left+f/2).attr("y",r.top+d/2).text(function(e){return e}),v}u.selectAll(".nv-noData").remove();var y=u.selectAll("g.nv-wrap.nv-pieChart").data([e]),b=y.enter().append("g").attr("class","nvd3 nv-wrap nv-pieChart").append("g"),w=y.select("g");b.append("g").attr("class","nv-pieWrap"),b.append("g").attr("class","nv-legendWrap"),o&&(n.width(f).key(t.x()),y.select(".nv-legendWrap").datum(e).call(n),r.top!=n.height()&&(r.top=n.height(),d=(s||parseInt(u.style("height"))||400)-r.top-r.bottom),y.select(".nv-legendWrap").attr("transform","translate(0,"+ -r.top+")")),y.attr("transform","translate("+r.left+","+r.top+")"),t.width(f).height(d);var E=w.select(".nv-pieWrap").datum([e]);d3.transition(E).call(t),n.dispatch.on("stateChange",function(e){l=e,p.stateChange(l),v.update()}),t.dispatch.on("elementMouseout.tooltip",function(e){p.tooltipHide(e)}),p.on("changeState",function(t){typeof t.disabled!="undefined"&&(e.forEach(function(e,n){e.disabled=t.disabled[n]}),l.disabled=t.disabled),v.update()})}),v}var t=e.models.pie(),n=e.models.legend(),r={top:30,right:20,bottom:20,left:20},i=null,s=null,o=!0,u=e.utils.defaultColor(),a=!0,f=function(e,t,n,r){return"

    "+e+"

    "+"

    "+t+"

    "},l={},c=null,h="No Data Available.",p=d3.dispatch("tooltipShow","tooltipHide","stateChange","changeState"),d=function(n,r){var i=t.description()(n.point)||t.x()(n.point),s=n.pos[0]+(r&&r.offsetLeft||0),o=n.pos[1]+(r&&r.offsetTop||0),u=t.valueFormat()(t.y()(n.point)),a=f(i,u,n,v);e.tooltip.show([s,o],a,n.value<0?"n":"s",null,r)};return t.dispatch.on("elementMouseover.tooltip",function(e){}),p.on("tooltipShow",function(e){a&&d(e)}),p.on("tooltipHide",function(){a&&e.tooltip.cleanup()}),v.legend=n,v.dispatch=p,v.pie=t,d3.rebind(v,t,"valueFormat","values","x","y","description","id","showLabels","donutLabelsOutside","pieLabelsOutside","labelType","donut","donutRatio","labelThreshold"),v.options=e.utils.optionsFunc.bind(v),v.margin=function(e){return arguments.length?(r.top=typeof e.top!="undefined"?e.top:r.top,r.right=typeof e.right!="undefined"?e.right:r.right,r.bottom=typeof e.bottom!="undefined"?e.bottom:r.bottom,r.left=typeof e.left!="undefined"?e.left:r.left,v):r},v.width=function(e){return arguments.length?(i=e,v):i},v.height=function(e){return arguments.length?(s=e,v):s},v.color=function(r){return arguments.length?(u=e.utils.getColor(r),n.color(u),t.color(u),v):u},v.showLegend=function(e){return arguments.length?(o=e,v):o},v.tooltips=function(e){return arguments.length?(a=e,v):a},v.tooltipContent=function(e){return arguments.length?(f=e,v):f},v.state=function(e){return arguments.length?(l=e,v):l},v.defaultState=function(e){return arguments.length?(c=e,v):c},v.noData=function(e){return arguments.length?(h=e,v):h},v},e.models.scatter=function(){"use strict";function I(q){return q.each(function(I){function Q(){if(!g)return!1;var e,i=d3.merge(I.map(function(e,t){return e.values.map(function(e,n){var r=f(e,n),i=l(e,n);return[o(r)+Math.random()*1e-7,u(i)+Math.random()*1e-7,t,n,e]}).filter(function(e,t){return b(e[4],t)})}));if(D===!0){if(x){var a=X.select("defs").selectAll(".nv-point-clips").data([s]).enter();a.append("clipPath").attr("class","nv-point-clips").attr("id","nv-points-clip-"+s);var c=X.select("#nv-points-clip-"+s).selectAll("circle").data(i);c.enter().append("circle").attr("r",T),c.exit().remove(),c.attr("cx",function(e){return e[0]}).attr("cy",function(e){return e[1]}),X.select(".nv-point-paths").attr("clip-path","url(#nv-points-clip-"+s+")")}i.length&&(i.push([o.range()[0]-20,u.range()[0]-20,null,null]),i.push([o.range()[1]+20,u.range()[1]+20,null,null]),i.push([o.range()[0]-20,u.range()[0]+20,null,null]),i.push([o.range()[1]+20,u.range()[1]-20,null,null]));var h=d3.geom.polygon([[-10,-10],[-10,r+10],[n+10,r+10],[n+10,-10]]),p=d3.geom.voronoi(i).map(function(e,t){return{data:h.clip(e),series:i[t][2],point:i[t][3]}}),d=X.select(".nv-point-paths").selectAll("path").data(p);d.enter().append("path").attr("class",function(e,t){return"nv-path-"+t}),d.exit().remove(),d.attr("d",function(e){return e.data.length===0?"M 0 0":"M"+e.data.join("L")+"Z"});var v=function(e,n){if(F)return 0;var r=I[e.series];if(typeof r=="undefined")return;var i=r.values[e.point];n({point:i,series:r,pos:[o(f(i,e.point))+t.left,u(l(i,e.point))+t.top],seriesIndex:e.series,pointIndex:e.point})};d.on("click",function(e){v(e,_.elementClick)}).on("mouseover",function(e){v(e,_.elementMouseover)}).on("mouseout",function(e,t){v(e,_.elementMouseout)})}else X.select(".nv-groups").selectAll(".nv-group").selectAll(".nv-point").on("click",function(e,n){if(F||!I[e.series])return 0;var r=I[e.series],i=r.values[n];_.elementClick({point:i,series:r,pos:[o(f(i,n))+t.left,u(l(i,n))+t.top],seriesIndex:e.series,pointIndex:n})}).on("mouseover",function(e,n){if(F||!I[e.series])return 0;var r=I[e.series],i=r.values[n];_.elementMouseover({point:i,series:r,pos:[o(f(i,n))+t.left,u(l(i,n))+t.top],seriesIndex:e.series,pointIndex:n})}).on("mouseout",function(e,t){if(F||!I[e.series])return 0;var n=I[e.series],r=n.values[t];_.elementMouseout({point:r,series:n,seriesIndex:e.series,pointIndex:t})});F=!1}var q=n-t.left-t.right,R=r-t.top-t.bottom,U=d3.select(this);I.forEach(function(e,t){e.values.forEach(function(e){e.series=t})});var W=N&&C&&A?[]:d3.merge(I.map(function(e){return e.values.map(function(e,t){return{x:f(e,t),y:l(e,t),size:c(e,t)}})}));o.domain(N||d3.extent(W.map(function(e){return e.x}).concat(d))),w&&I[0]?o.range(k||[(q*E+q)/(2*I[0].values.length),q-q*(1+E)/(2*I[0].values.length)]):o.range(k||[0,q]),u.domain(C||d3.extent(W.map(function(e){return e.y}).concat(v))).range(L||[R,0]),a.domain(A||d3.extent(W.map(function(e){return e.size}).concat(m))).range(O||[16,256]);if(o.domain()[0]===o.domain()[1]||u.domain()[0]===u.domain()[1])M=!0;o.domain()[0]===o.domain()[1]&&(o.domain()[0]?o.domain([o.domain()[0]-o.domain()[0]*.01,o.domain()[1]+o.domain()[1]*.01]):o.domain([-1,1])),u.domain()[0]===u.domain()[1]&&(u.domain()[0]?u.domain([u.domain()[0]-u.domain()[0]*.01,u.domain()[1]+u.domain()[1]*.01]):u.domain([-1,1])),isNaN(o.domain()[0])&&o.domain([-1,1]),isNaN(u.domain()[0])&&u.domain([-1,1]),P=P||o,H=H||u,B=B||a;var X=U.selectAll("g.nv-wrap.nv-scatter").data([I]),V=X.enter().append("g").attr("class","nvd3 nv-wrap nv-scatter nv-chart-"+s+(M?" nv-single-point":"")),$=V.append("defs"),J=V.append("g"),K=X.select("g");J.append("g").attr("class","nv-groups"),J.append("g").attr("class","nv-point-paths"),X.attr("transform","translate("+t.left+","+t.top+")"),$.append("clipPath").attr("id","nv-edge-clip-"+s).append("rect"),X.select("#nv-edge-clip-"+s+" rect").attr("width",q).attr("height",R>0?R:0),K.attr("clip-path",S?"url(#nv-edge-clip-"+s+")":""),F=!0;var G=X.select(".nv-groups").selectAll(".nv-group").data(function(e){return e},function(e){return e.key});G.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6),G.exit().remove(),G.attr("class",function(e,t){return"nv-group nv-series-"+t}).classed("hover",function(e){return e.hover}),G.transition().style("fill",function(e,t){return i(e,t)}).style("stroke",function(e,t){return i(e,t)}).style("stroke-opacity",1).style("fill-opacity",.5);if(p){var Y=G.selectAll("circle.nv-point").data(function(e){return e.values},y);Y.enter().append("circle").style("fill",function(e,t){return e.color}).style("stroke",function(e,t){return e.color}).attr("cx",function(t,n){return e.utils.NaNtoZero(P(f(t,n)))}).attr("cy",function(t,n){return e.utils.NaNtoZero(H(l(t,n)))}).attr("r",function(e,t){return Math.sqrt(a(c(e,t))/Math.PI)}),Y.exit().remove(),G.exit().selectAll("path.nv-point").transition().attr("cx",function(t,n){return e.utils.NaNtoZero(o(f(t,n)))}).attr("cy",function(t,n){return e.utils.NaNtoZero(u(l(t,n)))}).remove(),Y.each(function(e,t){d3.select(this).classed("nv-point",!0).classed("nv-point-"+t,!0).classed("hover",!1)}),Y.transition().attr("cx",function(t,n){return e.utils.NaNtoZero(o(f(t,n)))}).attr("cy",function(t,n){return e.utils.NaNtoZero(u(l(t,n)))}).attr("r",function(e,t){return Math.sqrt(a(c(e,t))/Math.PI)})}else{var Y=G.selectAll("path.nv-point").data(function(e){return e.values});Y.enter().append("path").style("fill",function(e,t){return e.color}).style("stroke",function(e,t){return e.color}).attr("transform",function(e,t){return"translate("+P(f(e,t))+","+H(l(e,t))+")"}).attr("d",d3.svg.symbol().type(h).size(function(e,t){return a(c(e,t))})),Y.exit().remove(),G.exit().selectAll("path.nv-point").transition().attr("transform",function(e,t){return"translate("+o(f(e,t))+","+u(l(e,t))+")"}).remove(),Y.each(function(e,t){d3.select(this).classed("nv-point",!0).classed("nv-point-"+t,!0).classed("hover",!1)}),Y.transition().attr("transform",function(e,t){return"translate("+o(f(e,t))+","+u(l(e,t))+")"}).attr("d",d3.svg.symbol().type(h).size(function(e,t){return a(c(e,t))}))}clearTimeout(j),j=setTimeout(Q,300),P=o.copy(),H=u.copy(),B=a.copy()}),I}var t={top:0,right:0,bottom:0,left:0},n=960,r=500,i=e.utils.defaultColor(),s=Math.floor(Math.random()*1e5),o=d3.scale.linear(),u=d3.scale.linear(),a=d3.scale.linear(),f=function(e){return e.x},l=function(e){return e.y},c=function(e){return e.size||1},h=function(e){return e.shape||"circle"},p=!0,d=[],v=[],m=[],g=!0,y=null,b=function(e){return!e.notActive},w=!1,E=.1,S=!1,x=!0,T=function(){return 25},N=null,C=null,k=null,L=null,A=null,O=null,M=!1,_=d3.dispatch("elementClick","elementMouseover","elementMouseout"),D=!0,P,H,B,j,F=!1;return I.clearHighlights=function(){d3.selectAll(".nv-chart-"+s+" .nv-point.hover").classed("hover",!1)},I.highlightPoint=function(e,t,n){d3.select(".nv-chart-"+s+" .nv-series-"+e+" .nv-point-"+t).classed("hover",n)},_.on("elementMouseover.point",function(e){g&&I.highlightPoint(e.seriesIndex,e.pointIndex,!0)}),_.on("elementMouseout.point",function(e){g&&I.highlightPoint(e.seriesIndex,e.pointIndex,!1)}),I.dispatch=_,I.options=e.utils.optionsFunc.bind(I),I.x=function(e){return arguments.length?(f=d3.functor(e),I):f},I.y=function(e){return arguments.length?(l=d3.functor(e),I):l},I.size=function(e){return arguments.length?(c=d3.functor(e),I):c},I.margin=function(e){return arguments.length?(t.top=typeof e.top!="undefined"?e.top:t.top,t.right=typeof e.right!="undefined"?e.right:t.right,t.bottom=typeof e.bottom!="undefined"?e.bottom:t.bottom,t.left=typeof e.left!="undefined"?e.left:t.left,I):t},I.width=function(e){return arguments.length?(n=e,I):n},I.height=function(e){return arguments.length?(r=e,I):r},I.xScale=function(e){return arguments.length?(o=e,I):o},I.yScale=function(e){return arguments.length?(u=e,I):u},I.zScale=function(e){return arguments.length?(a=e,I):a},I.xDomain=function(e){return arguments.length?(N=e,I):N},I.yDomain=function(e){return arguments.length?(C=e,I):C},I.sizeDomain=function(e){return arguments.length?(A=e,I):A},I.xRange=function(e){return arguments.length?(k=e,I):k},I.yRange=function(e){return arguments.length?(L=e,I):L},I.sizeRange=function(e){return arguments.length?(O=e,I):O},I.forceX=function(e){return arguments.length?(d=e,I):d},I.forceY=function(e){return arguments.length?(v=e,I):v},I.forceSize=function(e){return arguments.length?(m=e,I):m},I.interactive=function(e){return arguments.length?(g=e,I):g},I.pointKey=function(e){return arguments.length?(y=e,I):y},I.pointActive=function(e){return arguments.length?(b=e,I):b},I.padData=function(e){return arguments.length?(w=e,I):w},I.padDataOuter=function(e){return arguments.length?(E=e,I):E},I.clipEdge=function(e){return arguments.length?(S=e,I):S},I.clipVoronoi=function(e){return arguments.length?(x=e,I):x},I.useVoronoi=function(e){return arguments.length?(D=e,D===!1&&(x=!1),I):D},I.clipRadius=function(e){return arguments.length?(T=e,I):T},I.color=function(t){return arguments.length?(i=e.utils.getColor(t),I):i},I.shape=function(e){return arguments.length?(h=e,I):h},I.onlyCircles=function(e){return arguments.length?(p=e,I):p},I.id=function(e){return arguments.length?(s=e,I):s},I.singlePoint=function(e){return arguments.length?(M=e,I):M},I},e.models.scatterChart=function(){"use strict";function F(e){return e.each(function(e){function K(){if(T)return X.select(".nv-point-paths").style("pointer-events","all"),!1;X.select(".nv-point-paths").style("pointer-events","none");var i=d3.mouse(this);h.distortion(x).focus(i[0]),p.distortion(x).focus(i[1]),X.select(".nv-scatterWrap").call(t),b&&X.select(".nv-x.nv-axis").call(n),w&&X.select(".nv-y.nv-axis").call(r),X.select(".nv-distributionX").datum(e.filter(function(e){return!e.disabled})).call(o),X.select(".nv-distributionY").datum(e.filter(function(e){return!e.disabled})).call(u)}var C=d3.select(this),k=this,L=(f||parseInt(C.style("width"))||960)-a.left-a.right,I=(l||parseInt(C.style("height"))||400)-a.top-a.bottom;F.update=function(){C.transition().duration(D).call(F)},F.container=this,A.disabled=e.map(function(e){return!!e.disabled});if(!O){var q;O={};for(q in A)A[q]instanceof Array?O[q]=A[q].slice(0):O[q]=A[q]}if(!e||!e.length||!e.filter(function(e){return e.values.length}).length){var R=C.selectAll(".nv-noData").data([_]);return R.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),R.attr("x",a.left+L/2).attr("y",a.top+I/2).text(function(e){return e}),F}C.selectAll(".nv-noData").remove(),P=P||h,H=H||p;var U=C.selectAll("g.nv-wrap.nv-scatterChart").data([e]),z=U.enter().append("g").attr("class","nvd3 nv-wrap nv-scatterChart nv-chart-"+t.id()),W=z.append("g"),X=U.select("g");W.append("rect").attr("class","nvd3 nv-background"),W.append("g").attr("class","nv-x nv-axis"),W.append("g").attr("class","nv-y nv-axis"),W.append("g").attr("class","nv-scatterWrap"),W.append("g").attr("class","nv-distWrap"),W.append("g").attr("class","nv-legendWrap"),W.append("g").attr("class","nv-controlsWrap");if(y){var V=S?L/2:L;i.width(V),U.select(".nv-legendWrap").datum(e).call(i),a.top!=i.height()&&(a.top=i.height(),I=(l||parseInt(C.style("height"))||400)-a.top-a.bottom),U.select(".nv-legendWrap").attr("transform","translate("+(L-V)+","+ -a.top+")")}S&&(s.width(180).color(["#444"]),X.select(".nv-controlsWrap").datum(j).attr("transform","translate(0,"+ -a.top+")").call(s)),U.attr("transform","translate("+a.left+","+a.top+")"),E&&X.select(".nv-y.nv-axis").attr("transform","translate("+L+",0)"),t.width(L).height(I).color(e.map(function(e,t){return e.color||c(e,t)}).filter(function(t,n){return!e[n].disabled})),d!==0&&t.xDomain(null),v!==0&&t.yDomain(null),U.select(".nv-scatterWrap").datum(e.filter(function(e){return!e.disabled})).call(t);if(d!==0){var $=h.domain()[1]-h.domain()[0];t.xDomain([h.domain()[0]-d*$,h.domain()[1]+d*$])}if(v!==0){var J=p.domain()[1]-p.domain()[0];t.yDomain([p.domain()[0]-v*J,p.domain()[1]+v*J])}(v!==0||d!==0)&&U.select(".nv-scatterWrap").datum(e.filter(function(e){return!e.disabled})).call(t),b&&(n.scale(h).ticks(n.ticks()&&n.ticks().length?n.ticks():L/100).tickSize(-I,0),X.select(".nv-x.nv-axis").attr("transform","translate(0,"+p.range()[0]+")").call(n)),w&&(r.scale(p).ticks(r.ticks()&&r.ticks().length?r.ticks():I/36).tickSize(-L,0),X.select(".nv-y.nv-axis").call(r)),m&&(o.getData(t.x()).scale(h).width(L).color(e.map(function(e,t){return e.color||c(e,t)}).filter(function(t,n){return!e[n].disabled})),W.select(".nv-distWrap").append("g").attr("class","nv-distributionX"),X.select(".nv-distributionX").attr("transform","translate(0,"+p.range()[0]+")").datum(e.filter(function(e){return!e.disabled})).call(o)),g&&(u.getData(t.y()).scale(p).width(I).color(e.map(function(e,t){return e.color||c(e,t)}).filter(function(t,n){return!e[n].disabled})),W.select(".nv-distWrap").append("g").attr("class","nv-distributionY"),X.select(".nv-distributionY").attr("transform","translate("+(E?L:-u.size())+",0)").datum(e.filter(function(e){return!e.disabled})).call(u)),d3.fisheye&&(X.select(".nv-background").attr("width",L).attr("height",I),X.select(".nv-background").on("mousemove",K),X.select(".nv-background").on("click",function(){T=!T}),t.dispatch.on("elementClick.freezeFisheye",function(){T=!T})),s.dispatch.on("legendClick",function(e,i){e.disabled=!e.disabled,x=e.disabled?0:2.5,X.select(".nv-background").style("pointer-events",e.disabled?"none":"all"),X.select(".nv-point-paths").style("pointer-events",e.disabled?"all":"none"),e.disabled?(h.distortion(x).focus(0),p.distortion(x).focus(0),X.select(".nv-scatterWrap").call(t),X.select(".nv-x.nv-axis").call(n),X.select(".nv-y.nv-axis").call(r)):T=!1,F.update()}),i.dispatch.on("stateChange",function(e){A.disabled=e.disabled,M.stateChange(A),F.update()}),t.dispatch.on("elementMouseover.tooltip",function(e){d3.select(".nv-chart-"+t.id()+" .nv-series-"+e.seriesIndex+" .nv-distx-"+e.pointIndex).attr("y1",function(t,n){return e.pos[1]-I}),d3.select(".nv-chart-"+t.id()+" .nv-series-"+e.seriesIndex+" .nv-disty-"+e.pointIndex).attr("x2",e.pos[0]+o.size()),e.pos=[e.pos[0]+a.left,e.pos[1]+a.top],M.tooltipShow(e)}),M.on("tooltipShow",function(e){N&&B(e,k.parentNode)}),M.on("changeState",function(t){typeof t.disabled!="undefined"&&(e.forEach(function(e,n){e.disabled=t.disabled[n]}),A.disabled=t.disabled),F.update()}),P=h.copy(),H=p.copy()}),F}var t=e.models.scatter(),n=e.models.axis(),r=e.models.axis(),i=e.models.legend(),s=e.models.legend(),o=e.models.distribution(),u=e.models.distribution(),a={top:30,right:20,bottom:50,left:75},f=null,l=null,c=e.utils.defaultColor(),h=d3.fisheye?d3.fisheye.scale(d3.scale.linear).distortion(0):t.xScale(),p=d3.fisheye?d3.fisheye.scale(d3.scale.linear).distortion(0):t.yScale(),d=0,v=0,m=!1,g=!1,y=!0,b=!0,w=!0,E=!1,S=!!d3.fisheye,x=0,T=!1,N=!0,C=function(e,t,n){return""+t+""},k=function(e,t,n){return""+n+""},L=null,A={},O=null,M=d3.dispatch("tooltipShow","tooltipHide","stateChange","changeState"),_="No Data Available.",D=250;t.xScale(h).yScale(p),n.orient("bottom").tickPadding(10),r.orient(E?"right":"left").tickPadding(10),o.axis("x"),u.axis("y"),s.updateState(!1);var P,H,B=function(i,s){var o=i.pos[0]+(s.offsetLeft||0),u=i.pos[1]+(s.offsetTop||0),f=i.pos[0]+(s.offsetLeft||0),l=p.range()[0]+a.top+(s.offsetTop||0),c=h.range()[0]+a.left+(s.offsetLeft||0),d=i.pos[1]+(s.offsetTop||0),v=n.tickFormat()(t.x()(i.point,i.pointIndex)),m=r.tickFormat()(t.y()(i.point,i.pointIndex));C!=null&&e.tooltip.show([f,l],C(i.series.key,v,m,i,F),"n",1,s,"x-nvtooltip"),k!=null&&e.tooltip.show([c,d],k(i.series.key,v,m,i,F),"e",1,s,"y-nvtooltip"),L!=null&&e.tooltip.show([o,u],L(i.series.key,v,m,i,F),i.value<0?"n":"s",null,s)},j=[{key:"Magnify",disabled:!0}];return t.dispatch.on("elementMouseout.tooltip",function(e){M.tooltipHide(e),d3.select(".nv-chart-"+t.id()+" .nv-series-"+e.seriesIndex+" .nv-distx-"+e.pointIndex).attr("y1",0),d3.select(".nv-chart-"+t.id()+" .nv-series-"+e.seriesIndex+" .nv-disty-"+e.pointIndex).attr("x2",u.size())}),M.on("tooltipHide",function(){N&&e.tooltip.cleanup()}),F.dispatch=M,F.scatter=t,F.legend=i,F.controls=s,F.xAxis=n,F.yAxis=r,F.distX=o,F.distY=u,d3.rebind(F,t,"id","interactive","pointActive","x","y","shape","size","xScale","yScale","zScale","xDomain","yDomain","xRange","yRange","sizeDomain","sizeRange","forceX","forceY","forceSize","clipVoronoi","clipRadius","useVoronoi"),F.options=e.utils.optionsFunc.bind(F),F.margin=function(e){return arguments.length?(a.top=typeof e.top!="undefined"?e.top:a.top,a.right=typeof e.right!="undefined"?e.right:a.right,a.bottom=typeof e.bottom!="undefined"?e.bottom:a.bottom,a.left=typeof e.left!="undefined"?e.left:a.left,F):a},F.width=function(e){return arguments.length?(f=e,F):f},F.height=function(e){return arguments.length?(l=e,F):l},F.color=function(t){return arguments.length?(c=e.utils.getColor(t),i.color(c),o.color(c),u.color(c),F):c},F.showDistX=function(e){return arguments.length?(m=e,F):m},F.showDistY=function(e){return arguments.length?(g=e,F):g},F.showControls=function(e){return arguments.length?(S=e,F):S},F.showLegend=function(e){return arguments.length?(y=e,F):y},F.showXAxis=function(e){return arguments.length?(b=e,F):b},F.showYAxis=function(e){return arguments.length?(w=e,F):w},F.rightAlignYAxis=function(e){return arguments.length?(E=e,r.orient(e?"right":"left"),F):E},F.fisheye=function(e){return arguments.length?(x=e,F):x},F.xPadding=function(e){return arguments.length?(d=e,F):d},F.yPadding=function(e){return arguments.length?(v=e,F):v},F.tooltips=function(e){return arguments.length?(N=e,F):N},F.tooltipContent=function(e){return arguments.length?(L=e,F):L},F.tooltipXContent=function(e){return arguments.length?(C=e,F):C},F.tooltipYContent=function(e){return arguments.length?(k=e,F):k},F.state=function(e){return arguments.length?(A=e,F):A},F.defaultState=function(e){return arguments.length?(O=e,F):O},F.noData=function(e){return arguments.length?(_=e,F):_},F.transitionDuration=function(e){return arguments.length?(D=e,F):D},F},e.models.scatterPlusLineChart=function(){"use strict";function B(e){return e.each(function(e){function $(){if(S)return z.select(".nv-point-paths").style("pointer-events","all"),!1;z.select(".nv-point-paths").style("pointer-events","none");var i=d3.mouse(this);h.distortion(E).focus(i[0]),p.distortion(E).focus(i[1]),z.select(".nv-scatterWrap").datum(e.filter(function(e){return!e.disabled})).call(t),g&&z.select(".nv-x.nv-axis").call(n),y&&z.select(".nv-y.nv-axis").call(r),z.select(".nv-distributionX").datum(e.filter(function(e){return!e.disabled})).call(o),z.select(".nv-distributionY").datum(e.filter(function(e){return!e.disabled})).call(u)}var T=d3.select(this),N=this,C=(f||parseInt(T.style("width"))||960)-a.left-a.right,j=(l||parseInt(T.style("height"))||400)-a.top-a.bottom;B.update=function(){T.transition().duration(M).call(B)},B.container=this,k.disabled=e.map(function(e){return!!e.disabled});if(!L){var F;L={};for(F in k)k[F]instanceof Array?L[F]=k[F].slice(0):L[F]=k[F]}if(!e||!e.length||!e.filter(function(e){return e.values.length}).length){var I=T.selectAll(".nv-noData").data([O]);return I.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),I.attr("x",a.left+C/2).attr("y",a.top+j/2).text(function(e){return e}),B}T.selectAll(".nv-noData").remove(),h=t.xScale(),p=t.yScale(),_=_||h,D=D||p;var q=T.selectAll("g.nv-wrap.nv-scatterChart").data([e]),R=q.enter().append("g").attr("class","nvd3 nv-wrap nv-scatterChart nv-chart-"+t.id()),U=R.append("g"),z=q.select("g");U.append("rect").attr("class","nvd3 nv-background").style("pointer-events","none"),U.append("g").attr("class","nv-x nv-axis"),U.append("g").attr("class","nv-y nv-axis"),U.append("g").attr("class","nv-scatterWrap"),U.append("g").attr("class","nv-regressionLinesWrap"),U.append("g").attr("class","nv-distWrap"),U.append("g").attr("class","nv-legendWrap"),U.append("g").attr("class","nv-controlsWrap"),q.attr("transform","translate("+a.left+","+a.top+")"),b&&z.select(".nv-y.nv-axis").attr("transform","translate("+C+",0)"),m&&(i.width(C/2),q.select(".nv-legendWrap").datum(e).call(i),a.top!=i.height()&&(a.top=i.height(),j=(l||parseInt(T.style("height"))||400)-a.top-a.bottom),q.select(".nv-legendWrap").attr("transform","translate("+C/2+","+ -a.top+")")),w&&(s.width(180).color(["#444"]),z.select(".nv-controlsWrap").datum(H).attr("transform","translate(0,"+ -a.top+")").call(s)),t.width(C).height(j).color(e.map(function(e,t){return e.color||c(e,t)}).filter(function(t,n){return!e[n].disabled})),q.select(".nv-scatterWrap").datum(e.filter(function(e){return!e.disabled})).call(t),q.select(".nv-regressionLinesWrap").attr("clip-path","url(#nv-edge-clip-"+t.id()+")");var W=q.select(".nv-regressionLinesWrap").selectAll(".nv-regLines").data(function(e){return e});W.enter().append("g").attr("class","nv-regLines");var X=W.selectAll(".nv-regLine").data(function(e){return[e]}),V=X.enter().append("line").attr("class","nv-regLine").style("stroke-opacity",0);X.transition().attr("x1",h.range()[0]).attr("x2",h.range()[1]).attr("y1",function(e,t){return p(h.domain()[0]*e.slope+e.intercept)}).attr("y2",function(e,t){return p(h.domain()[1]*e.slope+e.intercept)}).style("stroke",function(e,t,n){return c(e,n)}).style("stroke-opacity",function(e,t){return e.disabled||typeof e.slope=="undefined"||typeof e.intercept=="undefined"?0:1}),g&&(n.scale(h).ticks(n.ticks()?n.ticks():C/100).tickSize(-j,0),z.select(".nv-x.nv-axis").attr("transform","translate(0,"+p.range()[0]+")").call(n)),y&&(r.scale(p).ticks(r.ticks()?r.ticks():j/36).tickSize(-C,0),z.select(".nv-y.nv-axis").call(r)),d&&(o.getData(t.x()).scale(h).width(C).color(e.map(function(e,t){return e.color||c(e,t)}).filter(function(t,n){return!e[n].disabled})),U.select(".nv-distWrap").append("g").attr("class","nv-distributionX"),z.select(".nv-distributionX").attr("transform","translate(0,"+p.range()[0]+")").datum(e.filter(function(e){return!e.disabled})).call(o)),v&&(u.getData(t.y()).scale(p).width( +j).color(e.map(function(e,t){return e.color||c(e,t)}).filter(function(t,n){return!e[n].disabled})),U.select(".nv-distWrap").append("g").attr("class","nv-distributionY"),z.select(".nv-distributionY").attr("transform","translate("+(b?C:-u.size())+",0)").datum(e.filter(function(e){return!e.disabled})).call(u)),d3.fisheye&&(z.select(".nv-background").attr("width",C).attr("height",j),z.select(".nv-background").on("mousemove",$),z.select(".nv-background").on("click",function(){S=!S}),t.dispatch.on("elementClick.freezeFisheye",function(){S=!S})),s.dispatch.on("legendClick",function(e,i){e.disabled=!e.disabled,E=e.disabled?0:2.5,z.select(".nv-background").style("pointer-events",e.disabled?"none":"all"),z.select(".nv-point-paths").style("pointer-events",e.disabled?"all":"none"),e.disabled?(h.distortion(E).focus(0),p.distortion(E).focus(0),z.select(".nv-scatterWrap").call(t),z.select(".nv-x.nv-axis").call(n),z.select(".nv-y.nv-axis").call(r)):S=!1,B.update()}),i.dispatch.on("stateChange",function(e){k=e,A.stateChange(k),B.update()}),t.dispatch.on("elementMouseover.tooltip",function(e){d3.select(".nv-chart-"+t.id()+" .nv-series-"+e.seriesIndex+" .nv-distx-"+e.pointIndex).attr("y1",e.pos[1]-j),d3.select(".nv-chart-"+t.id()+" .nv-series-"+e.seriesIndex+" .nv-disty-"+e.pointIndex).attr("x2",e.pos[0]+o.size()),e.pos=[e.pos[0]+a.left,e.pos[1]+a.top],A.tooltipShow(e)}),A.on("tooltipShow",function(e){x&&P(e,N.parentNode)}),A.on("changeState",function(t){typeof t.disabled!="undefined"&&(e.forEach(function(e,n){e.disabled=t.disabled[n]}),k.disabled=t.disabled),B.update()}),_=h.copy(),D=p.copy()}),B}var t=e.models.scatter(),n=e.models.axis(),r=e.models.axis(),i=e.models.legend(),s=e.models.legend(),o=e.models.distribution(),u=e.models.distribution(),a={top:30,right:20,bottom:50,left:75},f=null,l=null,c=e.utils.defaultColor(),h=d3.fisheye?d3.fisheye.scale(d3.scale.linear).distortion(0):t.xScale(),p=d3.fisheye?d3.fisheye.scale(d3.scale.linear).distortion(0):t.yScale(),d=!1,v=!1,m=!0,g=!0,y=!0,b=!1,w=!!d3.fisheye,E=0,S=!1,x=!0,T=function(e,t,n){return""+t+""},N=function(e,t,n){return""+n+""},C=function(e,t,n,r){return"

    "+e+"

    "+"

    "+r+"

    "},k={},L=null,A=d3.dispatch("tooltipShow","tooltipHide","stateChange","changeState"),O="No Data Available.",M=250;t.xScale(h).yScale(p),n.orient("bottom").tickPadding(10),r.orient(b?"right":"left").tickPadding(10),o.axis("x"),u.axis("y"),s.updateState(!1);var _,D,P=function(i,s){var o=i.pos[0]+(s.offsetLeft||0),u=i.pos[1]+(s.offsetTop||0),f=i.pos[0]+(s.offsetLeft||0),l=p.range()[0]+a.top+(s.offsetTop||0),c=h.range()[0]+a.left+(s.offsetLeft||0),d=i.pos[1]+(s.offsetTop||0),v=n.tickFormat()(t.x()(i.point,i.pointIndex)),m=r.tickFormat()(t.y()(i.point,i.pointIndex));T!=null&&e.tooltip.show([f,l],T(i.series.key,v,m,i,B),"n",1,s,"x-nvtooltip"),N!=null&&e.tooltip.show([c,d],N(i.series.key,v,m,i,B),"e",1,s,"y-nvtooltip"),C!=null&&e.tooltip.show([o,u],C(i.series.key,v,m,i.point.tooltip,i,B),i.value<0?"n":"s",null,s)},H=[{key:"Magnify",disabled:!0}];return t.dispatch.on("elementMouseout.tooltip",function(e){A.tooltipHide(e),d3.select(".nv-chart-"+t.id()+" .nv-series-"+e.seriesIndex+" .nv-distx-"+e.pointIndex).attr("y1",0),d3.select(".nv-chart-"+t.id()+" .nv-series-"+e.seriesIndex+" .nv-disty-"+e.pointIndex).attr("x2",u.size())}),A.on("tooltipHide",function(){x&&e.tooltip.cleanup()}),B.dispatch=A,B.scatter=t,B.legend=i,B.controls=s,B.xAxis=n,B.yAxis=r,B.distX=o,B.distY=u,d3.rebind(B,t,"id","interactive","pointActive","x","y","shape","size","xScale","yScale","zScale","xDomain","yDomain","xRange","yRange","sizeDomain","sizeRange","forceX","forceY","forceSize","clipVoronoi","clipRadius","useVoronoi"),B.options=e.utils.optionsFunc.bind(B),B.margin=function(e){return arguments.length?(a.top=typeof e.top!="undefined"?e.top:a.top,a.right=typeof e.right!="undefined"?e.right:a.right,a.bottom=typeof e.bottom!="undefined"?e.bottom:a.bottom,a.left=typeof e.left!="undefined"?e.left:a.left,B):a},B.width=function(e){return arguments.length?(f=e,B):f},B.height=function(e){return arguments.length?(l=e,B):l},B.color=function(t){return arguments.length?(c=e.utils.getColor(t),i.color(c),o.color(c),u.color(c),B):c},B.showDistX=function(e){return arguments.length?(d=e,B):d},B.showDistY=function(e){return arguments.length?(v=e,B):v},B.showControls=function(e){return arguments.length?(w=e,B):w},B.showLegend=function(e){return arguments.length?(m=e,B):m},B.showXAxis=function(e){return arguments.length?(g=e,B):g},B.showYAxis=function(e){return arguments.length?(y=e,B):y},B.rightAlignYAxis=function(e){return arguments.length?(b=e,r.orient(e?"right":"left"),B):b},B.fisheye=function(e){return arguments.length?(E=e,B):E},B.tooltips=function(e){return arguments.length?(x=e,B):x},B.tooltipContent=function(e){return arguments.length?(C=e,B):C},B.tooltipXContent=function(e){return arguments.length?(T=e,B):T},B.tooltipYContent=function(e){return arguments.length?(N=e,B):N},B.state=function(e){return arguments.length?(k=e,B):k},B.defaultState=function(e){return arguments.length?(L=e,B):L},B.noData=function(e){return arguments.length?(O=e,B):O},B.transitionDuration=function(e){return arguments.length?(M=e,B):M},B},e.models.sparkline=function(){"use strict";function d(e){return e.each(function(e){var i=n-t.left-t.right,d=r-t.top-t.bottom,v=d3.select(this);s.domain(l||d3.extent(e,u)).range(h||[0,i]),o.domain(c||d3.extent(e,a)).range(p||[d,0]);var m=v.selectAll("g.nv-wrap.nv-sparkline").data([e]),g=m.enter().append("g").attr("class","nvd3 nv-wrap nv-sparkline"),b=g.append("g"),w=m.select("g");m.attr("transform","translate("+t.left+","+t.top+")");var E=m.selectAll("path").data(function(e){return[e]});E.enter().append("path"),E.exit().remove(),E.style("stroke",function(e,t){return e.color||f(e,t)}).attr("d",d3.svg.line().x(function(e,t){return s(u(e,t))}).y(function(e,t){return o(a(e,t))}));var S=m.selectAll("circle.nv-point").data(function(e){function n(t){if(t!=-1){var n=e[t];return n.pointIndex=t,n}return null}var t=e.map(function(e,t){return a(e,t)}),r=n(t.lastIndexOf(o.domain()[1])),i=n(t.indexOf(o.domain()[0])),s=n(t.length-1);return[i,r,s].filter(function(e){return e!=null})});S.enter().append("circle"),S.exit().remove(),S.attr("cx",function(e,t){return s(u(e,e.pointIndex))}).attr("cy",function(e,t){return o(a(e,e.pointIndex))}).attr("r",2).attr("class",function(e,t){return u(e,e.pointIndex)==s.domain()[1]?"nv-point nv-currentValue":a(e,e.pointIndex)==o.domain()[0]?"nv-point nv-minValue":"nv-point nv-maxValue"})}),d}var t={top:2,right:0,bottom:2,left:0},n=400,r=32,i=!0,s=d3.scale.linear(),o=d3.scale.linear(),u=function(e){return e.x},a=function(e){return e.y},f=e.utils.getColor(["#000"]),l,c,h,p;return d.options=e.utils.optionsFunc.bind(d),d.margin=function(e){return arguments.length?(t.top=typeof e.top!="undefined"?e.top:t.top,t.right=typeof e.right!="undefined"?e.right:t.right,t.bottom=typeof e.bottom!="undefined"?e.bottom:t.bottom,t.left=typeof e.left!="undefined"?e.left:t.left,d):t},d.width=function(e){return arguments.length?(n=e,d):n},d.height=function(e){return arguments.length?(r=e,d):r},d.x=function(e){return arguments.length?(u=d3.functor(e),d):u},d.y=function(e){return arguments.length?(a=d3.functor(e),d):a},d.xScale=function(e){return arguments.length?(s=e,d):s},d.yScale=function(e){return arguments.length?(o=e,d):o},d.xDomain=function(e){return arguments.length?(l=e,d):l},d.yDomain=function(e){return arguments.length?(c=e,d):c},d.xRange=function(e){return arguments.length?(h=e,d):h},d.yRange=function(e){return arguments.length?(p=e,d):p},d.animate=function(e){return arguments.length?(i=e,d):i},d.color=function(t){return arguments.length?(f=e.utils.getColor(t),d):f},d},e.models.sparklinePlus=function(){"use strict";function v(e){return e.each(function(c){function O(){if(a)return;var e=C.selectAll(".nv-hoverValue").data(u),r=e.enter().append("g").attr("class","nv-hoverValue").style("stroke-opacity",0).style("fill-opacity",0);e.exit().transition().duration(250).style("stroke-opacity",0).style("fill-opacity",0).remove(),e.attr("transform",function(e){return"translate("+s(t.x()(c[e],e))+",0)"}).transition().duration(250).style("stroke-opacity",1).style("fill-opacity",1);if(!u.length)return;r.append("line").attr("x1",0).attr("y1",-n.top).attr("x2",0).attr("y2",b),r.append("text").attr("class","nv-xValue").attr("x",-6).attr("y",-n.top).attr("text-anchor","end").attr("dy",".9em"),C.select(".nv-hoverValue .nv-xValue").text(f(t.x()(c[u[0]],u[0]))),r.append("text").attr("class","nv-yValue").attr("x",6).attr("y",-n.top).attr("text-anchor","start").attr("dy",".9em"),C.select(".nv-hoverValue .nv-yValue").text(l(t.y()(c[u[0]],u[0])))}function M(){function r(e,n){var r=Math.abs(t.x()(e[0],0)-n),i=0;for(var s=0;s2){var h=M.yScale().invert(i.mouseY),p=Infinity,d=null;c.forEach(function(e,t){h=Math.abs(h);var n=Math.abs(e.stackedValue.y0),r=Math.abs(e.stackedValue.y);if(h>=n&&h<=r+n){d=t;return}}),d!=null&&(c[d].highlight=!0)}var v=n.tickFormat()(M.x()(s,a)),m=t.style()=="expand"?function(e,t){return d3.format(".1%")(e)}:function(e,t){return r.tickFormat()(e)};o.tooltip.position({left:f+u.left,top:i.mouseY+u.top}).chartContainer(D.parentNode).enabled(g).valueFormatter(m).data({value:v,series:c})(),o.renderGuideLine(f)}),o.dispatch.on("elementMouseout",function(e){N.tooltipHide(),t.clearHighlights()}),N.on("tooltipShow",function(e){g&&O(e,D.parentNode)}),N.on("changeState",function(e){typeof e.disabled!="undefined"&&y.length===e.disabled.length&&(y.forEach(function(t,n){t.disabled=e.disabled[n]}),S.disabled=e.disabled),typeof e.style!="undefined"&&t.style(e.style),M.update()})}),M}var t=e.models.stackedArea(),n=e.models.axis(),r=e.models.axis(),i=e.models.legend(),s=e.models.legend(),o=e.interactiveGuideline(),u={top:30,right:25,bottom:50,left:60},a=null,f=null,l=e.utils.defaultColor(),c=!0,h=!0,p=!0,d=!0,v=!1,m=!1,g=!0,y=function(e,t,n,r,i){return"

    "+e+"

    "+"

    "+n+" on "+t+"

    "},b,w,E=d3.format(",.2f"),S={style:t.style()},x=null,T="No Data Available.",N=d3.dispatch("tooltipShow","tooltipHide","stateChange","changeState"),C=250,k=["Stacked","Stream","Expanded"],L={},A=250;n.orient("bottom").tickPadding(7),r.orient(v?"right":"left"),s.updateState(!1);var O=function(i,s){var o=i.pos[0]+(s.offsetLeft||0),u=i.pos[1]+(s.offsetTop||0),a=n.tickFormat()(t.x()(i.point,i.pointIndex)),f=r.tickFormat()(t.y()(i.point,i.pointIndex)),l=y(i.series.key,a,f,i,M);e.tooltip.show([o,u],l,i.value<0?"n":"s",null,s)};return t.dispatch.on("tooltipShow",function(e){e.pos=[e.pos[0]+u.left,e.pos[1]+u.top],N.tooltipShow(e)}),t.dispatch.on("tooltipHide",function(e){N.tooltipHide(e)}),N.on("tooltipHide",function(){g&&e.tooltip.cleanup()}),M.dispatch=N,M.stacked=t,M.legend=i,M.controls=s,M.xAxis=n,M.yAxis=r,M.interactiveLayer=o,d3.rebind(M,t,"x","y","size","xScale","yScale","xDomain","yDomain","xRange","yRange","sizeDomain","interactive","useVoronoi","offset","order","style","clipEdge","forceX","forceY","forceSize","interpolate"),M.options=e.utils.optionsFunc.bind(M),M.margin=function(e){return arguments.length?(u.top=typeof e.top!="undefined"?e.top:u.top,u.right=typeof e.right!="undefined"?e.right:u.right,u.bottom=typeof e.bottom!="undefined"?e.bottom:u.bottom,u.left=typeof e.left!="undefined"?e.left:u.left,M):u},M.width=function(e){return arguments.length?(a=e,M):a},M.height=function(e){return arguments.length?(f=e,M):f},M.color=function(n){return arguments.length?(l=e.utils.getColor(n),i.color(l),t.color(l),M):l},M.showControls=function(e){return arguments.length?(c=e,M):c},M.showLegend=function(e){return arguments.length?(h=e,M):h},M.showXAxis=function(e){return arguments.length?(p=e,M):p},M.showYAxis=function(e){return arguments.length?(d=e,M):d},M.rightAlignYAxis=function(e){return arguments.length?(v=e,r.orient(e?"right":"left"),M):v},M.useInteractiveGuideline=function(e){return arguments.length?(m=e,e===!0&&(M.interactive(!1),M.useVoronoi(!1)),M):m},M.tooltip=function(e){return arguments.length?(y=e,M):y},M.tooltips=function(e){return arguments.length?(g=e,M):g},M.tooltipContent=function(e){return arguments.length?(y=e,M):y},M.state=function(e){return arguments.length?(S=e,M):S},M.defaultState=function(e){return arguments.length?(x=e,M):x},M.noData=function(e){return arguments.length?(T=e,M):T},M.transitionDuration=function(e){return arguments.length?(A=e,M):A},M.controlsData=function(e){return arguments.length?(k=e,M):k},M.controlLabels=function(e){return arguments.length?typeof e!="object"?L:(L=e,M):L},r.setTickFormat=r.tickFormat,r.tickFormat=function(e){return arguments.length?(E=e,r):E},M}})(); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/views/chart.js b/config/plugins/visualizations/charts/static/views/chart.js new file mode 100644 index 00000000000..02a443974d5 --- /dev/null +++ b/config/plugins/visualizations/charts/static/views/chart.js @@ -0,0 +1,201 @@ +// dependencies +define(['library/portlet', 'library/table', 'library/ui', 'library/utils', 'models/chart', 'views/groups'], + function(Portlet, Table, Ui, Utils, Chart, GroupsView) { + +// widget +return Backbone.View.extend( +{ + // defaults options + optionsDefault: { + header : true, + content : 'No content available.' + }, + + // initialize + initialize: function(app, options) + { + // link application + this.app = app; + + // get current chart object + this.chart = this.app.chart; + + // configure options + this.options = Utils.merge(options, this.optionsDefault); + + // main elements + this.message = new Ui.Message(); + this.title = new Ui.Input({placeholder: 'Chart title'}); + this.dataset = new Ui.Input({value : app.options.dataset.id, disabled: true, visible: false}); + + // configure dataset + this.groups_view = new GroupsView(this.app); + + // table + var self = this; + this.table = new Table({ + header : false, + onconfirm : function(type) { + if (self.chart.groups.length > 0) { + // show modal + self.app.modal.show({ + title : 'Switching the chart type?', + body : 'You configured data sources. If you switch chart types your configurations will be removed.', + buttons : { + 'Cancel' : function() { + // hide modal + self.app.modal.hide(); + }, + 'Continue' : function() { + // hide modal + self.app.modal.hide(); + + // confirm + self.table.confirm(type); + } + } + }); + } else { + // confirm + self.table.confirm(type); + } + }, + onchange : function(type) { + // update chart type + self.chart.set({type: type}); + + // reset groups + self.chart.groups.reset(); + }, + content: 'No chart types available' + }); + + // add types + var types_n = 0; + var types = app.types.attributes; + for (var id in types){ + var chart_type = types[id]; + this.table.add (++types_n + '.'); + this.table.add (chart_type.title); + this.table.append(id); + } + + // add table to portlet + var self = this; + this.portlet = new Portlet({ + icon : 'fa-edit', + label : 'Create a new chart:', + operations : { + 'save' : new Ui.ButtonIcon({ + icon : 'fa-save', + tooltip : 'Save Chart', + onclick : function() { + self._saveChart(); + } + }), + 'back' : new Ui.ButtonIcon({ + icon : 'fa-caret-left', + tooltip : 'Return', + onclick : function() { + self.$el.hide(); + self.app.charts_view.$el.show(); + } + }) + } + }); + this.portlet.append(this.message.$el); + this.portlet.append(this.dataset.$el); + this.portlet.append((new Ui.Label({ label : 'Provide a chart title:'})).$el); + this.portlet.append(this.title.$el); + this.portlet.append((new Ui.Label({ label : 'Select a chart type:'})).$el); + this.portlet.append(this.table.$el); + this.portlet.append(this.groups_view.$el); + + // elements + this.setElement(this.portlet.$el); + + // hide back button on startup + this.portlet.hideOperation('back'); + + // model events + var self = this; + this.chart.on('change:title', function(chart) { + self.title.value(chart.get('title')); + }); + this.chart.on('change:type', function(chart) { + self.table.value(chart.get('type')); + }); + this.chart.on('reset', function(chart) { + self._resetChart(); + }); + + // collection events + this.app.charts.on('add', function(chart) { + self.portlet.showOperation('back'); + }); + this.app.charts.on('remove', function(chart) { + if (self.app.charts.length == 1) { + self.portlet.hideOperation('back'); + } + }); + this.app.charts.on('reset', function(chart) { + self.portlet.hideOperation('back'); + }); + + // reset + this._resetChart(); + }, + + // reset + _resetChart: function() { + this.chart.set('id', Utils.uuid()); + this.chart.set('dataset_id', this.app.options.dataset.id); + this.chart.set('type', 'bardiagram'); + this.chart.set('title', 'Chart title'); + }, + + // create chart + _saveChart: function() { + // update chart data + this.chart.set({ + type : this.table.value(), + title : this.title.value(), + dataset_id : this.dataset.value(), + date : Utils.time() + }); + + // validate + if (!this.chart.get('title')) { + this.message.update({message : 'Please enter a title for your chart.'}); + return; + } + + if (!this.chart.get('type')) { + this.message.update({message : 'Please select a chart type.'}); + return; + } + + if (this.chart.groups.length == 0) { + this.message.update({message : 'Please configure at least one data source.'}); + return; + } + + // create/get chart + var current = this.app.charts.get(this.chart.id); + if (!current) { + current = this.chart.clone(); + this.app.charts.add(current); + } + + // update chart model + current.copy(this.chart); + + // hide + this.$el.hide(); + + // update main + this.app.charts_view.$el.show(); + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/views/charts.js b/config/plugins/visualizations/charts/static/views/charts.js new file mode 100644 index 00000000000..2b137d85caa --- /dev/null +++ b/config/plugins/visualizations/charts/static/views/charts.js @@ -0,0 +1,133 @@ +// dependencies +define(['library/portlet', 'library/table', 'library/ui', 'library/utils'], function(Portlet, Table, Ui, Utils) { + +// widget +return Backbone.View.extend( +{ + // initialize + initialize: function(app, options) + { + // link app + this.app = app; + + // table + this.table = new Table({ + content : 'Add charts to this table.', + ondblclick : function(chart_id) { + // get chart + var chart = self.app.charts.get(chart_id); + self.app.chart.copy(chart); + + // show edit + self.$el.hide(); + + // update model and show create + self.app.chart_view.$el.show(); + }, + onchange : function(chart_id) { + self.app.viewport_view.show(chart_id); + } + }); + + // add table to portlet + var self = this; + this.portlet = new Portlet({ + icon : 'fa-list', + label : 'List of created charts:', + height : 100, + operations : { + 'new' : new Ui.ButtonIcon({ + icon : 'fa-plus', + tooltip: 'Create', + onclick: function() { + self.$el.hide(); + self.app.chart.reset(); + self.app.chart_view.$el.show(); + } + }), + 'delete' : new Ui.ButtonIcon({ + icon : 'fa-minus', + tooltip: 'Delete', + onclick: function() { + + // check if element has been selected + var chart_id = self.table.value(); + if (!chart_id) { + return; + } + + // title + var chart = self.app.charts.get(chart_id); + + // show modal + self.app.modal.show({ + title : 'Are you sure?', + body : 'The selected chart "' + chart.get('title') + '" will be irreversibly deleted.', + buttons : { + 'Cancel' : function() {self.app.modal.hide();}, + 'Delete' : function() { + // hide modal + self.app.modal.hide(); + + // remove chart + self.app.charts.remove(chart_id); + } + } + }); + } + }), + } + }); + this.portlet.append(this.table.$el); + + // append to main + if (!this.app.options.config.widget) { + this.$el.append(this.portlet.$el); + } + + // events + var self = this; + this.app.charts.on('add', function(chart) { + // add + self._addChart(chart); + }); + this.app.charts.on('remove', function(chart) { + // remove + self._removeChart(chart); + }); + this.app.charts.on('change', function(chart) { + // replace + self._removeChart(chart); + self._addChart(chart); + }); + }, + + // append + append : function($el) { + this.$el.append(Utils.wrap('')); + this.$el.append($el); + }, + + // add + _addChart: function(chart) { + // add title to table + this.table.add(chart.get('title')); + + // get chart type + var type = this.app.types.get(chart.get('type')); + this.table.add(type.title); + + // add additional columns + this.table.add('Last change: ' + chart.get('date')); + this.table.prepend(chart.get('id')); + this.table.value(chart.get('id')); + }, + + // remove + _removeChart: function(chart) { + // remove from to table + this.table.remove(chart.id); + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/views/group.js b/config/plugins/visualizations/charts/static/views/group.js new file mode 100644 index 00000000000..d4de13ffc9c --- /dev/null +++ b/config/plugins/visualizations/charts/static/views/group.js @@ -0,0 +1,219 @@ +// dependencies +define(['library/portlet', 'library/table', 'library/ui', 'library/utils'], + function(Portlet, Table, Ui, Utils) { + +// chart config +return Backbone.View.extend( +{ + // columns + columns: [], + + // initialize + initialize: function(app, options) { + // link app + this.app = app; + + // get current chart object + this.chart = this.app.chart; + this.group = this.app.group; + + // ui elements + this.message = new Ui.Message(); + this.label = new Ui.Input({placeholder: 'Group label'}); + this.table = new Table({content: 'No data column.'}); + + // add table to portlet + var self = this; + this.portlet = new Portlet({ + icon : 'fa-edit', + label : 'Define group properties:', + operations : { + 'save' : new Ui.ButtonIcon({ + icon : 'fa-save', + tooltip : 'Save', + onclick : function() { + // save/add group + self._saveGroup(); + } + }), + 'back' : new Ui.ButtonIcon({ + icon : 'fa-caret-left', + tooltip : 'Return', + onclick : function() { + self.$el.hide(); + self.app.chart_view.$el.show(); + } + }) + } + }); + this.portlet.append(this.message.$el); + this.portlet.append(this.label.$el); + this.portlet.append(this.table.$el); + + // add element + this.setElement(this.portlet.$el); + + // change + var self = this; + this.chart.on('change:dataset_id', function() { + self._refreshDataset(); + }); + this.chart.on('change:type', function() { + self._refreshType(); + }); + this.group.on('change:label', function() { + self._refreshLabel(); + }); + this.group.on('change', function() { + self._refreshGroup(); + }); + this.group.on('reset', function() { + self._resetGroup(); + }); + }, + + // show + show: function() { + this.$el.show(); + }, + + // reset + _resetGroup: function() { + this.group.set('id', Utils.uuid()); + this.group.set('label', 'Group label'); + }, + + // update dataset + _refreshDataset: function() { + // identify datasets + var dataset_id = this.chart.get('dataset_id'); + + // check if dataset is available + if (!dataset_id) { + return; + } + + // get dataset + var dataset = this.app.datasets.get({id : dataset_id}); + + // check + if (!dataset) { + this.app.log('Config::render()', 'Failed to retrieve dataset.'); + return; + } + + // configure columns + this.columns = []; + var meta = dataset.metadata_column_types; + for (var key in meta){ + this.columns.push({ + 'label' : key + ' [' + meta[key] + ']', + 'value' : key + }); + } + + // update select fields + for (var key in this.list) { + this.list[key].update(this.columns); + } + }, + + // update + _refreshType: function() { + // configure chart type + var self = this; + var chart_type = this.chart.get('type'); + if (chart_type) { + var chart_settings = this.app.types.get(chart_type); + + // table + this.table.removeAll(); + this.list = {}; + for (var id in chart_settings.data) { + // create select field + var data_def = chart_settings.data[id]; + var select = new Ui.Select({ + id : 'select_' + id, + gid : id, + data : this.columns, + onchange : function(value) { + self.group.set(this.gid, value); + } + }); + + // add row to table + this.table.add(data_def.title); + this.table.add(select.$el); + this.table.append(id); + + // add select field to list + this.list[id] = select; + } + } + }, + + // update + _refreshGroup: function() { + // update select fields + for (var id in this.list) { + var col = this.group.get(id); + if (col === undefined) { + col = 0; + } + this.list[id].value(col); + } + }, + + // update label + _refreshLabel: function() { + var label_text = this.group.get('label'); + if (label_text === undefined) { + label_text = ''; + } + this.label.value(label_text); + }, + + // create group + _saveGroup: function() { + // get current chart + var chart = this.chart; + + // update group object + var group = this.group; + for (var key in this.list) { + group.set(key, this.list[key].value()); + } + + // add label + group.set({ + dataset_id : this.chart.get('dataset_id'), + label : this.label.value(), + date : Utils.time() + }); + + // validate + if (!group.get('label')) { + this.message.update({message : 'Please enter a label for your group.'}); + return; + } + + // get groups of current chart + var groups = this.chart.groups; + + // create/update group + var group_update = groups.get(group.id); + if (group_update) { + group_update.set(group.attributes); + } else { + groups.add(group.clone()); + } + + // hide + this.$el.hide(); + + // update main + this.app.chart_view.$el.show(); + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/views/groups.js b/config/plugins/visualizations/charts/static/views/groups.js new file mode 100644 index 00000000000..bd155c53288 --- /dev/null +++ b/config/plugins/visualizations/charts/static/views/groups.js @@ -0,0 +1,154 @@ +// dependencies +define(['library/portlet', 'library/table', 'library/ui', 'library/utils'], function(Portlet, Table, Ui, Utils) { + +// chart config +return Backbone.View.extend( +{ + // initialize + initialize: function(app, options) { + // link app + this.app = app; + + // get current chart object + this.chart = this.app.chart; + + // table + this.table = new Table({ + content : 'Add data columns to this table.', + ondblclick : function(group_id) { + // get group + var group = self.app.chart.groups.get(group_id); + + // show edit + self.app.chart_view.$el.hide(); + self.app.group_view.show(); + self.app.group_view.group.set(group.attributes); + } + }); + + // add table to portlet + var self = this; + this.portlet = new Portlet({ + icon : '', + label : 'Select data columns:', + height : 100, + operations : { + 'new' : new Ui.ButtonIcon({ + icon : 'fa-plus', + tooltip: 'Create', + onclick: function() { + self.app.group.reset(); + self.app.chart_view.$el.hide(); + self.app.group_view.show(); + } + }), + 'edit' : new Ui.ButtonIcon({ + icon : 'fa-pencil', + tooltip: 'Edit', + onclick: function() { + // check if element has been selected + var group_id = self.table.value(); + if (!group_id) { + return; + } + + // get group + var group = self.app.chart.groups.get(group_id); + + // show edit + self.app.chart_view.$el.hide(); + self.app.group_view.show(); + self.app.group_view.group.set(group.attributes); + } + }), + 'delete' : new Ui.ButtonIcon({ + icon : 'fa-minus', + tooltip: 'Delete', + onclick: function() { + // check if element has been selected + var id = self.table.value(); + if (!id) { + return; + } + // remove group from chart + self.chart.groups.remove(id); + } + }) + } + }); + this.portlet.append(this.table.$el); + + // add element + this.setElement(this.portlet.$el); + + // change + var self = this; + this.chart.on('change', function() { + self._refresh(); + }); + this.chart.on('reset', function() { + self._removeGroupAll(); + }); + this.chart.on('change:type', function() { + self.app.group_view.trigger('change'); + }); + this.chart.groups.on('add', function(group) { + self._addGroup(group); + }); + this.chart.groups.on('remove', function(group) { + self._removeGroup(group); + }); + this.chart.groups.on('reset', function(group) { + self._removeGroupAll(); + }); + this.chart.groups.on('change', function(group) { + self._removeGroup(group); + self._addGroup(group); + }); + }, + + // refresh + _refresh: function() { + this._removeGroupAll(); + var self = this; + var groups = this.chart.groups; + if (groups) { + groups.each(function(group) { self._addGroup(group); }); + } + }, + + // add + _addGroup: function(group) { + // make custom info string + var info = '['; + var chart_type = this.chart.get('type'); + if (chart_type) { + var chart_settings = this.app.types.get(chart_type); + for (var key in chart_settings.data) { + info += key + '=' + group.get(key) + ', ' + } + } + info = info.substring(0, info.length - 2) + ']'; + + // add to table + this.table.add(group.get('label')); + this.table.add(info); + this.table.add('Last changed: ' + group.get('date')); + this.table.prepend(group.id); + this.table.value(group.id); + }, + + // remove + _removeGroup: function(group) { + // remove from to table + this.table.remove(group.id); + }, + + // data config reset + _removeGroupAll: function() { + // reset options table + this.table.removeAll(); + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/views/viewport.js b/config/plugins/visualizations/charts/static/views/viewport.js new file mode 100644 index 00000000000..82776ade415 --- /dev/null +++ b/config/plugins/visualizations/charts/static/views/viewport.js @@ -0,0 +1,231 @@ +// dependencies +define(['library/portlet', 'library/ui', 'library/utils'], + function(Portlet, Ui, Utils) { + +// widget +return Backbone.View.extend( +{ + // list + list: {}, + + // options + optionsDefault : + { + height : 300 + }, + + // initialize + initialize: function(app, options) + { + // link app + this.app = app; + + // link options + this.options = Utils.merge(options, this.optionsDefault); + + // add table to portlet + this.portlet = new Portlet({ + label : '', + icon : 'fa-signal', + height : this.options.height, + overflow : 'hidden', + operations : { + 'edit' : new Ui.ButtonIcon({ + icon : 'fa-gear', + tooltip : 'Configure' + }) + } + }); + + // set this element + this.setElement(this.portlet.$el); + + // events + var self = this; + + // add + this.app.charts.on('add', function(chart) { + self._addChart(chart); + }); + + // remove + this.app.charts.on('remove', function(chart) { + self._removeChart(chart.id); + }); + + // replace + this.app.charts.on('change', function(chart) { + self._removeChart(chart.id); + self._addChart(chart); + }); + }, + + // show + show: function(chart_id) { + // hide all + this.$el.find('svg').hide(); + + // identify selected item from list + var item = this.list[chart_id]; + if (item) { + // show selected chart + this.$el.find(item.svg_id).show(); + + // get chart + var chart = self.app.charts.get(chart_id); + + // update portlet + this.portlet.label(chart.get('title')); + this.portlet.setOperation('edit', function() { + // get chart + self.app.chart.copy(chart); + + // show edit + self.app.charts_view.$el.hide(); + self.app.chart_view.$el.show(); + }); + + // this trigger d3 update events + $(window).trigger('resize'); + } + }, + + // add + _addChart: function(chart) { + // link this + var self = this; + + // backup chart details + var chart_id = chart.id; + + // make sure that svg does not exist already + this._removeChart(chart_id); + + // create id + var svg_id = '#svg_' + chart_id; + + // create element + var chart_el = $(this._template({id: svg_id, height : this.options.height})); + + // add to portlet + this.portlet.append(chart_el); + + // backup id + this.list[chart_id] = { + svg_id : svg_id + } + + // identify chart type + var chart_type = chart.get('type'); + var chart_settings = this.app.types.get(chart_type); + + // create chart view + var self = this; + require(['charts/' + chart_type], function(ChartView) { + // create chart + var view = new ChartView(self.app, {svg_id : svg_id, chart : chart}); + + // reset chart data + var chart_data = []; + + // request data + var chart_index = 0; + chart.groups.each(function(group) { + + // add group data + chart_data.push({ + key : (chart_index + 1) + ':' + group.get('label'), + values : [] + }); + + // get data + for (var key in chart_settings.data) { + // configure request + var data_options = { + view : view, + data : chart_data, + index : chart_index, + dataset_id : group.get('dataset_id'), + column : group.get(key), + column_key : key + } + + // make request + self._data(data_options); + } + + // count + chart_index++; + }); + + // show + self.show(chart_id); + }); + }, + + // data + _data: function(options) { + // gather objects + var view = options.view; + var data = options.data; + + // gather parameters + var index = options.index; + var column = options.column; + var dataset_id = options.dataset_id; + var column_key = options.column_key; + var group_data = data[options.index]; + + // send request + self.app.datasets.get({id : dataset_id, column: column}, function(dataset) { + // select column + var values = dataset.values[column]; + var n_values = values.length; + + // read column values + var column_values = []; + for (var i = 0; i < n_values; i++) { + var value = values[i]; + if(!isNaN(value)) { + column_values.push(value); + } else { + column_values.push(0) + } + } + + // write column values + for (var i = 0; i < n_values; i++) { + // make sure dictionary exists + if (group_data.values[i] === undefined) { + group_data.values[i] = { + x : i, + y : 0 + } + } + + // write data + group_data.values[i][column_key] = column_values[i]; + } + + // refresh view + view.refresh(data); + }); + + }, + + // remove + _removeChart: function(id) { + var item = this.list[id]; + if (item) { + d3.select(item.svg_id).remove(); + $(item.svg_id).remove(); + } + }, + + // template + _template: function(options) { + return ''; + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/templates/charts.mako b/config/plugins/visualizations/charts/templates/charts.mako new file mode 100644 index 00000000000..9c0f139be24 --- /dev/null +++ b/config/plugins/visualizations/charts/templates/charts.mako @@ -0,0 +1,78 @@ +<% + scripts = [ 'jquery/jquery.js', + 'bootstrap.js', + 'require.js', + 'underscore.js', + 'backbone/backbone.js', + 'd3.js'] +%> + + + + + + + ${hda.name} | ${visualization_name} + + %for v in scripts: + + %endfor + + ## css + + + ## install nv.d3 module + + + + + + + + + diff --git a/config/plugins/visualizations/circster/config/circster.xml b/config/plugins/visualizations/circster/config/circster.xml index 04243861d5d..ee39db9eefa 100644 --- a/config/plugins/visualizations/circster/config/circster.xml +++ b/config/plugins/visualizations/circster/config/circster.xml @@ -1,6 +1,6 @@ - + HistoryDatasetAssociation @@ -24,5 +24,5 @@ webapps/galaxy/visualization - _top + _top diff --git a/config/plugins/visualizations/common/templates/README.txt b/config/plugins/visualizations/common/templates/README.txt new file mode 100644 index 00000000000..be42b2e01d9 --- /dev/null +++ b/config/plugins/visualizations/common/templates/README.txt @@ -0,0 +1,8 @@ +Common templates for visualization plugins +========================================== + +Placing Mako templates in this directory will allow them to be properly +inherited and imported in plugin directories. E.g. if you have a template file +in this directory named 'config_utils.mako', you can import it in your plugin +templates using: + <%namespace name="config_utils" file="config_utils.mako" /> diff --git a/config/plugins/visualizations/common/templates/config_utils.mako b/config/plugins/visualizations/common/templates/config_utils.mako new file mode 100644 index 00000000000..49f5e188604 --- /dev/null +++ b/config/plugins/visualizations/common/templates/config_utils.mako @@ -0,0 +1,46 @@ +<%def name="add_config_defaults( defaults )"> +## overwrite default_config_dict with config (if any) then assign to config +<% + for key, default in defaults.items(): + if key not in config or config[ key ] is None: + config[ key ] = default +%> + + +<%def name="config_form( config_dict )"> +## render form for everything in possible config + + +<%def name="link_to_change_config( link_contents, new_settings, target='' )"> +<% + # assumes there's a config var + url_for_args = { + 'controller' : 'visualization', + 'action' : 'render', + 'visualization_name' : visualization_name, + 'title' : title + } + url_for_args.update( config ) + url_for_args.update( new_settings ) + if visualization_id: + url_for_args[ 'id' ] = visualization_id +%> + ${link_contents} + + +<%def name="save_button( text='Save' )"> +<% + # still a GET + url_for_args = { + 'controller' : 'visualization', + 'action' : 'saved', + 'type' : visualization_name, + 'title' : title, + 'config' : h.to_json_string( config ) + } + # save to existing visualization + if visualization_id: + url_for_args[ 'id' ] = visualization_id +%> +
    + diff --git a/config/plugins/visualizations/visualization_base.mako b/config/plugins/visualizations/common/templates/visualization_base.mako similarity index 100% rename from config/plugins/visualizations/visualization_base.mako rename to config/plugins/visualizations/common/templates/visualization_base.mako diff --git a/config/plugins/visualizations/phyloviz/config/phyloviz.xml b/config/plugins/visualizations/phyloviz/config/phyloviz.xml index 656d88f0319..db0229ffa92 100644 --- a/config/plugins/visualizations/phyloviz/config/phyloviz.xml +++ b/config/plugins/visualizations/phyloviz/config/phyloviz.xml @@ -1,11 +1,12 @@ - + HistoryDatasetAssociation data.Newick data.Nexus + xml.Phyloxml dataset_id @@ -16,5 +17,5 @@ webapps/galaxy/visualization - _top + _top diff --git a/config/plugins/visualizations/scatterplot/Gruntfile.js b/config/plugins/visualizations/scatterplot/Gruntfile.js index 55b9a3b7a8d..bb05e229af7 100644 --- a/config/plugins/visualizations/scatterplot/Gruntfile.js +++ b/config/plugins/visualizations/scatterplot/Gruntfile.js @@ -6,9 +6,10 @@ module.exports = function(grunt) { pkg: grunt.file.readJSON( 'package.json' ), handlebars: { + // compile all hb templates into a single file in the build dir compile: { options: { - namespace: 'Templates', + namespace: 'scatterplot', processName : function( filepath ){ return filepath.match( /\w*\.handlebars/ )[0].replace( '.handlebars', '' ); } @@ -20,6 +21,7 @@ module.exports = function(grunt) { }, concat: { + // concat the template file and any js files in the src dir into a single file in the build dir options: { separator: ';\n' }, @@ -31,16 +33,19 @@ module.exports = function(grunt) { }, uglify: { + // uglify the concat single file directly into the static dir options: { + //mangle : false, + //beautify : true }, dist: { src : 'build/scatterplot-concat.js', - // uglify directly into static dir - dest: 'static/scatterplot.js' + dest: 'static/scatterplot-edit.js' } }, watch: { + // watch for changes in the src dir files: [ 'src/**.js', 'src/handlebars/*.handlebars' ], tasks: [ 'default' ] } @@ -52,5 +57,6 @@ module.exports = function(grunt) { grunt.loadNpmTasks( 'grunt-contrib-watch' ); grunt.registerTask( 'default', [ 'handlebars', 'concat', 'uglify' ]); - grunt.registerTask( 'watch', [ 'handlebars', 'concat', 'uglify', 'watch' ]); + // you can run grunt watch directly: + // grunt watch }; diff --git a/config/plugins/visualizations/scatterplot/config/scatterplot.xml b/config/plugins/visualizations/scatterplot/config/scatterplot.xml index d206f46b2a1..642b528dd0f 100644 --- a/config/plugins/visualizations/scatterplot/config/scatterplot.xml +++ b/config/plugins/visualizations/scatterplot/config/scatterplot.xml @@ -1,6 +1,6 @@ - + HistoryDatasetAssociation diff --git a/config/plugins/visualizations/scatterplot/package.json b/config/plugins/visualizations/scatterplot/package.json index d0c81baafb2..84909ef83bf 100644 --- a/config/plugins/visualizations/scatterplot/package.json +++ b/config/plugins/visualizations/scatterplot/package.json @@ -19,6 +19,6 @@ "grunt-contrib-handlebars": "~0.5.10", "grunt-contrib-concat": "~0.3.0", "grunt-contrib-uglify": "~0.2.2", - "grunt-contrib-watch": "~0.5.1" + "grunt-contrib-watch": "~0.5.3" } } diff --git a/config/plugins/visualizations/scatterplot/src/handlebars/chartControl.handlebars b/config/plugins/visualizations/scatterplot/src/handlebars/chartControl.handlebars deleted file mode 100644 index b08521fbc8d..00000000000 --- a/config/plugins/visualizations/scatterplot/src/handlebars/chartControl.handlebars +++ /dev/null @@ -1,56 +0,0 @@ -

    - Use the following controls to how the chart is displayed. - The slide controls can be moved by the mouse or, if the 'handle' is in focus, your keyboard's arrow keys. - Move the focus between controls by using the tab or shift+tab keys on your keyboard. - Use the 'Draw' button to render (or re-render) the chart with the current settings. -

    - -
    - -
    {{datapointSize}}
    -
    -

    - Size of the graphic representation of each data point -

    -
    - -
    - - -

    - Uncheck this to disable the animations used on the chart -

    -
    - -
    - -
    {{width}}
    -
    -

    - (not including chart margins and axes) -

    -
    - -
    - -
    {{height}}
    -
    -

    - (not including chart margins and axes) -

    -
    - -
    - - -

    -
    - -
    - - -

    -
    - - \ No newline at end of file diff --git a/config/plugins/visualizations/scatterplot/src/handlebars/chartDisplay.handlebars b/config/plugins/visualizations/scatterplot/src/handlebars/chartDisplay.handlebars deleted file mode 100644 index f1c54ebc2f0..00000000000 --- a/config/plugins/visualizations/scatterplot/src/handlebars/chartDisplay.handlebars +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/config/plugins/visualizations/scatterplot/src/handlebars/chartcontrol.handlebars b/config/plugins/visualizations/scatterplot/src/handlebars/chartcontrol.handlebars new file mode 100644 index 00000000000..8ac966fa7d0 --- /dev/null +++ b/config/plugins/visualizations/scatterplot/src/handlebars/chartcontrol.handlebars @@ -0,0 +1,47 @@ +

    + Use the following controls to how the chart is displayed. + The slide controls can be moved by the mouse or, if the 'handle' is in focus, your keyboard's arrow keys. + Move the focus between controls by using the tab or shift+tab keys on your keyboard. + Use the 'Draw' button to render (or re-render) the chart with the current settings. +

    + +
    + +
    {{datapointSize}}
    +
    +

    + Size of the graphic representation of each data point +

    +
    + +
    + +
    {{width}}
    +
    +

    + (not including chart margins and axes) +

    +
    + +
    + +
    {{height}}
    +
    +

    + (not including chart margins and axes) +

    +
    + +
    + + +

    +
    + +
    + + +

    +
    + + diff --git a/config/plugins/visualizations/scatterplot/src/handlebars/dataControl.handlebars b/config/plugins/visualizations/scatterplot/src/handlebars/dataControl.handlebars deleted file mode 100644 index 3da6dc4b946..00000000000 --- a/config/plugins/visualizations/scatterplot/src/handlebars/dataControl.handlebars +++ /dev/null @@ -1,56 +0,0 @@ -

    - Use the following controls to change the data used by the chart. - Use the 'Draw' button to render (or re-render) the chart with the current settings. -

    - - {{! column selector containers }} -
    - - -
    -
    - - -
    - - {{! optional id column }} -
    - - -

    - These will be displayed (along with the x and y values) when you hover over - a data point. -

    -
    - - - {{! if we're using generic column selection names ('column 1') - allow the user to use the first line }} - - - -
    \ No newline at end of file diff --git a/config/plugins/visualizations/scatterplot/src/handlebars/datacontrol.handlebars b/config/plugins/visualizations/scatterplot/src/handlebars/datacontrol.handlebars new file mode 100644 index 00000000000..2e7eb35dc8f --- /dev/null +++ b/config/plugins/visualizations/scatterplot/src/handlebars/datacontrol.handlebars @@ -0,0 +1,55 @@ +

    + Use the following controls to change the data used by the chart. + Use the 'Draw' button to render (or re-render) the chart with the current settings. +

    + +{{! column selector containers }} +
    + + +
    +
    + + +
    + +{{! optional id column }} +
    + + +

    + These will be displayed (along with the x and y values) when you hover over + a data point. +

    +
    + + +{{! if we're using generic column selection names ('column 1') - allow the user to use the first line }} + + + diff --git a/config/plugins/visualizations/scatterplot/src/handlebars/editor.handlebars b/config/plugins/visualizations/scatterplot/src/handlebars/editor.handlebars new file mode 100644 index 00000000000..5b0859bd07b --- /dev/null +++ b/config/plugins/visualizations/scatterplot/src/handlebars/editor.handlebars @@ -0,0 +1,42 @@ +
    + {{! tab buttons/headers using Bootstrap }} + + + {{! data form, chart config form, chart all get their own tab }} +
    + {{! ---------------------------- tab for data settings form }} +
    + {{! rendered separately }} +
    + + {{! ---------------------------- tab for chart graphics control form }} +
    + {{! rendered separately }} +
    + + {{! ---------------------------- tab for actual chart }} +
    + +
    {{! end .tab-content }} +
    {{! end .chart-control }} diff --git a/config/plugins/visualizations/scatterplot/src/handlebars/scatterplotControlForm.handlebars b/config/plugins/visualizations/scatterplot/src/handlebars/scatterplotControlForm.handlebars deleted file mode 100644 index f203450f49e..00000000000 --- a/config/plugins/visualizations/scatterplot/src/handlebars/scatterplotControlForm.handlebars +++ /dev/null @@ -1,46 +0,0 @@ -{{! main layout }} - -
    - {{! tab buttons/headers using Bootstrap }} - - - {{! data form, chart config form, stats, and chart all get their own tab }} -
    - {{! ---------------------------- tab for data settings form }} -
    - {{! rendered separately }} -
    - - {{! ---------------------------- tab for chart graphics control form }} -
    - {{! rendered separately }} -
    - - {{! ---------------------------- tab for data statistics }} -
    - {{! rendered separately }} -
    - - {{! ---------------------------- tab for actual chart }} -
    - {{! chart rendered separately }} -
    - -
    {{! end .tab-content }} -
    {{! end .chart-control }} \ No newline at end of file diff --git a/config/plugins/visualizations/scatterplot/src/handlebars/statsDisplay.handlebars b/config/plugins/visualizations/scatterplot/src/handlebars/statsDisplay.handlebars deleted file mode 100644 index 038b4ea7b71..00000000000 --- a/config/plugins/visualizations/scatterplot/src/handlebars/statsDisplay.handlebars +++ /dev/null @@ -1,8 +0,0 @@ -

    By column:

    - - - {{#each stats}} - - - {{/each}} -
    XY
    {{name}}{{xval}}{{yval}}
    \ No newline at end of file diff --git a/config/plugins/visualizations/scatterplot/src/scatterplot-config-editor.js b/config/plugins/visualizations/scatterplot/src/scatterplot-config-editor.js new file mode 100644 index 00000000000..1089f511a57 --- /dev/null +++ b/config/plugins/visualizations/scatterplot/src/scatterplot-config-editor.js @@ -0,0 +1,276 @@ +/* ============================================================================= +todo: + import button(display), func(model) - when user doesn't match + Move margins into wid/hi calcs (so final svg dims are w/h) + Better separation of AJAX in scatterplot.js (maybe pass in function?) + Labels should auto fill in chart control when dataset has column_names + Allow column selection/config using the peek output as a base for UI + Allow setting perPage in config + Allow option to auto set width/height based on screen real estate avail. + Handle large number of pages better (Known genes hg19) + Use d3.nest to allow grouping, pagination/filtration by group (e.g. chromCol) + Semantic HTML (figure, caption) + Save as SVG/png + Does it work w/ Galaxy.Frame? + Embedding + Small multiples + Drag & Drop other splots onto current (redraw with new axis and differentiate the datasets) + Remove 'chart' names + Somehow link out from info box? + + Subclass on specific datatypes? (vcf, cuffdiff, etc.) + What can be common/useful to other visualizations? + +============================================================================= */ +/** + * Scatterplot config control UI as a backbone view + * handles: + * configuring which data will be used + * configuring the plot display + */ +var ScatterplotConfigEditor = Backbone.View.extend( LoggableMixin ).extend({ + //TODO: !should be a view on a visualization model + //logger : console, + className : 'scatterplot-control-form', + + /** initialize requires a configuration Object containing a dataset Object */ + initialize : function( attributes ){ + if( !this.model ){ + this.model = new Visualization({ type: 'scatterplot' }); + } + this.log( this + '.initialize, attributes:', attributes ); + + if( !attributes || !attributes.dataset ){ + throw new Error( "ScatterplotConfigEditor requires a dataset" ); + } + this.dataset = attributes.dataset; + this.log( 'dataset:', this.dataset ); + + this.display = new ScatterplotDisplay({ + dataset : attributes.dataset, + model : this.model + }); + }, + + // ------------------------------------------------------------------------- CONTROLS RENDERING + render : function(){ + //console.log( this + '.render' ); + // render the tab controls, areas and loading indicator + this.$el.empty().append( ScatterplotConfigEditor.templates.mainLayout({})); + if( this.model.id ){ + this.$el.find( '.copy-btn' ).show(); + this.$el.find( '.save-btn' ).text( 'Update saved' ); + } + this.$el.find( '[title]' ).tooltip(); + + // render the tab content + this._render_dataControl(); + this._render_chartControls(); + this._render_chartDisplay(); + + // set up behaviours + + // auto render if given both x, y column choices + var config = this.model.get( 'config' ); + if( this.model.id && _.isFinite( config.xColumn ) && _.isFinite( config.yColumn ) ){ + this.renderChart(); + } + return this; + }, + + /** controls for which columns are used to plot datapoints (and ids/additional info to attach if desired) */ + _render_dataControl : function( $where ){ + //TODO: better handling of missing column names, column types + $where = $where || this.$el; + var editor = this, + dataset = this.dataset, + column_names = dataset.metadata_column_names || [], + config = this.model.get( 'config' ); + //console.log( 'metadata_column_types:', this.dataset.metadata_column_types ); + //console.log( 'metadata_column_names:', this.dataset.metadata_column_names ); + +//TODO: to peek based control + var numericColumns = [], + allColumns = _.map( dataset.metadata_column_types, function( type, i ){ + // save column data for select rendering, adding metadata name if available in dataset + var column = { index: i, type: type, name: ( column_names[ i ] || ( 'column ' + ( i + 1 )) ) }; + // also add column to numerics if numeric type + if( ( column.type === 'int' ) || ( column.type === 'float' ) ){ + numericColumns.push( column ); + } + return column; + }); + if( numericColumns.length < 2 ){ + numericColumns = allColumns; + } + //console.log( 'allColumns:', allColumns ); + //console.log( 'numericColumns:', numericColumns ); + + // render the html + var $dataControl = $where.find( '.tab-pane#data-control' ); + $dataControl.html( ScatterplotConfigEditor.templates.dataControl({ + allColumns : allColumns, + numericColumns : numericColumns + })); + +//TODO: column selection boilerplate + // preset to column selectors if they were passed in the config in the query string; set up events + var newConfig = { + xColumn : ( _.isFinite( config.xColumn ) )? ( config.xColumn ): ( numericColumns[0].index ), + yColumn : ( _.isFinite( config.yColumn ) )? ( config.yColumn ): ( numericColumns[1].index ), + idColumn : allColumns[0].index + }; + // use an idColumn from the config or attempt to get one different from the numeric + if( _.isFinite( config.idColumn ) ){ + newConfig.idColumn = config.idColumn; + } else { + if( allColumns.length > 2 ){ + var uniqueCol = _.find( allColumns, function( column, i ){ + return i !== newConfig.xColumn && i !== newConfig.yColumn; + }); + newConfig.idColumn = uniqueCol.index; + } + } + config = this.model.set( 'config', newConfig, { silent: true }).get( 'config' ); + + $dataControl.find( '[name="xColumn"]' ).val( config.xColumn ).on( 'change', function(){ + editor.model.set( 'config', { xColumn: Number( $( this ).val() ) }); + }); + $dataControl.find( '[name="yColumn"]' ).val( config.yColumn ).on( 'change', function(){ + editor.model.set( 'config', { yColumn: Number( $( this ).val() ) }); + }); + $dataControl.find( 'select[name="idColumn"]' ).val( config.idColumn ).on( 'change', function(){ + editor.model.set( 'config', { idColumn: Number( $( this ).val() ) }); + }); + if( config.idColumn !== undefined ){ + $dataControl.find( '#include-id-checkbox' ).prop( 'checked', true ).trigger( 'change' ); + } + + $dataControl.find( '[title]' ).tooltip(); + return $dataControl; + }, + + /** tab content to control how the chart is rendered (data glyph size, chart size, etc.) */ + _render_chartControls : function( $where ){ + $where = $where || this.$el; + var editor = this, + config = this.model.get( 'config' ), + $chartControls = $where.find( '#chart-control' ); + + // ---- skeleton/form for controls + $chartControls.html( ScatterplotConfigEditor.templates.chartControl( config ) ); + //console.debug( '$chartControl:', $chartControls ); + + // ---- slider controls + // limits for controls (by control/chartConfig id) + //TODO: as class attribute + var controlRanges = { + 'datapointSize' : { min: 2, max: 10, step: 1 }, + 'width' : { min: 200, max: 800, step: 20 }, + 'height' : { min: 200, max: 800, step: 20 } + }; + + function onSliderChange(){ + // set the model config when changed and update the slider output text + var $this = $( this ), + //note: returns a number nicely enough + newVal = $this.slider( 'value' ); + // parent of slide event target has html5 attr data-config-key + editor.model.set( 'config', _.object([[ $this.parent().data( 'config-key' ), newVal ]]) ); + $this.siblings( '.slider-output' ).text( newVal ); + } + + //console.debug( 'numeric sliders:', $chartControls.find( '.numeric-slider-input' ) ); + $chartControls.find( '.numeric-slider-input' ).each( function(){ + // set up the slider with control ranges, change event; set output text to initial value + var $this = $( this ), + configKey = $this.attr( 'data-config-key' ), + sliderSettings = _.extend( controlRanges[ configKey ], { + value : config[ configKey ], + change : onSliderChange, + slide : onSliderChange + }); + //console.debug( configKey + ' slider settings:', sliderSettings ); + $this.find( '.slider' ).slider( sliderSettings ); + $this.children( '.slider-output' ).text( config[ configKey ] ); + }); + + // ---- axes labels + var columnNames = this.dataset.metadata_column_names || []; + var xLabel = config.xLabel || columnNames[ config.xColumn ] || 'X'; + var yLabel = config.yLabel || columnNames[ config.yColumn ] || 'Y'; + // set label inputs to current x, y metadata_column_names (if any) + $chartControls.find( 'input[name="X-axis-label"]' ).val( xLabel ) + .on( 'change', function(){ + editor.model.set( 'config', { xLabel: $( this ).val() }); + }); + $chartControls.find( 'input[name="Y-axis-label"]' ).val( yLabel ) + .on( 'change', function(){ + editor.model.set( 'config', { yLabel: $( this ).val() }); + }); + + //console.debug( '$chartControls:', $chartControls ); + $chartControls.find( '[title]' ).tooltip(); + return $chartControls; + }, + + /** render the tab content where the chart is displayed (but not the chart itself) */ + _render_chartDisplay : function( $where ){ + $where = $where || this.$el; + var $chartDisplay = $where.find( '.tab-pane#chart-display' ); + this.display.setElement( $chartDisplay ); + this.display.render(); + + $chartDisplay.find( '[title]' ).tooltip(); + return $chartDisplay; + }, + + // ------------------------------------------------------------------------- EVENTS + events : { + 'change #include-id-checkbox' : 'toggleThirdColumnSelector', + 'click #data-control .render-button' : 'renderChart', + 'click #chart-control .render-button' : 'renderChart', + 'click .save-btn' : 'saveVisualization', + //'click .copy-btn' : function(e){ this.model.save(); } + }, + + saveVisualization : function(){ + var editor = this; + this.model.save() + .fail( function( xhr, status, message ){ + console.error( xhr, status, message ); + editor.trigger( 'save:error', view ); + alert( 'Error loading data:\n' + xhr.responseText ); + }) + .then( function(){ + editor.render(); + }); + }, + + toggleThirdColumnSelector : function(){ + // show/hide the id selector on the data settings panel + this.$el.find( 'select[name="idColumn"]' ).parent().toggle(); + }, + + // ------------------------------------------------------------------------- CHART/STATS RENDERING + renderChart : function(){ + //console.log( this + '.renderChart' ); + // fetch the data, (re-)render the chart + this.$el.find( '.nav li.disabled' ).removeClass( 'disabled' ); + this.$el.find( 'ul.nav' ).find( 'a[href="#chart-display"]' ).tab( 'show' ); + this.display.fetchData(); + //console.debug( this.display.$el ); + }, + + toString : function(){ + return 'ScatterplotConfigEditor(' + (( this.dataset )?( this.dataset.id ):( '' )) + ')'; + } +}); + +ScatterplotConfigEditor.templates = { + mainLayout : scatterplot.editor, + dataControl : scatterplot.datacontrol, + chartControl : scatterplot.chartcontrol +}; + +//============================================================================== diff --git a/config/plugins/visualizations/scatterplot/src/scatterplot-display.js b/config/plugins/visualizations/scatterplot/src/scatterplot-display.js new file mode 100644 index 00000000000..ea9927430d4 --- /dev/null +++ b/config/plugins/visualizations/scatterplot/src/scatterplot-display.js @@ -0,0 +1,285 @@ +// ============================================================================= +/** + * Scatterplot display control UI as a backbone view + * handles: + * fetching the data (if needed) + * computing and displaying data stats + * controls for pagination of data (if needed) + */ +var ScatterplotDisplay = Backbone.View.extend({ + + initialize : function( attributes ){ + this.data = null, + this.dataset = attributes.dataset; + this.calcNumPages(); + }, + + calcNumPages : function(){ + var config = this.model.get( 'config' ); + this.lineCount = this.dataset.metadata_data_lines, + this.numPages = ( this.lineCount )?( Math.ceil( this.lineCount / config.pagination.perPage ) ):( undefined ); + if( !this.lineCount || this.numPages === undefined ){ + console.warn( 'no data total found' ); + } + }, + + fetchData : function(){ +//TODO: doesn't work bc it's rendered in render()... + this.showLoadingIndicator( 'getting data' ); + //console.debug( 'currPage', this.config.pagination.currPage ); + var view = this, + config = this.model.get( 'config' ), +//TODO: very tied to datasets - should be generalized eventually + xhr = jQuery.getJSON( '/api/datasets/' + this.dataset.id, { + data_type : 'raw_data', + provider : 'dataset-column', + limit : config.pagination.perPage, + offset : ( config.pagination.currPage * config.pagination.perPage ) + }); + xhr.done( function( data ){ + view.data = data.data; + view.trigger( 'data:fetched', view ); + view.renderData(); + }); + xhr.fail( function( xhr, status, message ){ + console.error( xhr, status, message ); + view.trigger( 'data:error', view ); + alert( 'Error loading data:\n' + xhr.responseText ); + }); + return xhr; + }, + + showLoadingIndicator : function(){ + // display the loading indicator over the tab panels if hidden, update message (if passed) + this.$el.find( '.scatterplot-data-info' ).html([ + '
    ', + '', + 'loading...', + '
    ' + ].join( '' )); + }, + + template : function(){ + var html = [ + '
    ', + '
    ', + '
    ', + '
    ', + '

    ', + '', + '', + '
    ', + '
    ', + '', //TODO: id + '
    ' + ].join( '' ); + return html; + }, + + render : function(){ + this.$el.addClass( 'scatterplot-display' ).html( this.template() ); + if( this.data ){ + this.renderData(); + } + return this; + }, + + renderData : function(){ + this.renderLeftControls(); + this.renderRightControls(); + this.renderPlot( this.data ); + this.getStats(); + }, + + renderLeftControls : function(){ + if( this.lineCount ){ + this.$el.find( '.controls .left' ).empty().append( this.renderPagination() ); + } else { + this.$el.find( '.controls .left' ).empty().append( this.renderPrevNext() ); + } + return this; + }, + + renderRightControls : function(){ + var view = this; + this.setLineInfo( this.data ); + // clear prev. handlers due to closure around data + this.$el.find( '.stats-toggle-btn' ) + .off().click( function(){ + view.toggleStats(); + }); + this.$el.find( '.rerender-btn' ) + .off().click( function(){ + view.resetZoom(); + view.renderPlot( this.data ); + }); + }, + + /** render and show the d3 plot into the svg node of the view */ + renderPlot : function(){ + var view = this, + $svg = this.$el.find( 'svg' ); + // turn off stats, clear previous svg, and make it visible + this.toggleStats( false ); + $svg.off().empty().show() + // set up listeners for events from plot + .on( 'zoom.scatterplot', function( ev, zoom ){ + //TODO: possibly throttle this + //console.debug( 'zoom.scatterplot', zoom.scale, zoom.translate ); + view.model.set( 'config', zoom ); + }); + //TODO: may not be necessary to off/on this more than the initial on + // call the sep. d3 function to generate the plot + scatterplot( $svg.get( 0 ), this.model.get( 'config' ), this.data ); + }, + + setLineInfo : function( data, contents ){ + if( data ){ + var config = this.model.get( 'config' ), + totalLines = this.lineCount || 'an unknown total', + lineStart = config.pagination.currPage * config.pagination.perPage, + lineEnd = lineStart + data.length; + this.$el.find( '.controls p.scatterplot-data-info' ) + .text([ lineStart + 1, 'to', lineEnd, 'of', totalLines ].join( ' ' )); + } else { + this.$el.find( '.controls p.scatterplot-data-info' ).html( contents || '' ); + } + + return this; + }, + + resetZoom : function( scale, translate ){ + scale = ( scale !== undefined )?( scale ):( 1 ); + translate = ( translate !== undefined )?( translate ):( [ 0, 0 ] ); + this.model.set( 'config', { scale: scale, translate: translate } ); + return this; + }, + + // ------------------------------------------------------------------------ data pagination +//TODO: to pagination control + goToPage : function( page ){ + var pagination = this.model.get( 'config' ).pagination; + //console.debug( 'goToPage', page, pagination, this.numPages ); + if( page <= 0 ){ page = 0; } + if( this.numPages && page >= this.numPages ){ page = this.numPages - 1; } + if( page === pagination.currPage ){ return this; } + + //console.debug( '\t going to page ' + page ) + pagination.currPage = page; + this.model.set( 'config', { pagination: pagination }); + this.resetZoom(); + this.fetchData(); + return this; + }, + + nextPage : function(){ + var currPage = this.model.get( 'config' ).pagination.currPage; + return this.goToPage( currPage + 1 ); + }, + + prevPage : function(){ + var currPage = this.model.get( 'config' ).pagination.currPage; + return this.goToPage( currPage - 1 ); + }, + + /** render previous and next pagination buttons */ + renderPrevNext : function(){ + var config = this.model.get( 'config' ); + // if there's no data or there's less than one page of data - return null + if( !this.data ){ return null; } + if( config.pagination.currPage === 0 && this.data.length < config.pagination.perPage ){ return null; } + + var view = this, + $prev = $( '
  • Prev
  • ' ) + .click( function(){ view.prevPage(); }), + $next = $( '
  • Next
  • ' ) + .click( function(){ view.nextPage(); }); + + // disable if it either end + if( config.pagination.currPage === 0 ){ + $prev.addClass( 'disabled' ); + } + if( this.numPages && config.pagination.currPage === ( this.numPages - 1 ) ){ + $next.addClass( 'disabled' ); + } + return $( '
      ' ).addClass( 'pagination data-prev-next' ).append([ $prev, $next ]); + }, + + /** render page links for each possible page (if we can) */ + renderPagination : function(){ + var config = this.model.get( 'config' ); + // if there's no data, no page count, or there's less than one page of data - return null + if( !this.data ){ return null; } + if( !this.numPages ){ return null; } + if( config.pagination.currPage === 0 && this.data.length < config.pagination.perPage ){ return null; } + + var view = this, + $pagesList = $( '
        ' ).addClass( 'pagination data-pages' ); + pageNumClick = function( ev ){ + view.goToPage( $( this ).data( 'page' ) ); + }; + + for( var i=0; i', i + 1, '' ].join( '' )) + .attr( 'data-page', i ).click( pageNumClick ); + // highlight the current page + if( i === config.pagination.currPage ){ + $pageLi.addClass( 'active' ); + } + $pagesList.append( $pageLi ); + } + return $pagesList; + }, + + // ------------------------------------------------------------------------ statistics display + /** create a webworker to calc stats for data given */ + getStats : function(){ + if( !this.data ){ return; } + var view = this, + config = this.model.get( 'config' ), + meanWorker = new Worker( '/plugins/visualizations/scatterplot/static/worker-stats.js' ); + meanWorker.postMessage({ + data : this.data, + keys : [ config.xColumn, config.yColumn ] + }); + meanWorker.onerror = function( event ){ + meanWorker.terminate(); + }; + meanWorker.onmessage = function( event ){ + view.renderStats( event.data ); + }; + }, + + renderStats : function( stats, error ){ + //console.debug( 'renderStats:', stats, error ); + //console.debug( JSON.stringify( stats, null, ' ' ) ); + var config = this.model.get( 'config' ), + $statsTable = this.$el.find( '.stats-display' ), + xLabel = config.x.label, yLabel = config.y.label, + $table = $( '' ).addClass( 'table' ) + .append([ '' ].join( '' )) + .append( _.map( stats, function( stat, key ){ + return $([ '' ].join( '' )); + })); + $statsTable.empty().append( $table ); + }, + + toggleStats : function( showStats ){ + var $statsDisplay = this.$el.find( '.stats-display' ); + showStats = ( showStats === undefined )?( $statsDisplay.is( ':hidden' ) ):( showStats ); + if( showStats ){ + this.$el.find( 'svg' ).hide(); + $statsDisplay.show(); + this.$el.find( '.controls .stats-toggle-btn' ).text( 'Plot' ); + } else { + $statsDisplay.hide(); + this.$el.find( 'svg' ).show(); + this.$el.find( '.controls .stats-toggle-btn' ).text( 'Stats' ); + } + }, + + toString : function(){ + return 'ScatterplotView()'; + } +}); diff --git a/config/plugins/visualizations/scatterplot/src/scatterplot.js b/config/plugins/visualizations/scatterplot/src/scatterplot.js index d6527d301d8..e8c0660b5d1 100644 --- a/config/plugins/visualizations/scatterplot/src/scatterplot.js +++ b/config/plugins/visualizations/scatterplot/src/scatterplot.js @@ -1,488 +1,283 @@ -/* ============================================================================= -todo: - outside this: - BUG: setting width, height in plot controls doesn't re-interpolate data locations!! - BUG?: get metadata_column_names (from datatype if necessary) - BUG: single vis in popupmenu should have tooltip with that name NOT 'Visualizations' - - wire label setters, anim setter - - TwoVarScatterplot: - ??: maybe better to do this with a canvas... - save as visualization - to seperate file? - remove underscore dependencies - add interface to change values (seperate)? - download svg -> base64 encode - incorporate glyphs, glyph state renderers - - ScatterplotSettingsForm: - some css bug that lowers the width of settings form when plot-controls tab is open - causes chart to shift - what can be abstracted/reused for other graphs? - avoid direct manipulation of this.plot - allow option to put plot into seperate tab of interface (for small multiples) - - provide callback in view to load data incrementally - for large sets - paginate - handle rerender - use endpoint (here and on the server (fileptr)) - fetch (new?) data - handle rerender - use d3.TSV? - render warning on long data (> maxDataPoints) - adjust endpoint - - selectable list of preset column comparisons (rnaseq etc.) - how to know what sort of Tabular the data is? - smarter about headers - validate columns selection (here or server) - - set stats column names by selected columns - move chart into tabbed area... - - Scatterplot.mako: - multiple plots on one page (small multiples) - ?? ensure svg styles thru d3 or css? - d3: configable (easily) - css: standard - better maintenance - ? override at config - -============================================================================= */ +// ============================================================================= /** * Two Variable scatterplot visualization using d3 * Uses semi transparent circles to show density of data in x, y grid * usage : - * var plot = new TwoVarScatterplot({ containerSelector : 'div#my-plot', ... }) - * plot.render( xColumnData, yColumnData ); - * - * depends on: d3, underscore + * var plot = new scatterplot( $( 'svg' ).get(0), config, data ) */ -function TwoVarScatterplot( config ){ - var TICK_LINE_AND_PADDING = 10, - GUESS_AT_SVG_CHAR_WIDTH = 7, - GUESS_AT_SVG_CHAR_HEIGHT = 10, - PADDING = 8, - X_LABEL_TOO_LONG_AT = 5; - - // set up logging - //this.debugging = true; - this.log = function(){ - if( this.debugging && console && console.debug ){ - var args = Array.prototype.slice.call( arguments ); - args.unshift( this.toString() ); - console.debug.apply( console, args ); - } - }; - this.log( 'new TwoVarScatterplot:', config ); - - // ........................................................ set up chart config - // config will default to these values when not passed in - //NOTE: called on new - this.defaults = { - id : 'TwoVarScatterplot', - containerSelector : 'body', - //TODO??: needed? - maxDataPoints : 30000, - datapointSize : 4, - animDuration : 500, - //TODO: variable effect (not always exactly # of ticks set to) - xNumTicks : 10, - yNumTicks : 10, - xAxisLabelBumpY : 40, - yAxisLabelBumpX : -40, - width : 400, - height : 400, - //TODO: anyway to make this a sub-obj? - marginTop : 50, - marginRight : 50, - marginBottom : 50, - marginLeft : 50, - - xMin : null, - xMax : null, - yMin : null, - yMax : null, - - xLabel : "X", - yLabel : "Y" - }; - this.config = _.extend( {}, this.defaults, config ); - this.log( 'intial config:', this.config ); - - this.updateConfig = function( newConfig, rerender ){ - // setter for chart config - //TODO: validate here - _.extend( this.config, newConfig ); - this.log( this + '.updateConfig:', this.config ); - //TODO: implement rerender flag - }; - - // ........................................................ helpers - this.toString = function(){ - return this.config.id; - }; - // conv. methods for svg transforms - this.translateStr = function( x, y ){ - return 'translate(' + x + ',' + y + ')'; - }; - this.rotateStr = function( d, x, y ){ - return 'rotate(' + d + ',' + x + ',' + y + ')'; +function scatterplot( renderTo, config, data ){ + //console.log( 'scatterplot', config ); + + var translateStr = function( x, y ){ + return 'translate(' + x + ',' + y + ')'; + }, + rotateStr = function( d, x, y ){ + return 'rotate(' + d + ',' + x + ',' + y + ')'; + }, + getX = function( d, i ){ + //console.debug( d[ config.xColumn ] ); + return d[ config.xColumn ]; + }, + getY = function( d, i ){ + //console.debug( d[ config.yColumn ] ); + return d[ config.yColumn ]; + }; + + // .................................................................... scales + var stats = { + x : { extent: d3.extent( data, getX ) }, + y : { extent: d3.extent( data, getY ) } + }; + + //TODO: set pan/zoom limits + // from http://stackoverflow.com/questions/10422738/limiting-domain-when-zooming-or-panning-in-d3-js + //self.x.domain([Math.max(self.x.domain()[0], self.options.xmin), Math.min(self.x.domain()[1], self.options.xmax)]); + //self.y.domain([Math.max(self.y.domain()[0], self.options.ymin), Math.min(self.y.domain()[1], self.options.ymax)]); + var interpolaterFns = { + x : d3.scale.linear() + .domain( stats.x.extent ) + .range([ 0, config.width ]), + y : d3.scale.linear() + .domain( stats.y.extent ) + .range([ config.height, 0 ]) }; - // ........................................................ initial element creation - this.adjustChartDimensions = function( top, right, bottom, left ){ - //this.log( this + '.adjustChartDimensions', arguments ); - top = top || 0; - right = right || 0; - bottom = bottom || 0; - left = left || 0; - this.svg - .attr( "width", this.config.width + ( this.config.marginRight + right ) - + ( this.config.marginLeft + left ) ) - .attr( "height", this.config.height + ( this.config.marginTop + top ) - + ( this.config.marginBottom + bottom ) ) - // initial is hidden - show it - .style( 'display', 'block' ); - - // move content group away from margins - //TODO: allow top, right axis - this.content = this.svg.select( "g.content" ) - .attr( "transform", this.translateStr( this.config.marginLeft + left, this.config.marginTop + top ) ); - }; - - // ........................................................ data and scales - this.preprocessData = function( data, min, max ){ - //this.log( this + '.preprocessData', arguments ); - //TODO: filter by min, max if set - - // set a cap on the data, limit to first n points - return ( data.length > this.config.maxDataPoints )? ( data.slice( 0, this.config.maxDataPoints ) ): ( data ); - }; - - this.findMinMaxes = function( xCol, yCol, meta ){ - //this.log( this + '.findMinMaxes', arguments ); - // configuration takes priority, otherwise meta (from the server) if passed, last-resort: compute it here - this.xMin = this.config.xMin || ( meta )?( meta[0].min ):( d3.min( xCol ) ); - this.xMax = this.config.xMax || ( meta )?( meta[0].max ):( d3.max( xCol ) ); - this.yMin = this.config.yMin || ( meta )?( meta[1].min ):( d3.min( yCol ) ); - this.yMax = this.config.yMax || ( meta )?( meta[1].max ):( d3.max( yCol ) ); - }; - - this.setUpScales = function(){ - //this.log( this + '.setUpScales', arguments ); - // Interpolation for x, y based on data domains - this.xScale = d3.scale.linear() - .domain([ this.xMin, this.xMax ]) - .range([ 0, this.config.width ]), - this.yScale = d3.scale.linear() - .domain([ this.yMin, this.yMax ]) - .range([ this.config.height, 0 ]); - }; - - // ........................................................ axis and ticks - this.setUpXAxis = function(){ - //this.log( this + '.setUpXAxis', arguments ); - // origin: bottom, left - //TODO: incoporate top, right - this.xAxisFn = d3.svg.axis() - .scale( this.xScale ) - .ticks( this.config.xNumTicks ) - .orient( 'bottom' ); - this.xAxis// = content.select( 'g#x-axis' ) - .attr( 'transform', this.translateStr( 0, this.config.height ) ) - .call( this.xAxisFn ); - //this.log( 'xAxis:', this.xAxis ); - - //TODO: adjust ticks when tick labels are long - move odds down and extend tick line - // (for now) hide them - var xLongestTickLabel = d3.max( _.map( [ this.xMin, this.xMax ], - function( number ){ return ( String( number ) ).length; } ) ); - //this.log( 'xLongestTickLabel:', xLongestTickLabel ); - if( xLongestTickLabel >= X_LABEL_TOO_LONG_AT ){ - this.xAxis.selectAll( 'g' ).filter( ':nth-child(odd)' ).style( 'display', 'none' ); - } - - this.log( 'this.config.xLabel:', this.config.xLabel ); - this.xAxisLabel// = xAxis.select( 'text#x-axis-label' ) - .attr( 'x', this.config.width / 2 ) - .attr( 'y', this.config.xAxisLabelBumpY ) - .attr( 'text-anchor', 'middle' ) - .text( this.config.xLabel ); - this.log( 'xAxisLabel:', this.xAxisLabel ); + // .................................................................... main components + var zoom = d3.behavior.zoom() + .x( interpolaterFns.x ) + .y( interpolaterFns.y ) + .scaleExtent([ 1, 30 ]) + .scale( config.scale || 1 ) + .translate( config.translate || [ 0, 0 ] ); + + //console.debug( renderTo ); + var svg = d3.select( renderTo ) + .attr( "class", "scatterplot" ) + //.attr( "width", config.width + ( config.margin.right + config.margin.left ) ) + .attr( "width", '100%' ) + .attr( "height", config.height + ( config.margin.top + config.margin.bottom ) ); + + var content = svg.append( "g" ) + .attr( "class", "content" ) + .attr( "transform", translateStr( config.margin.left, config.margin.top ) ) + .call( zoom ); + + // a BIG gotcha - zoom (or any mouse/touch event in SVG?) requires the pointer to be over an object + // create a transparent rect to be that object here + content.append( 'rect' ) + .attr( "class", "zoom-rect" ) + .attr( "width", config.width ).attr( "height", config.height ) + .style( "fill", "transparent" ); + + //console.log( 'svg:', svg, 'content:', content ); + + // .................................................................... axes + var axis = { x : {}, y : {} }; + //console.log( 'x.ticks:', config.x.ticks ); + //console.log( 'y.ticks:', config.y.ticks ); + axis.x.fn = d3.svg.axis() + .orient( 'bottom' ) + .scale( interpolaterFns.x ) + .ticks( config.x.ticks ) + // this will convert thousands -> k, millions -> M, etc. + .tickFormat( d3.format( 's' ) ); + + axis.y.fn = d3.svg.axis() + .orient( 'left' ) + .scale( interpolaterFns.y ) + .ticks( config.y.ticks ) + .tickFormat( d3.format( 's' ) ); + + axis.x.g = content.append( 'g' ) + .attr( 'class', 'x axis' ) + .attr( 'transform', translateStr( 0, config.height ) ) + .call( axis.x.fn ); + //console.log( 'axis.x.g:', axis.x.g ); + + axis.y.g = content.append( 'g' ) + .attr( 'class', 'y axis' ) + .call( axis.y.fn ); + //console.log( 'axis.y.g:', axis.y.g ); + + // ................................ axis labels + var padding = 6; + // x-axis label + axis.x.label = svg.append( 'text' ) + .attr( 'class', 'axis-label' ) + .text( config.x.label ) + // align to the top-middle + .attr( 'text-anchor', 'middle' ) + .attr( 'dominant-baseline', 'text-after-edge' ) + .attr( 'x', ( config.width / 2 ) + config.margin.left ) + // place 4 pixels below the axis bounds + .attr( 'y', ( config.height + config.margin.bottom + config.margin.top ) - padding ); + //console.log( 'axis.x.label:', axis.x.label ); + +//TODO: anchor to left of x margin/graph + // y-axis label + // place 4 pixels left of the axis.y.g left edge + axis.y.label = svg.append( 'text' ) + .attr( 'class', 'axis-label' ) + .text( config.y.label ) + // align to bottom-middle + .attr( 'text-anchor', 'middle' ) + .attr( 'dominant-baseline', 'text-before-edge' ) + .attr( 'x', padding ) + .attr( 'y', config.height / 2 ) + // rotate around the alignment point + .attr( 'transform', rotateStr( -90, padding, config.height / 2 ) ); + //console.log( 'axis.y.label:', axis.y.label ); + + axis.redraw = function _redrawAxis(){ + svg.select( ".x.axis" ).call( axis.x.fn ); + svg.select( ".y.axis" ).call( axis.y.fn ); }; - this.setUpYAxis = function(){ - //this.log( this + '.setUpYAxis', arguments ); - this.yAxisFn = d3.svg.axis() - .scale( this.yScale ) - .ticks( this.config.yNumTicks ) - .orient( 'left' ); - this.yAxis// = content.select( 'g#y-axis' ) - .call( this.yAxisFn ); - //this.log( 'yAxis:', this.yAxis ); - - // a too complicated section for increasing the left margin when tick labels are long - // get the tick labels for the y axis - var yTickLabels = this.yAxis.selectAll( 'text' ).filter( function( e, i ){ return i !== 0; } ); - this.log( 'yTickLabels:', yTickLabels ); - - // get the longest label length (or 0 if no labels) - this.yLongestLabel = d3.max( - //NOTE: d3 returns an nested array - use the plain array inside ([0]) - yTickLabels[0].map( function( e, i ){ - return ( d3.select( e ).text() ).length; - }) - ) || 0; - //this.log( 'yLongestLabel:', this.yLongestLabel ); - //TODO: lose the guessing if possible - var neededY = TICK_LINE_AND_PADDING + ( this.yLongestLabel * GUESS_AT_SVG_CHAR_WIDTH ) - + PADDING + GUESS_AT_SVG_CHAR_HEIGHT; - //this.log( 'neededY:', neededY ); - - // increase width for yLongerStr, increase margin for y - //TODO??: (or transform each number: 2k) - this.config.yAxisLabelBumpX = -( neededY - GUESS_AT_SVG_CHAR_HEIGHT ); - if( this.config.marginLeft < neededY ){ - var adjusting = ( neededY ) - this.config.marginLeft; - adjusting = ( adjusting < 0 )?( 0 ):( adjusting ); - //this.log( 'adjusting:', adjusting ); - - // update dimensions, translations - this.adjustChartDimensions( 0, 0, 0, adjusting ); - } - //this.log( 'this.config.yAxisLableBumpx, this.config.marginLeft:', - // this.config.yAxisLabelBumpX, this.config.marginLeft ); - - this.yAxisLabel// = yAxis.select( 'text#y-axis-label' ) - .attr( 'x', this.config.yAxisLabelBumpX ) - .attr( 'y', this.config.height / 2 ) - .attr( 'text-anchor', 'middle' ) - .attr( 'transform', this.rotateStr( -90, this.config.yAxisLabelBumpX, this.config.height / 2 ) ) - .text( this.config.yLabel ); - //this.log( 'yAxisLabel:', this.yAxisLabel ); - }; - - // ........................................................ grid lines - this.renderGrid = function(){ - //this.log( this + '.renderGrid', arguments ); - // VERTICAL - // select existing - this.vGridLines = this.content.selectAll( 'line.v-grid-line' ) - .data( this.xScale.ticks( this.xAxisFn.ticks()[0] ) ); - - // append any extra lines needed (more ticks) - this.vGridLines.enter().append( 'svg:line' ) + // .................................................................... grid + function renderGrid(){ + var grid = { v : {}, h: {} }; + // vertical + grid.v.lines = content.selectAll( 'line.v-grid-line' ) + // data are the axis ticks; enter, update, exit + .data( interpolaterFns.x.ticks( axis.x.fn.ticks()[0] ) ); + // enter: append any extra lines needed (more ticks) + grid.v.lines.enter() + .append( 'svg:line' ) .classed( 'grid-line v-grid-line', true ); - - // update the attributes of existing and appended - this.vGridLines - .attr( 'x1', this.xScale ) + // update: set coords + grid.v.lines + .attr( 'x1', interpolaterFns.x ) + .attr( 'x2', interpolaterFns.x ) .attr( 'y1', 0 ) - .attr( 'x2', this.xScale ) - .attr( 'y2', this.config.height ); - - // remove unneeded (less ticks) - this.vGridLines.exit().remove(); - //this.log( 'vGridLines:', this.vGridLines ); + .attr( 'y2', config.height ); + // exit: just remove them + grid.v.lines.exit().remove(); + //console.log( 'grid.v.lines:', grid.v.lines ); - // HORIZONTAL - this.hGridLines = this.content.selectAll( 'line.h-grid-line' ) - .data( this.yScale.ticks( this.yAxisFn.ticks()[0] ) ); - - this.hGridLines.enter().append( 'svg:line' ) + // horizontal + grid.h.lines = content.selectAll( 'line.h-grid-line' ) + .data( interpolaterFns.y.ticks( axis.y.fn.ticks()[0] ) ); + grid.h.lines.enter() + .append( 'svg:line' ) .classed( 'grid-line h-grid-line', true ); - - this.hGridLines + grid.h.lines .attr( 'x1', 0 ) - .attr( 'y1', this.yScale ) - .attr( 'x2', this.config.width ) - .attr( 'y2', this.yScale ); - - this.hGridLines.exit().remove(); - //this.log( 'hGridLines:', this.hGridLines ); - }; - - // ........................................................ data points - this.renderDatapoints = function( xCol, yCol, ids ){ - this.log( this + '.renderDatapoints', arguments ); - var count = 0, - plot = this, - xPosFn = function( d, i ){ - //if( d ){ this.log( 'x.data:', newXCol[ i ], 'plotted:', plot.xScale( newXCol[ i ] ) ); } - return plot.xScale( xCol[ i ] ); - }, - yPosFn = function( d, i ){ - //if( d ){ this.log( 'y.data:', newYCol[ i ], 'plotted:', plot.yScale( newYCol[ i ] ) ); } - return plot.yScale( yCol[ i ] ); - }; + .attr( 'x2', config.width ) + .attr( 'y1', interpolaterFns.y ) + .attr( 'y2', interpolaterFns.y ); + grid.h.lines.exit().remove(); + //console.log( 'grid.h.lines:', grid.h.lines ); + return grid; + } + var grid = renderGrid(); - //this.datapoints = this.addDatapoints( xCol, yCol, ids, ".glyph" ); - var datapoints = this.content.selectAll( '.glyph' ).data( xCol ); + //// .................................................................... datapoints + var datapoints = content.selectAll( '.glyph' ).data( data ) + // enter - NEW data to be added as glyphs + .enter().append( 'svg:circle' ) + .classed( "glyph", true ) + .attr( "cx", function( d, i ){ return interpolaterFns.x( getX( d, i ) ); }) + .attr( "cy", function( d, i ){ return interpolaterFns.y( getY( d, i ) ); }) + .attr( "r", 0 ); - // enter - NEW data to be added as glyphs: give them a 'entry' position and style - count = 0; - datapoints.enter() - .append( 'svg:circle' ) - .each( function(){ count += 1; } ) - .classed( "glyph", true ) - .attr( "cx", 0 ) - .attr( "cy", this.config.height ) - // start all bubbles small... - .attr( "r", 0 ); - this.log( count, ' new glyphs created' ); - - // for all EXISTING glyphs and those that need to be added: transition anim to final state - count = 0; - datapoints - // ...animate to final position - .transition().duration( this.config.animDuration ) - .each( function(){ count += 1; } ) - .attr( "cx", xPosFn ) - .attr( "cy", yPosFn ) - .attr( "r", plot.config.datapointSize ); - this.log( count, ' existing glyphs transitioned' ); - - // events - // glyphs that need to be removed: transition to from normal state to 'exit' state, remove from DOM - datapoints.exit() - .each( function(){ count += 1; } ) - .transition().duration( this.config.animDuration ) - .attr( "cy", this.config.height ) - .attr( "r", 0 ) - .remove(); - this.log( count, ' glyphs removed' ); + // for all EXISTING glyphs and those that need to be added: transition anim to final state + datapoints.transition().duration( config.animDuration ) + .attr( "r", config.datapointSize ); + //console.log( 'datapoints:', datapoints ); - this._addDatapointEventhandlers( datapoints, xCol, yCol, ids ); - }; + function _redrawDatapointsClipped(){ + return datapoints + //TODO: interpolates twice + .attr( "cx", function( d, i ){ return interpolaterFns.x( getX( d, i ) ); }) + .attr( "cy", function( d, i ){ return interpolaterFns.y( getY( d, i ) ); }) + .style( 'display', 'block' ) + // filter out points now outside the graph content area and hide them + .filter( function( d, i ){ + var cx = d3.select( this ).attr( "cx" ), + cy = d3.select( this ).attr( "cy" ); + if( cx < 0 || cx > config.width ){ return true; } + if( cy < 0 || cy > config.height ){ return true; } + return false; + }).style( 'display', 'none' ); + } + _redrawDatapointsClipped(); - this._addDatapointEventhandlers = function( datapoints, xCol, yCol, ids ){ - var plot = this; - datapoints - //TODO: remove magic numbers - .on( 'mouseover', function( d, i ){ - var datapoint = d3.select( this ); - datapoint - .style( 'fill', 'red' ) - .style( 'fill-opacity', 1 ); - - // create horiz, vert lines to axis - plot.content.append( 'line' ) - .attr( 'stroke', 'red' ) - .attr( 'stroke-width', 1 ) - // start not at center, but at the edge of the circle - to prevent mouseover thrashing - .attr( 'x1', datapoint.attr( 'cx' ) - plot.config.datapointSize ) - .attr( 'y1', datapoint.attr( 'cy' ) ) - .attr( 'x2', 0 ) - .attr( 'y2', datapoint.attr( 'cy' ) ) - .classed( 'hoverline', true ); + // .................................................................... behaviors + function zoomed( scale, translateX, translateY ){ + //console.debug( 'zoom', this, zoom.scale(), zoom.translate() ); - // if the vertical hoverline - if( datapoint.attr( 'cy' ) < plot.config.height ){ - plot.content.append( 'line' ) - .attr( 'stroke', 'red' ) - .attr( 'stroke-width', 1 ) - .attr( 'x1', datapoint.attr( 'cx' ) ) - .attr( 'y1', datapoint.attr( 'cy' ) + plot.config.datapointSize ) - .attr( 'x2', datapoint.attr( 'cx' ) ) - .attr( 'y2', plot.config.height ) - .classed( 'hoverline', true ); - } + // re-render axis, grid, and datapoints + $( '.chart-info-box' ).remove(); + axis.redraw(); + _redrawDatapointsClipped(); + grid = renderGrid(); - var datapointWindowPos = $( this ).offset(); - plot.datapointInfoBox = plot.infoBox( - datapointWindowPos.top, datapointWindowPos.left, - plot.infoHtml( xCol[ i ], yCol[ i ], ( ids )?( ids[ i ] ):( undefined ) ) - ); - $( 'body' ).append( plot.datapointInfoBox ); - }) - .on( 'mouseout', function(){ - d3.select( this ) - .style( 'fill', 'black' ) - .style( 'fill-opacity', 0.2 ); - plot.content.selectAll( '.hoverline' ).remove(); - if( plot.datapointInfoBox ){ - plot.datapointInfoBox.remove(); - } - }); - }, - - this.render = function( columnData, meta ){ - this.log( this + '.render', arguments ); - this.log( '\t config:', this.config ); + $( svg.node() ).trigger( 'zoom.scatterplot', { + scale : zoom.scale(), + translate : zoom.translate() + }); + } + //TODO: programmatically set zoom/pan and save in config + //TODO: set pan/zoom limits + zoom.on( "zoom", zoomed ); - // prepare the data - //pre: columns passed are numeric - //pre: at least two columns are passed - //assume: first column is x, second column is y, any remaining aren't used - var xCol = columnData[0], - yCol = columnData[1], - ids = ( columnData.length > 2 )?( columnData[2] ):( undefined ); - //this.log( this + '.render', xCol.length, yCol.length, this.config ); - - //pre: xCol.len == yCol.len - xCol = this.preprocessData( xCol ); - yCol = this.preprocessData( yCol ); - this.log( 'xCol len', xCol.length, 'yCol len', yCol.length ); - - this.findMinMaxes( xCol, yCol, meta ); - //this.log( 'xMin, xMax, yMin, yMax:', this.xMin, this.xMax, this.yMin, this.yMax ); - this.setUpScales(); + function infoBox( top, left, d ){ + // create an abs pos. element containing datapoint data (d) near the point (top, left) + // with added padding to clear the mouse pointer + left += 8; + return $([ + '
        ', + (( config.idColumn !== undefined )?( '
        ' + d[ config.idColumn ] + '
        ' ):( '' )), + '
        ', getX( d ), '
        ', + '
        ', getY( d ), '
        ', + '
        ' + ].join( '' ) ).css({ top: top, left: left, 'z-index': 2 }); + } - // find (or build if it doesn't exist) the svg dom infrastructure - if( !this.svg ){ this.svg = d3.select( 'svg' ).attr( "class", "chart" ); } - if( !this.content ){ - this.content = this.svg.append( "svg:g" ).attr( "class", "content" ).attr( 'id', this.config.id ); + datapoints.on( 'mouseover', function( d, i ){ + var datapoint = d3.select( this ); + datapoint + .style( 'fill', 'red' ) + .style( 'fill-opacity', 1 ); + + // create horiz line to axis + content.append( 'line' ) + .attr( 'stroke', 'red' ) + .attr( 'stroke-width', 1 ) + // start not at center, but at the edge of the circle - to prevent mouseover thrashing + .attr( 'x1', datapoint.attr( 'cx' ) - config.datapointSize ) + .attr( 'y1', datapoint.attr( 'cy' ) ) + .attr( 'x2', 0 ) + .attr( 'y2', datapoint.attr( 'cy' ) ) + .classed( 'hoverline', true ); + + // create vertical line to axis - if not on the x axis + if( datapoint.attr( 'cy' ) < config.height ){ + content.append( 'line' ) + .attr( 'stroke', 'red' ) + .attr( 'stroke-width', 1 ) + .attr( 'x1', datapoint.attr( 'cx' ) ) + // attributes are strings so, (accrd. to js) '3' - 1 = 2 but '3' + 1 = '31': coerce + .attr( 'y1', +datapoint.attr( 'cy' ) + config.datapointSize ) + .attr( 'x2', datapoint.attr( 'cx' ) ) + .attr( 'y2', config.height ) + .classed( 'hoverline', true ); } - //this.log( 'svg:', this.svg ); - //this.log( 'content:', this.content ); - this.adjustChartDimensions(); - - if( !this.xAxis ){ this.xAxis = this.content.append( 'g' ).attr( 'class', 'axis' ).attr( 'id', 'x-axis' ); } - if( !this.xAxisLabel ){ - this.xAxisLabel = this.xAxis.append( 'text' ).attr( 'class', 'axis-label' ).attr( 'id', 'x-axis-label' ); - } - //this.log( 'xAxis:', this.xAxis, 'xAxisLabel:', this.xAxisLabel ); - - if( !this.yAxis ){ this.yAxis = this.content.append( 'g' ).attr( 'class', 'axis' ).attr( 'id', 'y-axis' ); } - if( !this.yAxisLabel ){ - this.yAxisLabel = this.yAxis.append( 'text' ).attr( 'class', 'axis-label' ).attr( 'id', 'y-axis-label' ); - } - //this.log( 'yAxis:', this.yAxis, 'yAxisLabel:', this.yAxisLabel ); - - this.setUpXAxis(); - this.setUpYAxis(); - - this.renderGrid(); - this.renderDatapoints( xCol, yCol, ids ); - }; - - this.infoHtml = function( x, y, id ){ - var retDiv = $( '
        ' ); - if( id ){ - $( '
        ' ).text( id ).css( 'font-weight', 'bold' ).appendTo( retDiv ); - } - $( '
        ' ).text( x ).appendTo( retDiv ); - $( '
        ' ).text( y ).appendTo( retDiv ); - return retDiv.html(); - }; - - //TODO: html for now - this.infoBox = function( top, left, html, adjTop, adjLeft ){ - adjTop = adjTop || 0; - adjLeft = adjLeft || 20; - var infoBox = $( '
        ' ) - .addClass( 'chart-info-box' ) - .css({ - 'position' : 'absolute', - 'top' : top + adjTop, - 'left' : left + adjLeft - }); - infoBox.html( html ); - return infoBox; - }; + // show the info box and trigger an event + var bbox = this.getBoundingClientRect(); + $( 'body' ).append( infoBox( bbox.top, bbox.right, d ) ); + $( svg.node() ).trigger( 'mouseover-datapoint.scatterplot', [ this, d, i ] ); + }); + datapoints.on( 'mouseout', function(){ + // return the point to normal, remove hoverlines and info box + d3.select( this ) + .style( 'fill', 'black' ) + .style( 'fill-opacity', 0.2 ); + content.selectAll( '.hoverline' ).remove(); + $( '.chart-info-box' ).remove(); + }); } //============================================================================== diff --git a/config/plugins/visualizations/scatterplot/src/scatterplotControlForm.js b/config/plugins/visualizations/scatterplot/src/scatterplotControlForm.js deleted file mode 100644 index 95074114030..00000000000 --- a/config/plugins/visualizations/scatterplot/src/scatterplotControlForm.js +++ /dev/null @@ -1,631 +0,0 @@ -/* ============================================================================= -todo: - I'd like to move the svg creation out of the splot constr. to: - allow adding splots to an existing canvas - allow mult. splots sharing a canvas - - - outside this: - BUG: setting width, height in plot controls doesn't re-interpolate data locations!! - BUG?: get metadata_column_names (from datatype if necessary) - BUG: single vis in popupmenu should have tooltip with that name NOT 'Visualizations' - - wire label setters, anim setter - - TwoVarScatterplot: - ??: maybe better to do this with a canvas... - save as visualization - to seperate file? - remove underscore dependencies - add interface to change values (seperate)? - download svg -> base64 encode - incorporate glyphs, glyph state renderers - - ScatterplotSettingsForm: - some css bug that lowers the width of settings form when plot-controls tab is open - causes chart to shift - what can be abstracted/reused for other graphs? - avoid direct manipulation of this.plot - allow option to put plot into seperate tab of interface (for small multiples) - - provide callback in view to load data incrementally - for large sets - paginate - handle rerender - use endpoint (here and on the server (fileptr)) - fetch (new?) data - handle rerender - use d3.TSV? - render warning on long data (> maxDataPoints) - adjust endpoint - - selectable list of preset column comparisons (rnaseq etc.) - how to know what sort of Tabular the data is? - smarter about headers - validate columns selection (here or server) - - set stats column names by selected columns - move chart into tabbed area... - - Scatterplot.mako: - multiple plots on one page (small multiples) - ?? ensure svg styles thru d3 or css? - d3: configable (easily) - css: standard - better maintenance - ? override at config - -============================================================================= */ -/** - * Scatterplot control UI as a backbone view - * handles: - * getting the desired data - * configuring the plot display - * showing (general) statistics - * - * initialize attributes REQUIRES a dataset and an apiDatasetsURL - */ -var ScatterplotControlForm = BaseView.extend( LoggableMixin ).extend({ - //logger : console, - className : 'scatterplot-control-form', - - //NOTE: should include time needed to render - dataLoadDelay : 4000, - dataLoadSize : 5000, - - loadingIndicatorImage : 'loading_small_white_bg.gif', - fetchMsg : 'Fetching data...', - renderMsg : 'Rendering...', - - initialize : function( attributes ){ - this.log( this + '.initialize, attributes:', attributes ); - - this.dataset = null; - this.chartConfig = null; - this.chart = null; - this.loader = null; - - // set up refs to the four tab areas - this.$dataControl = null; - this.$chartControl = null; - this.$statsDisplay = null; - this.$chartDisplay = null; - - this.dataFetch = null; - - this.initializeFromAttributes( attributes ); - this.initializeChart( attributes ); - this.initializeDataLoader( attributes ); - }, - - initializeFromAttributes : function( attributes ){ - // required settings: ensure certain vars we need are passed in attributes - if( !attributes || !attributes.dataset ){ - throw( "ScatterplotView requires a dataset" ); - } else { - this.dataset = attributes.dataset; - } - if( jQuery.type( this.dataset.metadata_column_types ) === 'string' ){ - this.dataset.metadata_column_types = this.dataset.metadata_column_types.split( ', ' ); - } - this.log( '\t dataset:', this.dataset ); - - // attempt to get possible headers from the data's first line - if( this.dataset.comment_lines && this.dataset.comment_lines.length ){ - //TODO:?? - var firstLine = this.dataset.comment_lines[0], - possibleHeaders = firstLine.split( '\t' ); - if( possibleHeaders.length === this.dataset.metadata_column_types.length ){ - this.possibleHeaders = possibleHeaders; - } - } - - // passed from mako helper - //TODO: integrate to galaxyPaths - //TODO: ?? seems like data loader section would be better - if( !attributes.apiDatasetsURL ){ - throw( "ScatterplotView requires a apiDatasetsURL" ); - } else { - this.dataURL = attributes.apiDatasetsURL + '/' + this.dataset.id + '?'; - } - this.log( '\t dataURL:', this.dataURL ); - }, - - initializeChart : function( attributes ){ - // set up the basic chart infrastructure and config (if any) - this.chartConfig = attributes.chartConfig || {}; - //if( this.logger ){ this.chartConfig.debugging = true; } - this.log( '\t initial chartConfig:', this.chartConfig ); - - this.chart = new TwoVarScatterplot( this.chartConfig ); - //TODO: remove 2nd ref, use this.chart.config - this.chartConfig = this.chart.config; - }, - - initializeDataLoader : function( attributes ){ - // set up data loader - var view = this; - this.loader = new LazyDataLoader({ - //logger : ( this.logger )?( this.logger ):( null ), - // we'll generate this when columns are chosen - url : null, - start : attributes.start || 0, - //NOTE: metadata_data_lines can be null (so we won't know the total) - total : attributes.total || this.dataset.metadata_data_lines, - delay : this.dataLoadDelay, - size : this.dataLoadSize, - - buildUrl : function( start, size ){ - // currently VERY SPECIFIC to using data_providers.py start_val, max_vals params - return this.url + '&' + jQuery.param({ - start_val: start, - max_vals: size - }); - } - }); - $( this.loader ).bind( 'error', function( event, status, error ){ - view.log( 'ERROR:', status, error ); - alert( 'ERROR fetching data:\n' + status + '\n' + error ); - view.hideLoadingIndicator(); - }); - }, - - // ------------------------------------------------------------------------- CONTROLS RENDERING - render : function(){ - this.log( this + '.render' ); - - // render the tab controls, areas and loading indicator - this.$el.append( ScatterplotControlForm.templates.mainLayout({ - loadingIndicatorImagePath : '/static/images/' + this.loadingIndicatorImage, - message : '' - })); - - // render the tab content - this.$dataControl = this._render_dataControl(); - this.$chartControl = this._render_chartControl(); - this.$statsDisplay = this.$el.find( '.tab-pane#stats-display' ); - this.$chartDisplay = this._render_chartDisplay(); - - // auto render if given both x, y column choices in query for page - //TODO:?? add autoRender=1 to query maybe? - if( this.chartConfig.xColumn && this.chartConfig.yColumn ){ - this.renderChart(); - } - - // set up behaviours - this.$el.find( '.tooltip' ).tooltip(); - - // uncomment any of the following to have that tab show on initial load (for testing) - //this.$el.find( 'ul.nav' ).find( 'a[href="#data-control"]' ).tab( 'show' ); - //this.$el.find( 'ul.nav' ).find( 'a[href="#chart-control"]' ).tab( 'show' ); - //this.$el.find( 'ul.nav' ).find( 'a[href="#stats-display"]' ).tab( 'show' ); - //this.$el.find( 'ul.nav' ).find( 'a[href="#chart-display"]' ).tab( 'show' ); - return this; - }, - - _render_dataControl : function(){ - // controls for which columns are used to plot datapoints (and ids/additional info to attach if desired) - var view = this, - allColumns = [], - numericColumns = [], - usePossibleHeaders = ( this.possibleHeaders && this.$dataControl )? - ( this.$dataControl.find( '#first-line-header-checkbox' ).is( ':checked' ) ):( false ); - - // gather column indeces (from metadata_column_types) and names (from metadata_columnnames) - _.each( this.dataset.metadata_column_types, function( type, index ){ - // use a 1 based index in names/values within the form (will be dec. when parsed out) - var oneBasedIndex = index + 1, - // default name is 'column '... - name = 'column ' + oneBasedIndex; - - // ...but label with the name if available... - if( view.dataset.metadata_column_names ){ - name = view.dataset.metadata_column_names[ index ]; - - // ...or, use the first line as headers if the user wants - } else if( usePossibleHeaders ){ - name = view.possibleHeaders[ index ]; - } - - // cache all columns here - allColumns.push({ index: oneBasedIndex, name: name }); - - // filter numeric columns to their own list - if( type === 'int' || type === 'float' ){ - numericColumns.push({ index: oneBasedIndex, name: name }); - } - }); - //TODO: other vals: max_vals, start_val, pagination (chart-settings) - - // render the html - var $dataControl = this.$el.find( '.tab-pane#data-control' ); - $dataControl.html( ScatterplotControlForm.templates.dataControl({ - allColumns : allColumns, - numericColumns : numericColumns, - possibleHeaders : ( this.possibleHeaders )?( this.possibleHeaders.join( ', ' ) ):( '' ), - usePossibleHeaders : usePossibleHeaders - })); - - if( !this.dataset.metadata_column_names && this.possibleHeaders ){ - $dataControl.find( '#first-line-header' ).show(); - } - - // preset to column selectors if they were passed in the config in the query string - $dataControl.find( '#X-select' ).val( this.chartConfig.xColumn ); - $dataControl.find( '#Y-select' ).val( this.chartConfig.yColumn ); - if( this.chartConfig.idColumn !== undefined ){ - $dataControl.find( '#include-id-checkbox' ) - .attr( 'checked', true ).trigger( 'change' ); - $dataControl.find( '#ID-select' ).val( this.chartConfig.idColumn ); - } - - return $dataControl; - }, - - _render_chartControl : function(){ - // tab content to control how the chart is rendered (data glyph size, chart size, etc.) - var view = this, - $chartControl = this.$el.find( '.tab-pane#chart-control' ), - // limits for controls (by control/chartConfig id) - //TODO: move into TwoVarScatterplot - controlRanges = { - 'datapointSize' : { min: 2, max: 10, step: 1 }, - 'width' : { min: 200, max: 800, step: 20 }, - 'height' : { min: 200, max: 800, step: 20 } - }; - - // render the html - $chartControl.append( ScatterplotControlForm.templates.chartControl( this.chartConfig ) ); - - // set up behaviours, js on sliders - $chartControl.find( '.numeric-slider-input' ).each( function(){ - var $this = $( this ), - $output = $this.find( '.slider-output' ), - $slider = $this.find( '.slider' ), - id = $this.attr( 'id' ); - //chartControl.log( 'slider set up', 'this:', $this, 'slider:', $slider, 'id', id ); - - // what to do when the slider changes: update display and update chartConfig - //TODO: move out of loop - function onSliderChange(){ - var $this = $( this ), - newValue = $this.slider( 'value' ); - //chartControl.log( 'slider change', 'this:', $this, 'output:', $output, 'value', newValue ); - $output.text( newValue ); - //chartControl.chartConfig[ id ] = newValue; - } - - $slider.slider( _.extend( controlRanges[ id ], { - value : view.chartConfig[ id ], - change : onSliderChange, - slide : onSliderChange - })); - }); - - return $chartControl; - }, - - _render_chartDisplay : function(){ - // render the tab content where the chart is displayed (but not the chart itself) - var $chartDisplay = this.$el.find( '.tab-pane#chart-display' ); - $chartDisplay.append( ScatterplotControlForm.templates.chartDisplay( this.chartConfig ) ); - return $chartDisplay; - }, - - // ------------------------------------------------------------------------- EVENTS - events : { - 'change #include-id-checkbox' : 'toggleThirdColumnSelector', - 'change #first-line-header-checkbox' : 'rerenderDataControl', - 'click #data-control #render-button' : 'renderChart', - 'click #chart-control #render-button' : 'changeChartSettings' - }, - - toggleThirdColumnSelector : function(){ - // show/hide the id selector on the data settings panel - this.$el.find( 'select[name="ID"]' ).parent().toggle(); - }, - - rerenderDataControl : function(){ - this.$dataControl = this._render_dataControl(); - }, - - showLoadingIndicator : function( message, callback ){ - // display the loading indicator over the tab panels if hidden, update message (if passed) - message = message || ''; - var indicator = this.$el.find( 'div#loading-indicator' ); - messageBox = indicator.find( '.loading-message' ); - - if( indicator.is( ':visible' ) ){ - if( message ){ - messageBox.fadeOut( 'fast', function(){ - messageBox.text( message ); - messageBox.fadeIn( 'fast', callback ); - }); - } else { - callback(); - } - - } else { - if( message ){ messageBox.text( message ); } - indicator.fadeIn( 'fast', callback ); - } - }, - - hideLoadingIndicator : function( callback ){ - this.$el.find( 'div#loading-indicator' ).fadeOut( 'fast', callback ); - }, - - // ------------------------------------------------------------------------- CHART/STATS RENDERING - renderChart : function(){ - // fetch the data, (re-)render the chart - this.log( this + '.renderChart' ); - - //TODO: separate data fetch - - // this is a complete re-render, so clear the prev. data - this.data = null; - this.meta = null; - - // update the chartConfig (here and chart) using chart settings - //TODO: separate and improve (used in changeChartSettings too) - _.extend( this.chartConfig, this.getChartSettings() ); - this.log( '\t chartConfig:', this.chartConfig ); - this.chart.updateConfig( this.chartConfig, false ); - - // build the url with the current data settings - this.loader.url = this.dataURL + '&' + jQuery.param( this.getDataSettings() ); - this.log( '\t loader: total lines:', this.loader.total, ' url:', this.loader.url ); - - // bind the new data event to: aggregate data, update the chart and stats with new data - var view = this; - $( this.loader ).bind( 'loaded.new', function( event, response ){ - view.log( view + ' loaded.new', response ); - - // aggregate data and meta - view.postProcessDataFetchResponse( response ); - view.log( '\t postprocessed data:', view.data ); - view.log( '\t postprocessed meta:', view.meta ); - - // update the chart and stats - view.showLoadingIndicator( view.renderMsg, function(){ - view.chart.render( view.data, view.meta ); - view.renderStats( view.data, view.meta ); - view.hideLoadingIndicator(); - }); - }); - // when all data loaded - unbind (or we'll start doubling event handlers) - $( this.loader ).bind( 'complete', function( event, data ){ - view.log( view + ' complete', data ); - $( view.loader ).unbind(); - }); - - // begin loading the data, switch to the chart display tab - view.showLoadingIndicator( view.fetchMsg, function(){ - view.$el.find( 'ul.nav' ).find( 'a[href="#chart-display"]' ).tab( 'show' ); - view.loader.load(); - }); - }, - - renderStats : function(){ - this.log( this + '.renderStats' ); - // render the stats table in the stats panel - //TODO: there's a better way - this.$statsDisplay.html( ScatterplotControlForm.templates.statsDisplay({ - stats: [ - { name: 'Count', xval: this.meta[0].count, yval: this.meta[1].count }, - { name: 'Min', xval: this.meta[0].min, yval: this.meta[1].min }, - { name: 'Max', xval: this.meta[0].max, yval: this.meta[1].max }, - { name: 'Sum', xval: this.meta[0].sum, yval: this.meta[1].sum }, - { name: 'Mean', xval: this.meta[0].mean, yval: this.meta[1].mean }, - { name: 'Median', xval: this.meta[0].median, yval: this.meta[1].median } - ] - })); - }, - - changeChartSettings : function(){ - // re-render the chart with new chart settings and OLD data - var view = this; - newChartSettings = this.getChartSettings(); - - // update the chart config from the chartSettings panel controls - _.extend( this.chartConfig, newChartSettings ); - this.log( 'this.chartConfig:', this.chartConfig ); - this.chart.updateConfig( this.chartConfig, false ); - - // if there's current data, call chart.render with it (no data fetch) - if( view.data && view.meta ){ - view.showLoadingIndicator( view.renderMsg, function(){ - view.$el.find( 'ul.nav' ).find( 'a[href="#chart-display"]' ).tab( 'show' ); - view.chart.render( view.data, view.meta ); - view.hideLoadingIndicator(); - }); - - // no current data, call renderChart instead (which will fetch data) - } else { - this.renderChart(); - } - }, - - // ------------------------------------------------------------------------- DATA AGGREGATION - postProcessDataFetchResponse : function( response ){ - // the loader only returns new data - it's up to this to munge the fetches together properly - //TODO: we're now storing data in two places: loader and here - // can't we reduce incoming data into loader.data[0]? are there concurrency problems? - this.postProcessData( response.data ); - this.postProcessMeta( response.meta ); - }, - - postProcessData : function( newData ){ - // stack the column data on top of each other into this.data - //this.log( this + '.postProcessData:', newData ); - var view = this; - - // if we already have data: aggregate - if( view.data ){ - _.each( newData, function( newColData, colIndex ){ - //view.log( colIndex + ' data:', newColData ); - //TODO??: time, space efficiency of this? - view.data[ colIndex ] = view.data[ colIndex ].concat( newColData ); - }); - - // otherwise: assign (first load) - } else { - view.data = newData; - } - }, - - postProcessMeta : function( newMeta ){ - // munge the meta data (stats) from the server fetches together - //pre: this.data must be preprocessed (needed for medians) - //this.log( this + '.postProcessMeta:', newMeta ); - var view = this, - colTypes = this.dataset.metadata_column_types; - - // if we already have meta: aggregate - if( view.meta ){ - _.each( newMeta, function( newColMeta, colIndex ){ - var colMeta = view.meta[ colIndex ], - colType = colTypes[ colIndex ]; - //view.log( '\t ' + colIndex + ' postprocessing meta:', newColMeta ); - //view.log( colIndex + ' old meta:', - // 'min:', colMeta.min, - // 'max:', colMeta.max, - // 'sum:', colMeta.sum, - // 'mean:', colMeta.mean, - // 'median:', colMeta.median - //); - - //!TODO: at what point are we getting int/float overflow on these?! - //??: need to be null safe? - colMeta.count += ( newColMeta.count )?( newColMeta.count ):( 0 ); - //view.log( colIndex, 'count:', colMeta.count ); - - if( ( colType === 'int' ) || ( colType === 'float' ) ){ - //view.log( colIndex + ' incoming meta:', - // 'min:', newColMeta.min, - // 'max:', newColMeta.max, - // 'sum:', newColMeta.sum, - // 'mean:', newColMeta.mean, - // 'median:', newColMeta.median - //); - - colMeta.min = Math.min( newColMeta.min, colMeta.min ); - colMeta.max = Math.max( newColMeta.max, colMeta.max ); - colMeta.sum = newColMeta.sum + colMeta.sum; - colMeta.mean = ( colMeta.count )?( colMeta.sum / colMeta.count ):( null ); - - // median's a pain bc of sorting (requires the data as well) - var sortedCol = view.data[ colIndex ].slice().sort(), - middleIndex = Math.floor( sortedCol.length / 2 ); - - if( sortedCol.length % 2 === 0 ){ - colMeta.median = ( ( sortedCol[ middleIndex ] + sortedCol[( middleIndex + 1 )] ) / 2 ); - - } else { - colMeta.median = sortedCol[ middleIndex ]; - } - - //view.log( colIndex + ' new meta:', - // 'min:', colMeta.min, - // 'max:', colMeta.max, - // 'sum:', colMeta.sum, - // 'mean:', colMeta.mean, - // 'median:', colMeta.median - //); - } - }); - - // otherwise: assign (first load) - } else { - view.meta = newMeta; - //view.log( '\t meta (first load):', view.meta ); - } - }, - - // ------------------------------------------------------------------------- GET DATA/CHART SETTINGS - getDataSettings : function(){ - // parse the column values for both indeces (for the data fetch) and names (for the chart) - var columnSelections = this.getColumnSelections(), - columns = []; - this.log( '\t columnSelections:', columnSelections ); - - //TODO: validate columns - minimally: we can assume either set by selectors or via a good query string - - // get column indices for params, include the desired ID column (if any) - //NOTE: these are presented in human-readable 1 base index (to match the data.peek) - adjust - columns = [ - columnSelections.X.colIndex - 1, - columnSelections.Y.colIndex - 1 - ]; - if( this.$dataControl.find( '#include-id-checkbox' ).attr( 'checked' ) ){ - columns.push( columnSelections.ID.colIndex - 1 ); - } - //TODO: other vals: max, start, page - - var params = { - data_type : 'raw_data', - provider : 'column_with_stats', - columns : '[' + columns + ']' - }; - this.log( '\t data settings (url params):', params ); - return params; - }, - - getColumnSelections : function(){ - // gets the current user-selected values for which columns to fetch from the data settings panel - // returns a map: { column-select name (eg. X) : { colIndex : column-selector val, - // colName : selected option text }, ... } - var selections = {}; - this.$dataControl.find( 'div.column-select select' ).each( function(){ - var $this = $( this ), - val = $this.val(); - selections[ $this.attr( 'name' ) ] = { - colIndex : val, - colName : $this.children( '[value="' + val + '"]' ).text() - }; - }); - return selections; - }, - - getChartSettings : function(){ - // gets the user-selected chartConfig from the chart settings panel - var settings = {}, - colSelections = this.getColumnSelections(); - //this.log( 'colSelections:', colSelections ); - - //TODO: simplify with keys and loop - settings.datapointSize = this.$chartControl.find( '#datapointSize.numeric-slider-input' ) - .find( '.slider' ).slider( 'value' ); - settings.width = this.$chartControl.find( '#width.numeric-slider-input' ) - .find( '.slider' ).slider( 'value' ); - settings.height = this.$chartControl.find( '#height.numeric-slider-input' ) - .find( '.slider' ).slider( 'value' ); - - // update axes labels using chartSettings inputs (if not at defaults), otherwise the selects' colName - //TODO: a little confusing - var chartSettingsXLabel = this.$chartControl.find( 'input#X-axis-label' ).val(), - chartSettingsYLabel = this.$chartControl.find( 'input#Y-axis-label' ).val(); - settings.xLabel = ( chartSettingsXLabel === 'X' )? - ( colSelections.X.colName ):( chartSettingsXLabel ); - settings.yLabel = ( chartSettingsYLabel === 'Y' )? - ( colSelections.Y.colName ):( chartSettingsYLabel ); - - settings.animDuration = ( this.$chartControl.find( '#animate-chart' ).is( ':checked' ) )? - ( this.chart.defaults.animDuration ):( 0 ); - - this.log( '\t chartSettings:', settings ); - return settings; - }, - - toString : function(){ - return 'ScatterplotControlForm(' + (( this.dataset )?( this.dataset.id ):( '' )) + ')'; - } -}); - -ScatterplotControlForm.templates = { - mainLayout : Templates.scatterplotControlForm, - dataControl : Templates.dataControl, - chartControl : Templates.chartControl, - statsDisplay : Templates.statsDisplay, - chartDisplay : Templates.chartDisplay -}; - -//============================================================================== diff --git a/config/plugins/visualizations/scatterplot/src/visualization-templates.html b/config/plugins/visualizations/scatterplot/src/visualization-templates.html deleted file mode 100644 index 29c248143ca..00000000000 --- a/config/plugins/visualizations/scatterplot/src/visualization-templates.html +++ /dev/null @@ -1,182 +0,0 @@ - - - - - - - - - diff --git a/config/plugins/visualizations/scatterplot/static/numeric-column-stats.js b/config/plugins/visualizations/scatterplot/static/numeric-column-stats.js new file mode 100644 index 00000000000..8610c342ad3 --- /dev/null +++ b/config/plugins/visualizations/scatterplot/static/numeric-column-stats.js @@ -0,0 +1,89 @@ +var numericColumnStats = function( data, keys ){ + var console = console || { debug: function(){} }, + stats = {}; + + // precondition: keys is an array of keys accessible on each data in data + if( !keys || !keys.length ){ + throw new Error( 'keys is a required parameter and must be an array:' + keys ); + } + if( !data || !data.length ){ + return stats; + } + + function parseVal( val ){ + if( val === null ){ + throw new Error( 'Null value' ); + } + val = Number( val ); + if( isNaN( val ) || !isFinite( val ) ){ + throw new Error( 'NaN or non-finite number' ); + } + return val; + } + + // does this effect the data in the other thread? + [ 'min', 'max', 'sum', 'mean', 'median', 'count' ].forEach( function( name, i ){ + stats[ name ] = new Array( keys.length ); + }); + + var keyIndex = 0, + separatedCols = new Array( keys.length ); + keys.forEach( function( key, keyIndex ){ + stats.min[ keyIndex ] = 0; + stats.max[ keyIndex ] = 0; + stats.sum[ keyIndex ] = 0; + separatedCols[ keyIndex ] = []; + }); + + // work backwards to prevent co-modification problems while splitting up data into columns + for( var dataIndex=( data.length - 1 ); dataIndex>=0; dataIndex-=1 ){ + var datum = data.pop( dataIndex ); + + for( keyIndex=0; keyIndex straight forward median + var middleDataIndex = Math.floor( count / 2 ); + if( count % 2 === 1 ){ + stats.median[ keyIndex ] = separatedCols[ keyIndex ][ middleDataIndex ]; + } else { + var middleValA = separatedCols[ keyIndex ][ middleDataIndex ], + middleValB = separatedCols[ keyIndex ][( middleDataIndex + 1 )]; + stats.median[ keyIndex ] = ( middleValA + middleValB ) / 2; + } + } + return stats; +}; diff --git a/config/plugins/visualizations/scatterplot/static/scatterplot-edit.js b/config/plugins/visualizations/scatterplot/static/scatterplot-edit.js new file mode 100644 index 00000000000..c591a9317cf --- /dev/null +++ b/config/plugins/visualizations/scatterplot/static/scatterplot-edit.js @@ -0,0 +1 @@ +function scatterplot(a,b,c){function d(){var a={v:{},h:{}};return a.v.lines=p.selectAll("line.v-grid-line").data(m.x.ticks(q.x.fn.ticks()[0])),a.v.lines.enter().append("svg:line").classed("grid-line v-grid-line",!0),a.v.lines.attr("x1",m.x).attr("x2",m.x).attr("y1",0).attr("y2",b.height),a.v.lines.exit().remove(),a.h.lines=p.selectAll("line.h-grid-line").data(m.y.ticks(q.y.fn.ticks()[0])),a.h.lines.enter().append("svg:line").classed("grid-line h-grid-line",!0),a.h.lines.attr("x1",0).attr("x2",b.width).attr("y1",m.y).attr("y2",m.y),a.h.lines.exit().remove(),a}function e(){return t.attr("cx",function(a,b){return m.x(j(a,b))}).attr("cy",function(a,b){return m.y(k(a,b))}).style("display","block").filter(function(){var a=d3.select(this).attr("cx"),c=d3.select(this).attr("cy");return 0>a||a>b.width?!0:0>c||c>b.height?!0:!1}).style("display","none")}function f(){$(".chart-info-box").remove(),q.redraw(),e(),s=d(),$(o.node()).trigger("zoom.scatterplot",{scale:n.scale(),translate:n.translate()})}function g(a,c,d){return c+=8,$(['
        ',void 0!==b.idColumn?"
        "+d[b.idColumn]+"
        ":"","
        ",j(d),"
        ","
        ",k(d),"
        ","
        "].join("")).css({top:a,left:c,"z-index":2})}var h=function(a,b){return"translate("+a+","+b+")"},i=function(a,b,c){return"rotate("+a+","+b+","+c+")"},j=function(a){return a[b.xColumn]},k=function(a){return a[b.yColumn]},l={x:{extent:d3.extent(c,j)},y:{extent:d3.extent(c,k)}},m={x:d3.scale.linear().domain(l.x.extent).range([0,b.width]),y:d3.scale.linear().domain(l.y.extent).range([b.height,0])},n=d3.behavior.zoom().x(m.x).y(m.y).scaleExtent([1,30]).scale(b.scale||1).translate(b.translate||[0,0]),o=d3.select(a).attr("class","scatterplot").attr("width","100%").attr("height",b.height+(b.margin.top+b.margin.bottom)),p=o.append("g").attr("class","content").attr("transform",h(b.margin.left,b.margin.top)).call(n);p.append("rect").attr("class","zoom-rect").attr("width",b.width).attr("height",b.height).style("fill","transparent");var q={x:{},y:{}};q.x.fn=d3.svg.axis().orient("bottom").scale(m.x).ticks(b.x.ticks).tickFormat(d3.format("s")),q.y.fn=d3.svg.axis().orient("left").scale(m.y).ticks(b.y.ticks).tickFormat(d3.format("s")),q.x.g=p.append("g").attr("class","x axis").attr("transform",h(0,b.height)).call(q.x.fn),q.y.g=p.append("g").attr("class","y axis").call(q.y.fn);var r=6;q.x.label=o.append("text").attr("class","axis-label").text(b.x.label).attr("text-anchor","middle").attr("dominant-baseline","text-after-edge").attr("x",b.width/2+b.margin.left).attr("y",b.height+b.margin.bottom+b.margin.top-r),q.y.label=o.append("text").attr("class","axis-label").text(b.y.label).attr("text-anchor","middle").attr("dominant-baseline","text-before-edge").attr("x",r).attr("y",b.height/2).attr("transform",i(-90,r,b.height/2)),q.redraw=function(){o.select(".x.axis").call(q.x.fn),o.select(".y.axis").call(q.y.fn)};var s=d(),t=p.selectAll(".glyph").data(c).enter().append("svg:circle").classed("glyph",!0).attr("cx",function(a,b){return m.x(j(a,b))}).attr("cy",function(a,b){return m.y(k(a,b))}).attr("r",0);t.transition().duration(b.animDuration).attr("r",b.datapointSize),e(),n.on("zoom",f),t.on("mouseover",function(a,c){var d=d3.select(this);d.style("fill","red").style("fill-opacity",1),p.append("line").attr("stroke","red").attr("stroke-width",1).attr("x1",d.attr("cx")-b.datapointSize).attr("y1",d.attr("cy")).attr("x2",0).attr("y2",d.attr("cy")).classed("hoverline",!0),d.attr("cy")= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var f,g="",h="function",i=this.escapeExpression;return g+='

        \n Use the following controls to how the chart is displayed.\n The slide controls can be moved by the mouse or, if the \'handle\' is in focus, your keyboard\'s arrow keys.\n Move the focus between controls by using the tab or shift+tab keys on your keyboard.\n Use the \'Draw\' button to render (or re-render) the chart with the current settings.\n

        \n\n
        \n \n
        ',(f=c.datapointSize)?f=f.call(b,{hash:{},data:e}):(f=b.datapointSize,f=typeof f===h?f.apply(b):f),g+=i(f)+'
        \n
        \n

        \n Size of the graphic representation of each data point\n

        \n
        \n\n
        \n \n
        ',(f=c.width)?f=f.call(b,{hash:{},data:e}):(f=b.width,f=typeof f===h?f.apply(b):f),g+=i(f)+'
        \n
        \n

        \n (not including chart margins and axes)\n

        \n
        \n\n
        \n \n
        ',(f=c.height)?f=f.call(b,{hash:{},data:e}):(f=b.height,f=typeof f===h?f.apply(b):f),g+=i(f)+'
        \n
        \n

        \n (not including chart margins and axes)\n

        \n
        \n\n
        \n \n \n

        \n
        \n\n
        \n \n \n

        \n
        \n\n\n'}),this.scatterplot.datacontrol=Handlebars.template(function(a,b,c,d,e){function f(a,b){var d,e="";return e+='\n \n "}function g(){return'checked="true"'}this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var h,i="",j="function",k=this.escapeExpression,l=this;return i+='

        \n Use the following controls to change the data used by the chart.\n Use the \'Draw\' button to render (or re-render) the chart with the current settings.\n

        \n\n\n
        \n \n \n
        \n
        \n \n \n
        \n\n\n
        \n \n \n

        \n These will be displayed (along with the x and y values) when you hover over\n a data point.\n

        \n
        \n\n\n\n\n\n\n'}),this.scatterplot.editor=Handlebars.template(function(a,b,c,d,e){this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var f="";return f+='
        \n \n \n\n \n
        \n \n
        \n \n
        \n \n \n
        \n \n
        \n\n \n
        \n\n
        \n
        \n'});var ScatterplotConfigEditor=Backbone.View.extend(LoggableMixin).extend({className:"scatterplot-control-form",initialize:function(a){if(this.model||(this.model=new Visualization({type:"scatterplot"})),this.log(this+".initialize, attributes:",a),!a||!a.dataset)throw new Error("ScatterplotConfigEditor requires a dataset");this.dataset=a.dataset,this.log("dataset:",this.dataset),this.display=new ScatterplotDisplay({dataset:a.dataset,model:this.model})},render:function(){this.$el.empty().append(ScatterplotConfigEditor.templates.mainLayout({})),this.model.id&&(this.$el.find(".copy-btn").show(),this.$el.find(".save-btn").text("Update saved")),this.$el.find("[title]").tooltip(),this._render_dataControl(),this._render_chartControls(),this._render_chartDisplay();var a=this.model.get("config");return this.model.id&&_.isFinite(a.xColumn)&&_.isFinite(a.yColumn)&&this.renderChart(),this},_render_dataControl:function(a){a=a||this.$el;var b=this,c=this.dataset,d=c.metadata_column_names||[],e=this.model.get("config"),f=[],g=_.map(c.metadata_column_types,function(a,b){var c={index:b,type:a,name:d[b]||"column "+(b+1)};return("int"===c.type||"float"===c.type)&&f.push(c),c});f.length<2&&(f=g);var h=a.find(".tab-pane#data-control");h.html(ScatterplotConfigEditor.templates.dataControl({allColumns:g,numericColumns:f}));var i={xColumn:_.isFinite(e.xColumn)?e.xColumn:f[0].index,yColumn:_.isFinite(e.yColumn)?e.yColumn:f[1].index,idColumn:g[0].index};if(_.isFinite(e.idColumn))i.idColumn=e.idColumn;else if(g.length>2){var j=_.find(g,function(a,b){return b!==i.xColumn&&b!==i.yColumn});i.idColumn=j.index}return e=this.model.set("config",i,{silent:!0}).get("config"),h.find('[name="xColumn"]').val(e.xColumn).on("change",function(){b.model.set("config",{xColumn:Number($(this).val())})}),h.find('[name="yColumn"]').val(e.yColumn).on("change",function(){b.model.set("config",{yColumn:Number($(this).val())})}),h.find('select[name="idColumn"]').val(e.idColumn).on("change",function(){b.model.set("config",{idColumn:Number($(this).val())})}),void 0!==e.idColumn&&h.find("#include-id-checkbox").prop("checked",!0).trigger("change"),h.find("[title]").tooltip(),h},_render_chartControls:function(a){function b(){var a=$(this),b=a.slider("value");c.model.set("config",_.object([[a.parent().data("config-key"),b]])),a.siblings(".slider-output").text(b)}a=a||this.$el;var c=this,d=this.model.get("config"),e=a.find("#chart-control");e.html(ScatterplotConfigEditor.templates.chartControl(d));var f={datapointSize:{min:2,max:10,step:1},width:{min:200,max:800,step:20},height:{min:200,max:800,step:20}};e.find(".numeric-slider-input").each(function(){var a=$(this),c=a.attr("data-config-key"),e=_.extend(f[c],{value:d[c],change:b,slide:b});a.find(".slider").slider(e),a.children(".slider-output").text(d[c])});var g=this.dataset.metadata_column_names||[],h=d.xLabel||g[d.xColumn]||"X",i=d.yLabel||g[d.yColumn]||"Y";return e.find('input[name="X-axis-label"]').val(h).on("change",function(){c.model.set("config",{xLabel:$(this).val()})}),e.find('input[name="Y-axis-label"]').val(i).on("change",function(){c.model.set("config",{yLabel:$(this).val()})}),e.find("[title]").tooltip(),e},_render_chartDisplay:function(a){a=a||this.$el;var b=a.find(".tab-pane#chart-display");return this.display.setElement(b),this.display.render(),b.find("[title]").tooltip(),b},events:{"change #include-id-checkbox":"toggleThirdColumnSelector","click #data-control .render-button":"renderChart","click #chart-control .render-button":"renderChart","click .save-btn":"saveVisualization"},saveVisualization:function(){var a=this;this.model.save().fail(function(b,c,d){console.error(b,c,d),a.trigger("save:error",view),alert("Error loading data:\n"+b.responseText)}).then(function(){a.render()})},toggleThirdColumnSelector:function(){this.$el.find('select[name="idColumn"]').parent().toggle()},renderChart:function(){this.$el.find(".nav li.disabled").removeClass("disabled"),this.$el.find("ul.nav").find('a[href="#chart-display"]').tab("show"),this.display.fetchData()},toString:function(){return"ScatterplotConfigEditor("+(this.dataset?this.dataset.id:"")+")"}});ScatterplotConfigEditor.templates={mainLayout:scatterplot.editor,dataControl:scatterplot.datacontrol,chartControl:scatterplot.chartcontrol};var ScatterplotDisplay=Backbone.View.extend({initialize:function(a){this.data=null,this.dataset=a.dataset,this.calcNumPages()},calcNumPages:function(){var a=this.model.get("config");this.lineCount=this.dataset.metadata_data_lines,this.numPages=this.lineCount?Math.ceil(this.lineCount/a.pagination.perPage):void 0,this.lineCount&&void 0!==this.numPages||console.warn("no data total found")},fetchData:function(){this.showLoadingIndicator("getting data");var a=this,b=this.model.get("config"),c=jQuery.getJSON("/api/datasets/"+this.dataset.id,{data_type:"raw_data",provider:"dataset-column",limit:b.pagination.perPage,offset:b.pagination.currPage*b.pagination.perPage});return c.done(function(b){a.data=b.data,a.trigger("data:fetched",a),a.renderData()}),c.fail(function(b,c,d){console.error(b,c,d),a.trigger("data:error",a),alert("Error loading data:\n"+b.responseText)}),c},showLoadingIndicator:function(){this.$el.find(".scatterplot-data-info").html(['
        ','','loading...',"
        "].join(""))},template:function(){var a=['
        ','
        ',"
        ",'
        ','

        ','','',"
        ","
        ","",'
        '].join("");return a},render:function(){return this.$el.addClass("scatterplot-display").html(this.template()),this.data&&this.renderData(),this},renderData:function(){this.renderLeftControls(),this.renderRightControls(),this.renderPlot(this.data),this.getStats()},renderLeftControls:function(){return this.lineCount?this.$el.find(".controls .left").empty().append(this.renderPagination()):this.$el.find(".controls .left").empty().append(this.renderPrevNext()),this},renderRightControls:function(){var a=this;this.setLineInfo(this.data),this.$el.find(".stats-toggle-btn").off().click(function(){a.toggleStats()}),this.$el.find(".rerender-btn").off().click(function(){a.resetZoom(),a.renderPlot(this.data)})},renderPlot:function(){var a=this,b=this.$el.find("svg");this.toggleStats(!1),b.off().empty().show().on("zoom.scatterplot",function(b,c){a.model.set("config",c)}),scatterplot(b.get(0),this.model.get("config"),this.data)},setLineInfo:function(a,b){if(a){var c=this.model.get("config"),d=this.lineCount||"an unknown total",e=c.pagination.currPage*c.pagination.perPage,f=e+a.length;this.$el.find(".controls p.scatterplot-data-info").text([e+1,"to",f,"of",d].join(" "))}else this.$el.find(".controls p.scatterplot-data-info").html(b||"");return this},resetZoom:function(a,b){return a=void 0!==a?a:1,b=void 0!==b?b:[0,0],this.model.set("config",{scale:a,translate:b}),this},goToPage:function(a){var b=this.model.get("config").pagination;return 0>=a&&(a=0),this.numPages&&a>=this.numPages&&(a=this.numPages-1),a===b.currPage?this:(b.currPage=a,this.model.set("config",{pagination:b}),this.resetZoom(),this.fetchData(),this)},nextPage:function(){var a=this.model.get("config").pagination.currPage;return this.goToPage(a+1)},prevPage:function(){var a=this.model.get("config").pagination.currPage;return this.goToPage(a-1)},renderPrevNext:function(){var a=this.model.get("config");if(!this.data)return null;if(0===a.pagination.currPage&&this.data.lengthPrev').click(function(){b.prevPage()}),d=$('
      • Next
      • ').click(function(){b.nextPage()});return 0===a.pagination.currPage&&c.addClass("disabled"),this.numPages&&a.pagination.currPage===this.numPages-1&&d.addClass("disabled"),$("
          ").addClass("pagination data-prev-next").append([c,d])},renderPagination:function(){var a=this.model.get("config");if(!this.data)return null;if(!this.numPages)return null;if(0===a.pagination.currPage&&this.data.length").addClass("pagination data-pages");pageNumClick=function(){b.goToPage($(this).data("page"))};for(var d=0;d',d+1,""].join("")).attr("data-page",d).click(pageNumClick);d===a.pagination.currPage&&e.addClass("active"),c.append(e)}return c},getStats:function(){if(this.data){var a=this,b=this.model.get("config"),c=new Worker("/plugins/visualizations/scatterplot/static/worker-stats.js");c.postMessage({data:this.data,keys:[b.xColumn,b.yColumn]}),c.onerror=function(){c.terminate()},c.onmessage=function(b){a.renderStats(b.data)}}},renderStats:function(a){var b=this.model.get("config"),c=this.$el.find(".stats-display"),d=b.x.label,e=b.y.label,f=$("
        ', xLabel, '', yLabel, '
        ', key, '', stat[0], '', stat[1], '
        ").addClass("table").append([""].join("")).append(_.map(a,function(a,b){return $([""].join(""))}));c.empty().append(f)},toggleStats:function(a){var b=this.$el.find(".stats-display");a=void 0===a?b.is(":hidden"):a,a?(this.$el.find("svg").hide(),b.show(),this.$el.find(".controls .stats-toggle-btn").text("Plot")):(b.hide(),this.$el.find("svg").show(),this.$el.find(".controls .stats-toggle-btn").text("Stats"))},toString:function(){return"ScatterplotView()"}}),ScatterplotModel=Visualization.extend({defaults:{type:"scatterplot",config:{pagination:{currPage:0,perPage:3e3},width:400,height:400,margin:{top:16,right:16,bottom:40,left:54},x:{ticks:10,label:"X"},y:{ticks:10,label:"Y"},datapointSize:4,animDuration:500,scale:1,translate:[0,0]}}}); \ No newline at end of file diff --git a/config/plugins/visualizations/scatterplot/static/scatterplot.css b/config/plugins/visualizations/scatterplot/static/scatterplot.css index e97c9334004..f3105d7568c 100644 --- a/config/plugins/visualizations/scatterplot/static/scatterplot.css +++ b/config/plugins/visualizations/scatterplot/static/scatterplot.css @@ -1,171 +1,213 @@ -/*TODO: use/move into base.less*/ -* { margin: 0px; padding: 0px; } - -/* -------------------------------------------- general layout */ -div.tab-pane { - padding: 8px; -} +* { margin: 0; padding: 0; } /* -------------------------------------------- header */ -.header { - margin-bottom: 8px; -} +.chart-header > * { margin: 0; padding: 0; } -#chart-header { - padding : 8px; - background-color: #ebd9b2; +.chart-header { margin-bottom: 16px; overflow: auto; + background-color: #ebd9b2; + padding : 16px 12px 12px 12px; } -#chart-header .subtitle { - margin: -4px 0px 0px 4px; - padding : 0; +.chart-header h2 { + margin-bottom: 4px; +} + +.chart-header p { color: white; font-size: small; } /* -------------------------------------------- main layout */ -#scatterplot { - /*from width + margin of chart?*/ +.scatterplot-editor > * { margin: 0; padding: 0; } +.scatterplot-editor .tab-pane { + padding: 8px; } -.scatterplot-container .tab-pane { +.scatterplot-editor .help-text, +.scatterplot-editor .help-text-small { + color: grey; + margin-bottom: 8px; } -/* -------------------------------------------- all controls */ +.scatterplot-editor .help-text { + margin-top: 4px; +} -#scatterplot input[type=button], -#scatterplot select { +.scatterplot-editor .help-text-small { + font-size: smaller; +} + +.scatterplot-editor .file-controls { + float: right; + /*margin-right: 16px;*/ +} + +.file-controls .btn { + height: 24px; + padding: 0px 10px 0px 10px; + line-height: 24px; +} + +.file-controls .copy-btn { + display: none; +} + + +/* ============================================ config controls */ +.scatterplot-config-control { + max-width: 768px; +} + +.scatterplot-config-control input[type="button"], +.scatterplot-config-control button, +.scatterplot-config-control select { width: 100%; max-width: 256px; margin-bottom: 8px; } - -#scatterplot .help-text, -#scatterplot .help-text-small { - color: grey; +.scatterplot-config-control input[type="text"] { + width: 100%; + max-width: 512px; + border: 1px solid lightgrey; } -#scatterplot .help-text { - padding-bottom: 16px; +.scatterplot-config-control input[type="checkbox"] { + display: inline-block; + margin-top: -3px; } -#scatterplot .help-text-small { - padding: 4px; - font-size: smaller; -} - -#scatterplot > * { -} - -#scatterplot input[value=Draw] { +.scatterplot-config-control .render-button { display: block; margin-top: 16px; } -#scatterplot .numeric-slider-input { - max-width: 70%; -} - -/* -------------------------------------------- data controls */ - -/* -------------------------------------------- chart controls */ -#chart-control .form-input { - /*display: table-row;*/ -} - -#chart-control label { - /*text-align: right;*/ +.scatterplot-config-control .form-input { margin-bottom: 8px; - /*display: table-cell;*/ } -#chart-control .slider { - /*display: table-cell;*/ +.scatterplot-config-control .slider { height: 8px; display: block; - margin: 8px 0px 0px 8px; + margin-bottom: 8px; } -#chart-control .slider-output { - /*display: table-cell;*/ +.scatterplot-config-control .slider-output { float: right; } -#chart-control input[type="text"] { - border: 1px solid lightgrey; -} - - -/* -------------------------------------------- statistics */ -#stats-display table#chart-stats-table { - width: 100%; -} - -#stats-display #chart-stats-table th { - width: 30%; - padding: 4px; - font-weight: bold; - color: grey; -} - -#stats-display #chart-stats-table td { - border: solid lightgrey; - border-width: 1px 0px 0px 1px; - padding: 4px; -} - -#stats-display #chart-stats-table td:nth-child(1) { - border-width: 1px 0px 0px 0px; - padding-right: 1em; - text-align: right; - font-weight: bold; - color: grey; -} - +/* ============================================ plot display */ /* -------------------------------------------- load indicators */ -#loading-indicator { - margin: 12px 0px 0px 8px; +.scatterplot-display .loading-indicator { + color: grey; } -#scatterplot #loading-indicator .loading-message { +.scatterplot-display .loading-indicator .loading-indicator-message { + display: inline-block; + margin-left: 4px; font-style: italic; - font-size: smaller; - color: grey; +} + +/* -------------------------------------------- display controls */ +.scatterplot-display .controls { + width: 100%; + margin-bottom: 4px; + border: 1px solid lightgrey; + border-radius: 3px; + padding: 8px 8px 4px 8px; +} + +.scatterplot-display .controls button { + display: inline-block; + min-width: 0; + width: auto; + margin: 0px; +} + +.scatterplot-display .controls .left { + float: left; +} + +.scatterplot-display .controls .pagination { + display: inline-block; + margin: 0px; +} + +.scatterplot-display .controls .data-prev-next { + display: inline-block; + margin: 0px 4px 0px 0px; +} + +.scatterplot-display .controls .pagination-container { + display: inline-block; + overflow: auto; + max-width: 256px; + /*very tweaky*/ + margin: 0px 8px 0px 0px; + height: 27px; + padding-top: 5px; +} +.scatterplot-display .controls .pagination-container a { + display: inline; + /* WTH, bootstrap? */ + position: static; + float: none; + /* up from 4px -> 5px */ + padding: 5px 10px 5px 10px; +} + +.scatterplot-display .controls .right { + float: right; + margin-bottom: 4px; +} + +.scatterplot-display .controls p { + display: inline-block; + margin: 0px 4px 0px 4px; + /*to better match the buttons*/ + vertical-align: middle; + line-height: 24px; +} + +.scatterplot-display button.rerender-btn { + margin-left: 4px; +} + +.scatterplot-display button.stats-toggle-btn { + margin-left: 4px; } /* -------------------------------------------- chart area */ -#chart-holder { - overflow: auto; - margin-left: 8px; +svg.scatterplot { + border: 1px solid lightgrey; + border-radius: 3px; } -svg .grid-line { +svg.scatterplot .grid-line { fill: none; stroke: lightgrey; - stroke-opacity: 0.5; + stroke-opacity: 0.2; shape-rendering: crispEdges; - stroke-dasharray: 3, 3; } -svg .axis path, svg .axis line { +svg.scatterplot .axis path, +svg.scatterplot .axis line { fill: none; stroke: black; shape-rendering: crispEdges; } -svg .axis text { - font-family: monospace; - font-size: 12px; +svg.scatterplot .axis text { + font-size: 9px; } -svg #x-axis-label, svg #y-axis-label { +svg.scatterplot .axis-label { font-family: sans-serif; - font-size: 10px; + font-size: 13px; } -svg .glyph { +svg.scatterplot .glyph { stroke: none; fill: black; fill-opacity: 0.2; diff --git a/config/plugins/visualizations/scatterplot/static/scatterplot.js b/config/plugins/visualizations/scatterplot/static/scatterplot.js deleted file mode 100644 index fbb5e49695e..00000000000 --- a/config/plugins/visualizations/scatterplot/static/scatterplot.js +++ /dev/null @@ -1 +0,0 @@ -function TwoVarScatterplot(a){var b=10,c=7,d=10,e=8,f=5;this.log=function(){if(this.debugging&&console&&console.debug){var a=Array.prototype.slice.call(arguments);a.unshift(this.toString()),console.debug.apply(console,a)}},this.log("new TwoVarScatterplot:",a),this.defaults={id:"TwoVarScatterplot",containerSelector:"body",maxDataPoints:3e4,datapointSize:4,animDuration:500,xNumTicks:10,yNumTicks:10,xAxisLabelBumpY:40,yAxisLabelBumpX:-40,width:400,height:400,marginTop:50,marginRight:50,marginBottom:50,marginLeft:50,xMin:null,xMax:null,yMin:null,yMax:null,xLabel:"X",yLabel:"Y"},this.config=_.extend({},this.defaults,a),this.log("intial config:",this.config),this.updateConfig=function(a){_.extend(this.config,a),this.log(this+".updateConfig:",this.config)},this.toString=function(){return this.config.id},this.translateStr=function(a,b){return"translate("+a+","+b+")"},this.rotateStr=function(a,b,c){return"rotate("+a+","+b+","+c+")"},this.adjustChartDimensions=function(a,b,c,d){a=a||0,b=b||0,c=c||0,d=d||0,this.svg.attr("width",this.config.width+(this.config.marginRight+b)+(this.config.marginLeft+d)).attr("height",this.config.height+(this.config.marginTop+a)+(this.config.marginBottom+c)).style("display","block"),this.content=this.svg.select("g.content").attr("transform",this.translateStr(this.config.marginLeft+d,this.config.marginTop+a))},this.preprocessData=function(a){return a.length>this.config.maxDataPoints?a.slice(0,this.config.maxDataPoints):a},this.findMinMaxes=function(a,b,c){this.xMin=this.config.xMin||c?c[0].min:d3.min(a),this.xMax=this.config.xMax||c?c[0].max:d3.max(a),this.yMin=this.config.yMin||c?c[1].min:d3.min(b),this.yMax=this.config.yMax||c?c[1].max:d3.max(b)},this.setUpScales=function(){this.xScale=d3.scale.linear().domain([this.xMin,this.xMax]).range([0,this.config.width]),this.yScale=d3.scale.linear().domain([this.yMin,this.yMax]).range([this.config.height,0])},this.setUpXAxis=function(){this.xAxisFn=d3.svg.axis().scale(this.xScale).ticks(this.config.xNumTicks).orient("bottom"),this.xAxis.attr("transform",this.translateStr(0,this.config.height)).call(this.xAxisFn);var a=d3.max(_.map([this.xMin,this.xMax],function(a){return String(a).length}));a>=f&&this.xAxis.selectAll("g").filter(":nth-child(odd)").style("display","none"),this.log("this.config.xLabel:",this.config.xLabel),this.xAxisLabel.attr("x",this.config.width/2).attr("y",this.config.xAxisLabelBumpY).attr("text-anchor","middle").text(this.config.xLabel),this.log("xAxisLabel:",this.xAxisLabel)},this.setUpYAxis=function(){this.yAxisFn=d3.svg.axis().scale(this.yScale).ticks(this.config.yNumTicks).orient("left"),this.yAxis.call(this.yAxisFn);var a=this.yAxis.selectAll("text").filter(function(a,b){return 0!==b});this.log("yTickLabels:",a),this.yLongestLabel=d3.max(a[0].map(function(a){return d3.select(a).text().length}))||0;var f=b+this.yLongestLabel*c+e+d;if(this.config.yAxisLabelBumpX=-(f-d),this.config.marginLeftg?0:g,this.adjustChartDimensions(0,0,0,g)}this.yAxisLabel.attr("x",this.config.yAxisLabelBumpX).attr("y",this.config.height/2).attr("text-anchor","middle").attr("transform",this.rotateStr(-90,this.config.yAxisLabelBumpX,this.config.height/2)).text(this.config.yLabel)},this.renderGrid=function(){this.vGridLines=this.content.selectAll("line.v-grid-line").data(this.xScale.ticks(this.xAxisFn.ticks()[0])),this.vGridLines.enter().append("svg:line").classed("grid-line v-grid-line",!0),this.vGridLines.attr("x1",this.xScale).attr("y1",0).attr("x2",this.xScale).attr("y2",this.config.height),this.vGridLines.exit().remove(),this.hGridLines=this.content.selectAll("line.h-grid-line").data(this.yScale.ticks(this.yAxisFn.ticks()[0])),this.hGridLines.enter().append("svg:line").classed("grid-line h-grid-line",!0),this.hGridLines.attr("x1",0).attr("y1",this.yScale).attr("x2",this.config.width).attr("y2",this.yScale),this.hGridLines.exit().remove()},this.renderDatapoints=function(a,b,c){this.log(this+".renderDatapoints",arguments);var d=0,e=this,f=function(b,c){return e.xScale(a[c])},g=function(a,c){return e.yScale(b[c])},h=this.content.selectAll(".glyph").data(a);d=0,h.enter().append("svg:circle").each(function(){d+=1}).classed("glyph",!0).attr("cx",0).attr("cy",this.config.height).attr("r",0),this.log(d," new glyphs created"),d=0,h.transition().duration(this.config.animDuration).each(function(){d+=1}).attr("cx",f).attr("cy",g).attr("r",e.config.datapointSize),this.log(d," existing glyphs transitioned"),h.exit().each(function(){d+=1}).transition().duration(this.config.animDuration).attr("cy",this.config.height).attr("r",0).remove(),this.log(d," glyphs removed"),this._addDatapointEventhandlers(h,a,b,c)},this._addDatapointEventhandlers=function(a,b,c,d){var e=this;a.on("mouseover",function(a,f){var g=d3.select(this);g.style("fill","red").style("fill-opacity",1),e.content.append("line").attr("stroke","red").attr("stroke-width",1).attr("x1",g.attr("cx")-e.config.datapointSize).attr("y1",g.attr("cy")).attr("x2",0).attr("y2",g.attr("cy")).classed("hoverline",!0),g.attr("cy")2?a[2]:void 0;c=this.preprocessData(c),d=this.preprocessData(d),this.log("xCol len",c.length,"yCol len",d.length),this.findMinMaxes(c,d,b),this.setUpScales(),this.svg||(this.svg=d3.select("svg").attr("class","chart")),this.content||(this.content=this.svg.append("svg:g").attr("class","content").attr("id",this.config.id)),this.adjustChartDimensions(),this.xAxis||(this.xAxis=this.content.append("g").attr("class","axis").attr("id","x-axis")),this.xAxisLabel||(this.xAxisLabel=this.xAxis.append("text").attr("class","axis-label").attr("id","x-axis-label")),this.yAxis||(this.yAxis=this.content.append("g").attr("class","axis").attr("id","y-axis")),this.yAxisLabel||(this.yAxisLabel=this.yAxis.append("text").attr("class","axis-label").attr("id","y-axis-label")),this.setUpXAxis(),this.setUpYAxis(),this.renderGrid(),this.renderDatapoints(c,d,e)},this.infoHtml=function(a,b,c){var d=$("
        ");return c&&$("
        ").text(c).css("font-weight","bold").appendTo(d),$("
        ").text(a).appendTo(d),$("
        ").text(b).appendTo(d),d.html()},this.infoBox=function(a,b,c,d,e){d=d||0,e=e||20;var f=$("
        ").addClass("chart-info-box").css({position:"absolute",top:a+d,left:b+e});return f.html(c),f}}this.Templates=this.Templates||{},this.Templates.chartControl=Handlebars.template(function(a,b,c,d,e){function f(){return' checked="true"'}this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var g,h="",i="function",j=this.escapeExpression,k=this;return h+='

        \n Use the following controls to how the chart is displayed.\n The slide controls can be moved by the mouse or, if the \'handle\' is in focus, your keyboard\'s arrow keys.\n Move the focus between controls by using the tab or shift+tab keys on your keyboard.\n Use the \'Draw\' button to render (or re-render) the chart with the current settings.\n

        \n\n
        \n \n
        ',(g=c.datapointSize)?g=g.call(b,{hash:{},data:e}):(g=b.datapointSize,g=typeof g===i?g.apply(b):g),h+=j(g)+'
        \n
        \n

        \n Size of the graphic representation of each data point\n

        \n
        \n\n
        \n \n \n

        \n Uncheck this to disable the animations used on the chart\n

        \n
        \n\n
        \n \n
        ',(g=c.width)?g=g.call(b,{hash:{},data:e}):(g=b.width,g=typeof g===i?g.apply(b):g),h+=j(g)+'
        \n
        \n

        \n (not including chart margins and axes)\n

        \n
        \n\n
        \n \n
        ',(g=c.height)?g=g.call(b,{hash:{},data:e}):(g=b.height,g=typeof g===i?g.apply(b):g),h+=j(g)+'
        \n
        \n

        \n (not including chart margins and axes)\n

        \n
        \n\n
        \n \n \n

        \n
        \n\n
        \n \n \n

        \n
        \n\n '}),this.Templates.chartDisplay=Handlebars.template(function(a,b,c,d,e){this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var f,g="",h="function",i=this.escapeExpression;return g+=''}),this.Templates.dataControl=Handlebars.template(function(a,b,c,d,e){function f(a,b){var d,e="";return e+='\n \n "}function g(){return'checked="true"'}this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var h,i="",j="function",k=this.escapeExpression,l=this;return i+='

        \n Use the following controls to change the data used by the chart.\n Use the \'Draw\' button to render (or re-render) the chart with the current settings.\n

        \n\n \n
        \n \n \n
        \n
        \n \n \n
        \n\n \n
        \n \n \n

        \n These will be displayed (along with the x and y values) when you hover over\n a data point.\n

        \n
        \n \n\n \n \n\n \n
        '}),this.Templates.scatterplotControlForm=Handlebars.template(function(a,b,c,d,e){this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var f,g="",h="function",i=this.escapeExpression;return g+='\n\n
        \n \n \n\n "+'\n
        \n '+'\n
        \n '+"\n
        \n \n "+'\n
        \n '+"\n
        \n\n "+'\n
        \n '+"\n
        \n\n "+'\n
        \n '+"\n
        \n\n
        "+"\n
        "}),this.Templates.statsDisplay=Handlebars.template(function(a,b,c,d,e){function f(a,b){var d,e="";return e+="\n
        \n \n "}this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var g,h="",i="function",j=this.escapeExpression,k=this;return h+='

        By column:

        \n
        ",d,"",e,"
        ",b,"",a[0],"",a[1],"
        ",(d=c.name)?d=d.call(a,{hash:{},data:b}):(d=a.name,d=typeof d===i?d.apply(a):d),e+=j(d)+"",(d=c.xval)?d=d.call(a,{hash:{},data:b}):(d=a.xval,d=typeof d===i?d.apply(a):d),e+=j(d)+"",(d=c.yval)?d=d.call(a,{hash:{},data:b}):(d=a.yval,d=typeof d===i?d.apply(a):d),e+=j(d)+"
        \n \n ',g=c.each.call(b,b.stats,{hash:{},inverse:k.noop,fn:k.program(1,f,e),data:e}),(g||0===g)&&(h+=g),h+="\n
        XY
        "});var ScatterplotControlForm=BaseView.extend(LoggableMixin).extend({className:"scatterplot-control-form",dataLoadDelay:4e3,dataLoadSize:5e3,loadingIndicatorImage:"loading_small_white_bg.gif",fetchMsg:"Fetching data...",renderMsg:"Rendering...",initialize:function(a){this.log(this+".initialize, attributes:",a),this.dataset=null,this.chartConfig=null,this.chart=null,this.loader=null,this.$dataControl=null,this.$chartControl=null,this.$statsDisplay=null,this.$chartDisplay=null,this.dataFetch=null,this.initializeFromAttributes(a),this.initializeChart(a),this.initializeDataLoader(a)},initializeFromAttributes:function(a){if(!a||!a.dataset)throw"ScatterplotView requires a dataset";if(this.dataset=a.dataset,"string"===jQuery.type(this.dataset.metadata_column_types)&&(this.dataset.metadata_column_types=this.dataset.metadata_column_types.split(", ")),this.log(" dataset:",this.dataset),this.dataset.comment_lines&&this.dataset.comment_lines.length){var b=this.dataset.comment_lines[0],c=b.split(" ");c.length===this.dataset.metadata_column_types.length&&(this.possibleHeaders=c)}if(!a.apiDatasetsURL)throw"ScatterplotView requires a apiDatasetsURL";this.dataURL=a.apiDatasetsURL+"/"+this.dataset.id+"?",this.log(" dataURL:",this.dataURL)},initializeChart:function(a){this.chartConfig=a.chartConfig||{},this.log(" initial chartConfig:",this.chartConfig),this.chart=new TwoVarScatterplot(this.chartConfig),this.chartConfig=this.chart.config},initializeDataLoader:function(a){var b=this;this.loader=new LazyDataLoader({url:null,start:a.start||0,total:a.total||this.dataset.metadata_data_lines,delay:this.dataLoadDelay,size:this.dataLoadSize,buildUrl:function(a,b){return this.url+"&"+jQuery.param({start_val:a,max_vals:b})}}),$(this.loader).bind("error",function(a,c,d){b.log("ERROR:",c,d),alert("ERROR fetching data:\n"+c+"\n"+d),b.hideLoadingIndicator()})},render:function(){return this.log(this+".render"),this.$el.append(ScatterplotControlForm.templates.mainLayout({loadingIndicatorImagePath:"/static/images/"+this.loadingIndicatorImage,message:""})),this.$dataControl=this._render_dataControl(),this.$chartControl=this._render_chartControl(),this.$statsDisplay=this.$el.find(".tab-pane#stats-display"),this.$chartDisplay=this._render_chartDisplay(),this.chartConfig.xColumn&&this.chartConfig.yColumn&&this.renderChart(),this.$el.find(".tooltip").tooltip(),this},_render_dataControl:function(){var a=this,b=[],c=[],d=this.possibleHeaders&&this.$dataControl?this.$dataControl.find("#first-line-header-checkbox").is(":checked"):!1;_.each(this.dataset.metadata_column_types,function(e,f){var g=f+1,h="column "+g;a.dataset.metadata_column_names?h=a.dataset.metadata_column_names[f]:d&&(h=a.possibleHeaders[f]),b.push({index:g,name:h}),("int"===e||"float"===e)&&c.push({index:g,name:h})});var e=this.$el.find(".tab-pane#data-control");return e.html(ScatterplotControlForm.templates.dataControl({allColumns:b,numericColumns:c,possibleHeaders:this.possibleHeaders?this.possibleHeaders.join(", "):"",usePossibleHeaders:d})),!this.dataset.metadata_column_names&&this.possibleHeaders&&e.find("#first-line-header").show(),e.find("#X-select").val(this.chartConfig.xColumn),e.find("#Y-select").val(this.chartConfig.yColumn),void 0!==this.chartConfig.idColumn&&(e.find("#include-id-checkbox").attr("checked",!0).trigger("change"),e.find("#ID-select").val(this.chartConfig.idColumn)),e},_render_chartControl:function(){var a=this,b=this.$el.find(".tab-pane#chart-control"),c={datapointSize:{min:2,max:10,step:1},width:{min:200,max:800,step:20},height:{min:200,max:800,step:20}};return b.append(ScatterplotControlForm.templates.chartControl(this.chartConfig)),b.find(".numeric-slider-input").each(function(){function b(){var a=$(this),b=a.slider("value");e.text(b)}var d=$(this),e=d.find(".slider-output"),f=d.find(".slider"),g=d.attr("id");f.slider(_.extend(c[g],{value:a.chartConfig[g],change:b,slide:b}))}),b},_render_chartDisplay:function(){var a=this.$el.find(".tab-pane#chart-display");return a.append(ScatterplotControlForm.templates.chartDisplay(this.chartConfig)),a},events:{"change #include-id-checkbox":"toggleThirdColumnSelector","change #first-line-header-checkbox":"rerenderDataControl","click #data-control #render-button":"renderChart","click #chart-control #render-button":"changeChartSettings"},toggleThirdColumnSelector:function(){this.$el.find('select[name="ID"]').parent().toggle()},rerenderDataControl:function(){this.$dataControl=this._render_dataControl()},showLoadingIndicator:function(a,b){a=a||"";var c=this.$el.find("div#loading-indicator");messageBox=c.find(".loading-message"),c.is(":visible")?a?messageBox.fadeOut("fast",function(){messageBox.text(a),messageBox.fadeIn("fast",b)}):b():(a&&messageBox.text(a),c.fadeIn("fast",b))},hideLoadingIndicator:function(a){this.$el.find("div#loading-indicator").fadeOut("fast",a)},renderChart:function(){this.log(this+".renderChart"),this.data=null,this.meta=null,_.extend(this.chartConfig,this.getChartSettings()),this.log(" chartConfig:",this.chartConfig),this.chart.updateConfig(this.chartConfig,!1),this.loader.url=this.dataURL+"&"+jQuery.param(this.getDataSettings()),this.log(" loader: total lines:",this.loader.total," url:",this.loader.url);var a=this;$(this.loader).bind("loaded.new",function(b,c){a.log(a+" loaded.new",c),a.postProcessDataFetchResponse(c),a.log(" postprocessed data:",a.data),a.log(" postprocessed meta:",a.meta),a.showLoadingIndicator(a.renderMsg,function(){a.chart.render(a.data,a.meta),a.renderStats(a.data,a.meta),a.hideLoadingIndicator()})}),$(this.loader).bind("complete",function(b,c){a.log(a+" complete",c),$(a.loader).unbind()}),a.showLoadingIndicator(a.fetchMsg,function(){a.$el.find("ul.nav").find('a[href="#chart-display"]').tab("show"),a.loader.load()})},renderStats:function(){this.log(this+".renderStats"),this.$statsDisplay.html(ScatterplotControlForm.templates.statsDisplay({stats:[{name:"Count",xval:this.meta[0].count,yval:this.meta[1].count},{name:"Min",xval:this.meta[0].min,yval:this.meta[1].min},{name:"Max",xval:this.meta[0].max,yval:this.meta[1].max},{name:"Sum",xval:this.meta[0].sum,yval:this.meta[1].sum},{name:"Mean",xval:this.meta[0].mean,yval:this.meta[1].mean},{name:"Median",xval:this.meta[0].median,yval:this.meta[1].median}]}))},changeChartSettings:function(){var a=this;newChartSettings=this.getChartSettings(),_.extend(this.chartConfig,newChartSettings),this.log("this.chartConfig:",this.chartConfig),this.chart.updateConfig(this.chartConfig,!1),a.data&&a.meta?a.showLoadingIndicator(a.renderMsg,function(){a.$el.find("ul.nav").find('a[href="#chart-display"]').tab("show"),a.chart.render(a.data,a.meta),a.hideLoadingIndicator()}):this.renderChart()},postProcessDataFetchResponse:function(a){this.postProcessData(a.data),this.postProcessMeta(a.meta)},postProcessData:function(a){var b=this;b.data?_.each(a,function(a,c){b.data[c]=b.data[c].concat(a)}):b.data=a},postProcessMeta:function(a){var b=this,c=this.dataset.metadata_column_types;b.meta?_.each(a,function(a,d){var e=b.meta[d],f=c[d];if(e.count+=a.count?a.count:0,"int"===f||"float"===f){e.min=Math.min(a.min,e.min),e.max=Math.max(a.max,e.max),e.sum=a.sum+e.sum,e.mean=e.count?e.sum/e.count:null;var g=b.data[d].slice().sort(),h=Math.floor(g.length/2);e.median=0===g.length%2?(g[h]+g[h+1])/2:g[h]}}):b.meta=a},getDataSettings:function(){var a=this.getColumnSelections(),b=[];this.log(" columnSelections:",a),b=[a.X.colIndex-1,a.Y.colIndex-1],this.$dataControl.find("#include-id-checkbox").attr("checked")&&b.push(a.ID.colIndex-1);var c={data_type:"raw_data",provider:"column_with_stats",columns:"["+b+"]"};return this.log(" data settings (url params):",c),c},getColumnSelections:function(){var a={};return this.$dataControl.find("div.column-select select").each(function(){var b=$(this),c=b.val();a[b.attr("name")]={colIndex:c,colName:b.children('[value="'+c+'"]').text()}}),a},getChartSettings:function(){var a={},b=this.getColumnSelections();a.datapointSize=this.$chartControl.find("#datapointSize.numeric-slider-input").find(".slider").slider("value"),a.width=this.$chartControl.find("#width.numeric-slider-input").find(".slider").slider("value"),a.height=this.$chartControl.find("#height.numeric-slider-input").find(".slider").slider("value");var c=this.$chartControl.find("input#X-axis-label").val(),d=this.$chartControl.find("input#Y-axis-label").val();return a.xLabel="X"===c?b.X.colName:c,a.yLabel="Y"===d?b.Y.colName:d,a.animDuration=this.$chartControl.find("#animate-chart").is(":checked")?this.chart.defaults.animDuration:0,this.log(" chartSettings:",a),a},toString:function(){return"ScatterplotControlForm("+(this.dataset?this.dataset.id:"")+")"}});ScatterplotControlForm.templates={mainLayout:Templates.scatterplotControlForm,dataControl:Templates.dataControl,chartControl:Templates.chartControl,statsDisplay:Templates.statsDisplay,chartDisplay:Templates.chartDisplay}; \ No newline at end of file diff --git a/config/plugins/visualizations/scatterplot/static/worker-stats.js b/config/plugins/visualizations/scatterplot/static/worker-stats.js new file mode 100644 index 00000000000..315374694a7 --- /dev/null +++ b/config/plugins/visualizations/scatterplot/static/worker-stats.js @@ -0,0 +1,5 @@ +onmessage = function( event ){ + importScripts( 'numeric-column-stats.js' ); + postMessage( numericColumnStats( event.data.data, event.data.keys ) ); + self.close(); +}; diff --git a/config/plugins/visualizations/scatterplot/templates/scatterplot.mako b/config/plugins/visualizations/scatterplot/templates/scatterplot.mako index 9cd2920a34a..06d9a65b4e9 100644 --- a/config/plugins/visualizations/scatterplot/templates/scatterplot.mako +++ b/config/plugins/visualizations/scatterplot/templates/scatterplot.mako @@ -1,8 +1,19 @@ +<% + default_title = "Scatterplot of '" + hda.name + "'" + info = hda.name + if hda.info: + info += ' : ' + hda.info + + # optionally bootstrap data from dprov + ##data = list( hda.datatype.dataset_column_dataprovider( hda, limit=10000 ) ) +%> +## ---------------------------------------------------------------------------- + -${hda.name} | ${visualization_name} +${title or default_title} | ${visualization_display_name} ## ---------------------------------------------------------------------------- @@ -13,55 +24,50 @@ ## ---------------------------------------------------------------------------- + + - - - - + + - - + ## ---------------------------------------------------------------------------- %if not embedded: ## dataset info: only show if on own page -
        -

        Scatterplot of '${hda.name}'

        -

        ${hda.info}

        +
        +

        ${title or default_title}

        +

        ${info}

        -%endif - -
        +
        +%endif diff --git a/config/plugins/visualizations/sweepster/config/sweepster.xml b/config/plugins/visualizations/sweepster/config/sweepster.xml index e2ef46223d8..66ea9b9797b 100644 --- a/config/plugins/visualizations/sweepster/config/sweepster.xml +++ b/config/plugins/visualizations/sweepster/config/sweepster.xml @@ -1,6 +1,6 @@ - + HistoryDatasetAssociation @@ -23,5 +23,5 @@ webapps/galaxy/visualization - _top + _top diff --git a/config/plugins/visualizations/trackster/config/trackster.xml b/config/plugins/visualizations/trackster/config/trackster.xml index ff420a44ac7..a27356096cc 100644 --- a/config/plugins/visualizations/trackster/config/trackster.xml +++ b/config/plugins/visualizations/trackster/config/trackster.xml @@ -1,6 +1,6 @@ - + @@ -26,5 +26,5 @@ webapps/galaxy/visualization/tracks - _top + _top diff --git a/config/plugins/visualizations/visualization.dtd b/config/plugins/visualizations/visualization.dtd index d874f6fd77a..b84f2949f00 100644 --- a/config/plugins/visualizations/visualization.dtd +++ b/config/plugins/visualizations/visualization.dtd @@ -1,10 +1,17 @@ - - + + + + + - - - - + diff --git a/doc/source/lib/galaxy.tools.util.rst b/doc/source/lib/galaxy.tools.util.rst index a68c484c408..04efb192a74 100644 --- a/doc/source/lib/galaxy.tools.util.rst +++ b/doc/source/lib/galaxy.tools.util.rst @@ -9,14 +9,6 @@ util Package :undoc-members: :show-inheritance: -:mod:`hyphy_util` Module ------------------------- - -.. automodule:: galaxy.tools.util.hyphy_util - :members: - :undoc-members: - :show-inheritance: - :mod:`maf_utilities` Module --------------------------- diff --git a/doc/source/lib/galaxy.webapps.galaxy.api.rst b/doc/source/lib/galaxy.webapps.galaxy.api.rst index 7c3b79d9306..0acce3e432d 100644 --- a/doc/source/lib/galaxy.webapps.galaxy.api.rst +++ b/doc/source/lib/galaxy.webapps.galaxy.api.rst @@ -205,6 +205,11 @@ used in the web form, like 'link_data_only' and etc. The request and response format should be considered alpha and are subject to change. +API Return Codes and Formats +================== + +A set of error codes for API requests is being established and will be +documented here. This is a long-term project however so stayed tuned. API Controllers =============== @@ -297,6 +302,14 @@ API Controllers :undoc-members: :show-inheritance: +:mod:`lda_datasets` Module +-------------------------- + +.. automodule:: galaxy.webapps.galaxy.api.lda_datasets + :members: + :undoc-members: + :show-inheritance: + :mod:`libraries` Module ----------------------- @@ -393,3 +406,67 @@ API Controllers :undoc-members: :show-inheritance: + +API Design Guidelines +===================== + +The following section outlines guidelines related to extending and/or modifing +the Galaxy API. The Galaxy API has grown in an ad-hoc fashion over time by +many contributors and so clients SHOULD NOT expect the API will conform to +these guidelines - but developers contributing to the Galaxy API SHOULD follow +these guidelines. + + - API functionality should include docstring documentation for consumption + by readthedocs.org. + - Developers should familarize themselves with the HTTP status code definitions + http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html. The API responses + should properly set the status code according to the result - in particular + 2XX responses should be used for successful requests, 4XX for various + kinds of client errors, and 5XX for errors on the server side. + - If there is an error processing some part of request (one item in a list + for instance), the status code should be set to reflect the error and the + partial result may or may not be returned depending on the controller - + this behavior should be documented. + - (TODO) API methods should throw a finite number of exceptions (defined in) + `galaxy.exceptions` and these should subclass `MessageException` and not + paste/wsgi HTTP exceptions. When possible, the framework itself should be + responsible catching these exceptions, setting the status code, and + building an error response. + - Error responses should not consist of plain text strings - they should be + dictionaries describing the error and containing the following keys (TODO: + spell out nature of this.) Various error conditions (once a format has + been chosen and framework to enforce it in place) should be spelled out + in this document. + - Backward compatibility is important and should maintained when possible. + If changing behavior in a non-backward compatibile way please ensure one + of the following holds - there is a strong reason to believe no consumers + depend on a behavior, the behavior is effectively broken, or the API + method being modified has not been part of a tagged dist release. + +The following bullet points represent good practices more than guidelines, please +consider them when modifying the API. + + - Functionality should not be copied and pasted between controllers - + consider refactoring functionality into associated classes or short of + that into Mixins (http://en.wikipedia.org/wiki/Composition_over_inheritance). + - API additions are more permanent changes to Galaxy than many other potential + changes and so a second opinion on API changes should be sought. (Consider a + pull request!) + - New API functionality should include functional tests. These functional + tests should be implemented in Python and placed in + `test/functional/api`. (Once such a framework is in place - it is not + right now). + - Changes to reflect modifications to the API should be pushed upstream to + the BioBlend project possible. + +Longer term goals/notes. + + - It would be advantageous to have a clearer separation of anonymous and + admin handling functionality. + - If at some point in the future, functionality needs to be added that + breaks backward compatibility in a significant way to a compontent used by + the community should be alerted - a "dev" variant of the API will be + established and the community should be alerted and given a timeframe + for when the old behavior will be replaced with the new behavior. + - Consistent standards for range-based requests, batch requests, filtered + requests, etc... should be established and documented here. diff --git a/eggs.ini b/eggs.ini index 1c0491984b7..43e432a184c 100644 --- a/eggs.ini +++ b/eggs.ini @@ -26,9 +26,8 @@ pycrypto = 2.5 pysam = 0.4.2 pysqlite = 2.5.6 python_lzo = 1.08_2.03_static -simplejson = 2.1.1 threadframe = 0.2 -guppy = 0.1.8 +guppy = 0.1.10 SQLAlchemy = 0.7.9 ; msgpack_python = 0.2.4 @@ -54,6 +53,7 @@ PasteDeploy = 1.5.0 pexpect = 2.4 python_openid = 2.2.5 python_daemon = 1.5.5 +pytz = 2013.9 Routes = 1.12.3 sqlalchemy_migrate = 0.7.2 ssh = 1.7.14 @@ -61,10 +61,10 @@ SVGFig = 1.1.6 Tempita = 0.5.1 twill = 0.9 WebError = 0.8a -WebHelpers = 0.2 +WebHelpers = 1.3 WebOb = 0.8.5 wsgiref = 0.1.2 -Babel = 0.9.4 +Babel = 1.3 wchartype = 0.1 Whoosh = 0.3.18 ; fluent_logger = 0.3.3 diff --git a/install_and_test_tool_shed_repositories.sh b/install_and_test_tool_shed_repositories.sh index 730c2322a53..880dc695fed 100644 --- a/install_and_test_tool_shed_repositories.sh +++ b/install_and_test_tool_shed_repositories.sh @@ -2,20 +2,31 @@ # A good place to look for nose info: http://somethingaboutorange.com/mrl/projects/nose/ -# The test/install_and_test_tool_shed_repositories/functional_tests.py can not be executed directly, because it must have certain functional test definitions -# in sys.argv. Running it through this shell script is the best way to ensure that it has the required definitions. +# The test/install_and_test_tool_shed_repositories/functional_tests.py cannot be executed directly because it must +# have certain functional test definitions in sys.argv. Running it through this shell script is the best way to +# ensure that it has the required definitions. -# This script requires the following environment variables: +# This script requires setting of the following environment variables: # GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY - must be set to the API key for the tool shed that is being checked. # GALAXY_INSTALL_TEST_TOOL_SHED_URL - must be set to a URL that the tool shed is listening on. -# If the tool shed url is not specified in tool_sheds_conf.xml, GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF must be set to a tool sheds configuration file -# that does specify that url, otherwise repository installation will fail. + +# If the tool shed url is not specified in tool_sheds_conf.xml, GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF must be set to +# a tool sheds configuration file that does specify that url or repository installation will fail. + +# This script accepts the command line option -w to select which set of tests to run. The default behavior is to test +# first tool_dependency_definition repositories and then repositories with tools. Provide the value 'dependencies' +# to test only tool_dependency_definition repositories or 'tools' to test only repositories with tools. if [ -z $GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY ] ; then echo "This script requires the GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY environment variable to be set and non-empty." exit 1 fi +if [ -z $GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR ] ; then + echo "This script requires the GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR environment variable to be set to your configured tool dependency path." + exit 1 +fi + if [ -z $GALAXY_INSTALL_TEST_TOOL_SHED_URL ] ; then echo "This script requires the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable to be set and non-empty." exit 1 @@ -37,7 +48,66 @@ else fi fi -python test/install_and_test_tool_shed_repositories/functional_tests.py $* -v --with-nosehtml --html-report-file \ - test/install_and_test_tool_shed_repositories/run_functional_tests.html \ - test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py \ - test/functional/test_toolbox.py +if [ -z $GALAXY_INSTALL_TEST_SHED_TOOL_PATH ] ; then + export GALAXY_INSTALL_TEST_SHED_TOOL_PATH='/tmp/shed_tools' +fi + +if [ ! -d $GALAXY_INSTALL_TEST_SHED_TOOL_PATH ] ; then + mkdir -p $GALAXY_INSTALL_TEST_SHED_TOOL_PATH +fi + +test_tool_dependency_definitions () { + # Test installation of repositories of type tool_dependency_definition. + if [ -f $GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR/stage_1_complete ] ; then + rm $GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR/stage_1_complete + fi + echo "Starting stage 1, tool dependency definitions." + python test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py $* -v --with-nosehtml --html-report-file \ + test/install_and_test_tool_shed_repositories/tool_dependency_definitions/run_functional_tests.html \ + test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py \ + test/functional/test_toolbox.py + echo "Stage 1 complete, exit code $?" + touch $GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR/stage_1_complete +} + +test_repositories_with_tools () { + if [ ! -f $GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR/stage_1_complete ] ; then + echo 'Stage 1 did not complete its run, exiting.' + exit 1 + fi + echo "Starting stage 2, repositories with tools." + # Test installation of repositories that contain valid tools with defined functional tests and a test-data directory containing test files. + python test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py $* -v --with-nosehtml --html-report-file \ + test/install_and_test_tool_shed_repositories/repositories_with_tools/run_functional_tests.html \ + test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py \ + test/functional/test_toolbox.py + echo "Stage 2 complete, exit code $?" + rm $GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR/stage_1_complete +} + +which='both' + +while getopts "w:" arg; do + case $arg in + w) + which=$OPTARG + ;; + esac +done + +case $which in + # Use "-w tool_dependency_definitions" when you want to test repositories of type tool_dependency_definition. + tool_dependency_definitions) + test_tool_dependency_definitions + ;; + # Use "-w repositories_with_tools" parameter when you want to test repositories that contain tools. + repositories_with_tools) + test_repositories_with_tools + ;; + # No received parameters or any received parameter not in [ tool_dependency_definitions, repositories_with_tools ] + # will execute both scripts. + *) + test_tool_dependency_definitions + test_repositories_with_tools + ;; +esac diff --git a/job_conf.xml.sample_advanced b/job_conf.xml.sample_advanced index 512f5ff6c30..8c9741241bb 100644 --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -6,7 +6,19 @@ --> - + + + ok + 0 + ok + 0 + + + + /sge/lib/libdrmaa.so + @@ -14,6 +26,7 @@ + + + + + @@ -52,15 +74,44 @@ 123456789changeme + server). Alternatively action can be set to 'copy' - to replace + http transfers with file system copies. --> + + + + + + + + + + + SecureShell Torque foo foo.example.org - 24:00:00 + walltime=24:00:00,ncpus=4 + + ucsc_tools + grep -v "^track" $input | wigToBigWig -clip stdin $chromInfo $output @@ -9,4 +12,4 @@ - \ No newline at end of file + diff --git a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml index b4890808a13..b4bd82c03a4 100644 --- a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml +++ b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml @@ -1,6 +1,9 @@ + + ucsc_tools + faToTwoBit $input $output diff --git a/lib/galaxy/datatypes/converters/fasta_to_bowtie_base_index_converter.xml b/lib/galaxy/datatypes/converters/fasta_to_bowtie_base_index_converter.xml index b7645d1ae21..e1917c1a618 100644 --- a/lib/galaxy/datatypes/converters/fasta_to_bowtie_base_index_converter.xml +++ b/lib/galaxy/datatypes/converters/fasta_to_bowtie_base_index_converter.xml @@ -2,6 +2,9 @@ bowtie + + bowtie + mkdir ${output.files_path} && bowtie-build --quiet diff --git a/lib/galaxy/datatypes/converters/fasta_to_bowtie_color_index_converter.xml b/lib/galaxy/datatypes/converters/fasta_to_bowtie_color_index_converter.xml index 1059a9a796d..f34c623d97c 100644 --- a/lib/galaxy/datatypes/converters/fasta_to_bowtie_color_index_converter.xml +++ b/lib/galaxy/datatypes/converters/fasta_to_bowtie_color_index_converter.xml @@ -2,6 +2,9 @@ bowtie + + bowtie + mkdir ${output.files_path} && bowtie-build --quiet diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.xml b/lib/galaxy/datatypes/converters/sam_to_bam.xml index 1a5ce77b794..8d743d8ad7a 100644 --- a/lib/galaxy/datatypes/converters/sam_to_bam.xml +++ b/lib/galaxy/datatypes/converters/sam_to_bam.xml @@ -5,6 +5,9 @@ To fix this: (a) merge sam_to_bam tool in tools with this conversion (like fasta_to_len conversion); and (b) define a datatype-specific way to set converter parameters. --> + + samtools + sam_to_bam.py $input1 $output diff --git a/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml index d90702efe78..b32de112523 100644 --- a/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml +++ b/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml @@ -1,5 +1,8 @@ \ No newline at end of file + diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 32f47575669..62022dddecb 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -199,6 +199,26 @@ class Data( object ): out = "Can't create peek %s" % str( exc ) return out + def _archive_main_file(self, archive, display_name, data_filename): + """Called from _archive_composite_dataset to add central file to archive. + + Unless subclassed, this will add the main dataset file (argument data_filename) + to the archive, as an HTML file with its filename derived from the dataset name + (argument outfname). + + Returns a tuple of boolean, string, string: (error, msg, messagetype) + """ + error, msg, messagetype = False, "", "" + archname = '%s.html' % display_name # fake the real nature of the html file + try: + archive.add(data_filename, archname) + except IOError: + error = True + log.exception("Unable to add composite parent %s to temporary library download archive" % data_filename) + msg = "Unable to create archive for download, please report this error" + messagetype = "error" + return error, msg, messagetype + def _archive_composite_dataset( self, trans, data=None, **kwd ): # save a composite object into a compressed archive for downloading params = util.Params( kwd ) @@ -237,29 +257,27 @@ class Data( object ): path = data.file_name fname = os.path.split(path)[-1] efp = data.extra_files_path - htmlname = os.path.splitext(outfname)[0] - if not htmlname.endswith(ext): - htmlname = '%s_%s' % (htmlname,ext) - archname = '%s.html' % htmlname # fake the real nature of the html file - try: - archive.add(data.file_name,archname) - except IOError: - error = True - log.exception( "Unable to add composite parent %s to temporary library download archive" % data.file_name) - msg = "Unable to create archive for download, please report this error" - messagetype = 'error' - for root, dirs, files in os.walk(efp): - for fname in files: - fpath = os.path.join(root,fname) - rpath = os.path.relpath(fpath,efp) - try: - archive.add( fpath,rpath ) - except IOError: - error = True - log.exception( "Unable to add %s to temporary library download archive" % rpath) - msg = "Unable to create archive for download, please report this error" - messagetype = 'error' - continue + #Add any central file to the archive, + + display_name = os.path.splitext(outfname)[0] + if not display_name.endswith(ext): + display_name = '%s_%s' % (display_name, ext) + + error, msg, messagetype = self._archive_main_file(archive, display_name, path) + if not error: + #Add any child files to the archive, + for root, dirs, files in os.walk(efp): + for fname in files: + fpath = os.path.join(root,fname) + rpath = os.path.relpath(fpath,efp) + try: + archive.add( fpath,rpath ) + except IOError: + error = True + log.exception( "Unable to add %s to temporary library download archive" % rpath) + msg = "Unable to create archive for download, please report this error" + messagetype = 'error' + continue if not error: if params.do_action == 'zip': archive.close() @@ -288,7 +306,14 @@ class Data( object ): return open( dataset.file_name ) def display_data(self, trans, data, preview=False, filename=None, to_ext=None, size=None, offset=None, **kwd): - """ Old display method, for transition """ + """ Old display method, for transition - though still used by API and + test framework. Datatypes should be very careful if overridding this + method and this interface between datatypes and Galaxy will likely + change. + + TOOD: Document alternatives to overridding this method (data + providers?). + """ #Relocate all composite datatype display to a common location. composite_extensions = trans.app.datatypes_registry.get_composite_extensions( ) composite_extensions.append('html') # for archiving composite datatypes @@ -487,13 +512,14 @@ class Data( object ): def before_setting_metadata( self, dataset ): """This function is called on the dataset before metadata is set.""" dataset.clear_associated_files( metadata_safe = True ) - def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, space_to_tab = False, **kwds ): + def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, to_posix_lines = True, space_to_tab = False, **kwds ): kwds[ 'name' ] = name kwds[ 'optional' ] = optional kwds[ 'mimetype' ] = mimetype kwds[ 'description' ] = description kwds[ 'substitute_name_with_metadata' ] = substitute_name_with_metadata kwds[ 'is_binary' ] = is_binary + kwds[ 'to_posix_lines' ] = to_posix_lines kwds[ 'space_to_tab' ] = space_to_tab return Bunch( **kwds ) def add_composite_file( self, name, **kwds ): diff --git a/lib/galaxy/datatypes/dataproviders/decorators.py b/lib/galaxy/datatypes/dataproviders/decorators.py index 83b1e6efc27..f429f0c4065 100644 --- a/lib/galaxy/datatypes/dataproviders/decorators.py +++ b/lib/galaxy/datatypes/dataproviders/decorators.py @@ -18,6 +18,7 @@ DataProvider related decorators. from functools import wraps #from types import MethodType +import urllib2 import copy import logging @@ -143,8 +144,9 @@ def _parse_query_string_settings( query_kwargs, settings=None ): 'int' : int, 'float' : float, 'bool' : bool, - 'list:str' : lambda s: list_from_query_string( s ), - 'list:int' : lambda s: [ int( i ) for i in list_from_query_string( s ) ], + 'list:str' : lambda s: list_from_query_string( s ), + 'list:escaped' : lambda s: [ urllib2.unquote( e ) for e in list_from_query_string( s ) ], + 'list:int' : lambda s: [ int( i ) for i in list_from_query_string( s ) ], } settings = settings or {} # yay! yet another set of query string parsers! <-- sarcasm diff --git a/lib/galaxy/datatypes/dataproviders/line.py b/lib/galaxy/datatypes/dataproviders/line.py index 4d2e127b0d2..3c8deedacd5 100644 --- a/lib/galaxy/datatypes/dataproviders/line.py +++ b/lib/galaxy/datatypes/dataproviders/line.py @@ -78,7 +78,7 @@ class FilteredLineDataProvider( base.LimitedOffsetDataProvider ): line = line.strip( '\n' ) if not self.provide_blank and line == '': return None - elif line.startswith( self.comment_char ): + elif self.comment_char and line.startswith( self.comment_char ): return None return super( FilteredLineDataProvider, self ).filter( line ) @@ -94,7 +94,7 @@ class RegexLineDataProvider( FilteredLineDataProvider ): the line it is considered valid and will be provided). """ settings = { - 'regex_list' : 'list:str', + 'regex_list' : 'list:escaped', 'invert' : 'bool', } diff --git a/lib/galaxy/datatypes/images.py b/lib/galaxy/datatypes/images.py index f2fc663a9f8..f1a4b95efe5 100644 --- a/lib/galaxy/datatypes/images.py +++ b/lib/galaxy/datatypes/images.py @@ -213,7 +213,6 @@ Binary.register_sniffable_binary_format("pdf", "pdf", Pdf) def create_applet_tag_peek( class_name, archive, params ): text = """ - @@ -221,17 +220,14 @@ def create_applet_tag_peek( class_name, archive, params ): for name, value in params.iteritems(): text += """""" % ( name, value ) text += """ - """ % ( class_name, archive ) for name, value in params.iteritems(): text += """""" % ( name, value ) - text += """ - + text += """
        You must install and enable Java in your browser in order to access this applet.
        - """ return """

        %s

        """ % text diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index 3d3928f6fe6..bcbc6b7ab2a 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -389,7 +389,7 @@ class Bed( Interval ): MetadataElement( name="endCol", default=3, desc="End column", param=metadata.ColumnParameter ) MetadataElement( name="strandCol", desc="Strand column (click box & select)", param=metadata.ColumnParameter, optional=True, no_value=0 ) MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True, visible=False ) - MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[4], param=metadata.ColumnParameter, multiple=True ) + MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[4], param=metadata.ColumnParameter, optional=True, multiple=True ) ###do we need to repeat these? they are the same as should be inherited from interval type def set_meta( self, dataset, overwrite = True, **kwd ): diff --git a/lib/galaxy/datatypes/metadata.py b/lib/galaxy/datatypes/metadata.py index 2bf4de1fcd8..54efe4d2949 100644 --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -2,15 +2,13 @@ Galaxy Metadata """ -from galaxy import eggs -eggs.require("simplejson") import copy import cPickle +import json import logging import os import shutil -import simplejson import sys import tempfile import weakref @@ -130,7 +128,7 @@ class MetadataCollection( object ): def from_JSON_dict( self, filename ): dataset = self.parent log.debug( 'loading metadata from file for: %s %s' % ( dataset.__class__.__name__, dataset.id ) ) - JSONified_dict = simplejson.load( open( filename ) ) + JSONified_dict = json.load( open( filename ) ) for name, spec in self.spec.items(): if name in JSONified_dict: dataset._metadata[ name ] = spec.param.from_external_value( JSONified_dict[ name ], dataset ) @@ -146,7 +144,7 @@ class MetadataCollection( object ): for name, spec in self.spec.items(): if name in dataset_meta_dict: meta_dict[ name ] = spec.param.to_external_value( dataset_meta_dict[ name ] ) - simplejson.dump( meta_dict, open( filename, 'wb+' ) ) + json.dump( meta_dict, open( filename, 'wb+' ) ) def __getstate__( self ): return None #cannot pickle a weakref item (self._parent), when data._metadata_collection is None, it will be recreated on demand @@ -456,7 +454,7 @@ class ListParameter( MetadataParameter ): class DictParameter( MetadataParameter ): def to_string( self, value ): - return simplejson.dumps( value ) + return json.dumps( value ) class PythonObjectParameter( MetadataParameter ): @@ -594,7 +592,7 @@ class MetadataTempFile( object ): @classmethod def cleanup_from_JSON_dict_filename( cls, filename ): try: - for key, value in simplejson.load( open( filename ) ).items(): + for key, value in json.load( open( filename ) ).items(): if cls.is_JSONified_value( value ): value = cls.from_JSON( value ) if isinstance( value, cls ) and os.path.exists( value.file_name ): @@ -686,10 +684,10 @@ class JobExternalOutputMetadataWrapper( object ): #file to store a 'return code' indicating the results of the set_meta() call #results code is like (True/False - if setting metadata was successful/failed , exception or string of reason of success/failure ) metadata_files.filename_results_code = abspath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_results_%s_" % key ).name ) - simplejson.dump( ( False, 'External set_meta() not called' ), open( metadata_files.filename_results_code, 'wb+' ) ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) + json.dump( ( False, 'External set_meta() not called' ), open( metadata_files.filename_results_code, 'wb+' ) ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) #file to store kwds passed to set_meta() metadata_files.filename_kwds = abspath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_kwds_%s_" % key ).name ) - simplejson.dump( kwds, open( metadata_files.filename_kwds, 'wb+' ), ensure_ascii=True ) + json.dump( kwds, open( metadata_files.filename_kwds, 'wb+' ), ensure_ascii=True ) #existing metadata file parameters need to be overridden with cluster-writable file locations metadata_files.filename_override_metadata = abspath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_override_%s_" % key ).name ) open( metadata_files.filename_override_metadata, 'wb+' ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) @@ -699,7 +697,7 @@ class JobExternalOutputMetadataWrapper( object ): metadata_temp = MetadataTempFile() shutil.copy( dataset.metadata.get( meta_key, None ).file_name, metadata_temp.file_name ) override_metadata.append( ( meta_key, metadata_temp.to_JSON() ) ) - simplejson.dump( override_metadata, open( metadata_files.filename_override_metadata, 'wb+' ) ) + json.dump( override_metadata, open( metadata_files.filename_override_metadata, 'wb+' ) ) #add to session and flush sa_session.add( metadata_files ) sa_session.flush() @@ -711,7 +709,7 @@ class JobExternalOutputMetadataWrapper( object ): metadata_files = self.get_output_filenames_by_dataset( dataset, sa_session ) if not metadata_files: return False # this file doesn't exist - rval, rstring = simplejson.load( open( metadata_files.filename_results_code ) ) + rval, rstring = json.load( open( metadata_files.filename_results_code ) ) if not rval: log.debug( 'setting metadata externally failed for %s %s: %s' % ( dataset.__class__.__name__, dataset.id, rstring ) ) return rval diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 11fcf85d02a..6685b678e83 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -1,16 +1,38 @@ """ Provides mapping between extensions and datatypes, mime-types, etc. """ -import os, sys, tempfile, threading, logging, imp -import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo, binary, assembly, ngsindex, graph +import os +import sys +import tempfile +import threading +import logging +import imp +import data +import tabular +import interval +import images +import sequence +import qualityscore +import genetics +import xml +import coverage +import tracks +import chrominfo +import binary +import assembly +import ngsindex +import graph import galaxy.util from galaxy.util.odict import odict from display_applications.application import DisplayApplication + class ConfigurationError( Exception ): pass + class Registry( object ): + def __init__( self ): self.log = logging.getLogger(__name__) self.log.addHandler( logging.NullHandler() ) @@ -47,19 +69,29 @@ class Registry( object ): self.datatype_elems = [] self.sniffer_elems = [] self.xml_filename = None + def load_datatypes( self, root_dir=None, config=None, deactivate=False, override=True ): """ - Parse a datatypes XML file located at root_dir/config. If deactivate is True, an installed tool shed - repository that includes proprietary datatypes is being deactivated, so appropriate loaded datatypes - will be removed from the registry. The value of override will be False when a tool shed repository is - being installed. Since installation is occurring after the datatypes registry has been initialized, its - contents cannot be overridden by new introduced conflicting data types. + Parse a datatypes XML file located at root_dir/config (if processing the Galaxy distributed config) or contained within + an installed Tool Shed repository. If deactivate is True, an installed Tool Shed repository that includes custom datatypes + is being deactivated or uninstalled, so appropriate loaded datatypes will be removed from the registry. The value of + override will be False when a Tool Shed repository is being installed. Since installation is occurring after the datatypes + registry has been initialized at server startup, it's contents cannot be overridden by newly introduced conflicting data types. """ + def __import_module( full_path, datatype_module, datatype_class_name ): open_file_obj, file_name, description = imp.find_module( datatype_module, [ full_path ] ) imported_module = imp.load_module( datatype_class_name, open_file_obj, file_name, description ) return imported_module + if root_dir and config: + # If handling_proprietary_datatypes is determined as True below, we'll have an elem that looks something like this: + # handling_proprietary_datatypes = False # Parse datatypes_conf.xml tree = galaxy.util.parse_xml( config ) @@ -82,308 +114,281 @@ class Registry( object ): # Proprietary datatype's tag may have special attributes, proprietary_converter_path and proprietary_display_path. proprietary_converter_path = registration.get( 'proprietary_converter_path', None ) proprietary_display_path = registration.get( 'proprietary_display_path', None ) - if proprietary_converter_path or proprietary_display_path and not handling_proprietary_datatypes: + if proprietary_converter_path is not None or proprietary_display_path is not None and not handling_proprietary_datatypes: handling_proprietary_datatypes = True for elem in registration.findall( 'datatype' ): - try: - extension = elem.get( 'extension', None ) - dtype = elem.get( 'type', None ) - type_extension = elem.get( 'type_extension', None ) - mimetype = elem.get( 'mimetype', None ) - display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) ) - make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) ) - # Proprietary datatypes included in installed tool shed repositories will include two special attributes - # (proprietary_path and proprietary_datatype_module) if they depend on proprietary datatypes classes. - proprietary_path = elem.get( 'proprietary_path', None ) - proprietary_datatype_module = elem.get( 'proprietary_datatype_module', None ) - if proprietary_path or proprietary_datatype_module and not handling_proprietary_datatypes: - handling_proprietary_datatypes = True - if deactivate: - # We are deactivating an installed tool shed repository, so eliminate the - # datatype elem from the in-memory list of datatype elems. - for in_memory_elem in self.datatype_elems: - in_memory_extension = in_memory_elem.get( 'extension', None ) - if in_memory_extension == extension: - in_memory_dtype = elem.get( 'type', None ) - in_memory_type_extension = elem.get( 'type_extension', None ) - in_memory_mimetype = elem.get( 'mimetype', None ) - in_memory_display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) ) - in_memory_make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) ) - if in_memory_dtype == dtype and in_memory_type_extension == type_extension and in_memory_mimetype == mimetype \ - and in_memory_display_in_upload == display_in_upload and in_memory_make_subclass == make_subclass: - self.datatype_elems.remove( in_memory_elem ) - else: - # Keep an in-memory list of datatype elems to enable persistence. - if extension not in self.datatypes_by_extension: - self.datatype_elems.append( elem ) - if extension and extension in self.datatypes_by_extension and deactivate: - # We are deactivating an installed tool shed repository, so eliminate the datatype from the registry. - # TODO: Handle deactivating datatype converters, etc before removing from self.datatypes_by_extension. - self.log.debug( "Removing datatype with extension '%s' from the registry." % extension ) + # Keep a status of the process steps to enable stopping the process of handling the datatype if necessary. + ok = True + extension = elem.get( 'extension', None ) + dtype = elem.get( 'type', None ) + type_extension = elem.get( 'type_extension', None ) + mimetype = elem.get( 'mimetype', None ) + display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) ) + # If make_subclass is True, it does not necessarily imply that we are subclassing a datatype that is contained + # in the distribution. + make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) ) + # Proprietary datatypes included in installed tool shed repositories will include two special attributes + # (proprietary_path and proprietary_datatype_module) if they depend on proprietary datatypes classes. + # The value of proprietary_path is the path to the cloned location of the tool shed repository's contained + # datatypes_conf.xml file. + proprietary_path = elem.get( 'proprietary_path', None ) + proprietary_datatype_module = elem.get( 'proprietary_datatype_module', None ) + if proprietary_path is not None or proprietary_datatype_module is not None and not handling_proprietary_datatypes: + handling_proprietary_datatypes = True + if deactivate: + # We are deactivating or uninstalling an installed tool shed repository, so eliminate the datatype + # elem from the in-memory list of datatype elems. + for in_memory_elem in self.datatype_elems: + in_memory_extension = in_memory_elem.get( 'extension', None ) + if in_memory_extension == extension: + in_memory_dtype = elem.get( 'type', None ) + in_memory_type_extension = elem.get( 'type_extension', None ) + in_memory_mimetype = elem.get( 'mimetype', None ) + in_memory_display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) ) + in_memory_make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) ) + if in_memory_dtype == dtype and \ + in_memory_type_extension == type_extension and \ + in_memory_mimetype == mimetype and \ + in_memory_display_in_upload == display_in_upload and \ + in_memory_make_subclass == make_subclass: + self.datatype_elems.remove( in_memory_elem ) + if extension is not None and extension in self.datatypes_by_extension: + # We are deactivating or uninstalling an installed tool shed repository, so eliminate the datatype + # from the registry. TODO: Handle deactivating datatype converters, etc before removing from + # self.datatypes_by_extension. del self.datatypes_by_extension[ extension ] if extension in self.upload_file_formats: self.upload_file_formats.remove( extension ) - can_process_datatype = False - else: - can_process_datatype = ( extension and ( dtype or type_extension ) ) and ( extension not in self.datatypes_by_extension or override ) + self.log.debug( "Removed datatype with extension '%s' from the registry." % extension ) + else: + # We are loading new datatype, so we'll make sure it is correctly defined before proceeding. + can_process_datatype = False + if extension is not None: + if dtype is not None or type_extension is not None: + if override or extension not in self.datatypes_by_extension: + can_process_datatype = True if can_process_datatype: - if dtype: - fields = dtype.split( ':' ) - datatype_module = fields[0] - datatype_class_name = fields[1] - datatype_class = None - if proprietary_path and proprietary_datatype_module and datatype_class_name: - # We need to change the value of sys.path, so do it in a way that is thread-safe. - lock = threading.Lock() - lock.acquire( True ) - try: - imported_module = __import_module( proprietary_path, proprietary_datatype_module, datatype_class_name ) - if imported_module not in self.imported_modules: - self.imported_modules.append( imported_module ) - if hasattr( imported_module, datatype_class_name ): - datatype_class = getattr( imported_module, datatype_class_name ) - except Exception, e: - full_path = os.path.join( proprietary_path, proprietary_datatype_module ) - self.log.debug( "Exception importing proprietary code file %s: %s" % ( str( full_path ), str( e ) ) ) - finally: - lock.release() - if datatype_class is None: - # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution. - fields = datatype_module.split( '.' ) - module = __import__( fields.pop(0) ) - for mod in fields: - module = getattr( module, mod ) - datatype_class = getattr( module, datatype_class_name ) - elif type_extension: - datatype_class = self.datatypes_by_extension[type_extension].__class__ - if make_subclass: - datatype_class = type( datatype_class_name, (datatype_class,), {} ) - if extension in self.datatypes_by_extension: - self.log.warning( "Overriding conflicting datatype with extension '%s', using datatype from %s." % ( extension, config ) ) - self.datatypes_by_extension[ extension ] = datatype_class() - if mimetype is None: - # Use default mime type as per datatype spec - mimetype = self.datatypes_by_extension[ extension ].get_mime() - self.mimetypes_by_extension[ extension ] = mimetype - if datatype_class.track_type: - self.available_tracks.append( extension ) - if display_in_upload and extension not in self.upload_file_formats: - self.upload_file_formats.append( extension ) - # Max file size cut off for setting optional metadata - self.datatypes_by_extension[ extension ].max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None ) - for converter in elem.findall( 'converter' ): - # Build the list of datatype converters which will later be loaded into the calling app's toolbox. - converter_config = converter.get( 'file', None ) - target_datatype = converter.get( 'target_datatype', None ) - depends_on = converter.get( 'depends_on', None ) - if depends_on and target_datatype: - if extension not in self.converter_deps: - self.converter_deps[extension] = {} - self.converter_deps[extension][target_datatype] = depends_on.split(',') - if converter_config and target_datatype: - #if imported_modules: - if proprietary_converter_path: - self.proprietary_converters.append( ( converter_config, extension, target_datatype ) ) - else: - self.converters.append( ( converter_config, extension, target_datatype ) ) - for composite_file in elem.findall( 'composite_file' ): - # add composite files - name = composite_file.get( 'name', None ) - if name is None: - self.log.warning( "You must provide a name for your composite_file (%s)." % composite_file ) - optional = composite_file.get( 'optional', False ) - mimetype = composite_file.get( 'mimetype', None ) - self.datatypes_by_extension[extension].add_composite_file( name, optional=optional, mimetype=mimetype ) - for display_app in elem.findall( 'display' ): - #if imported_modules: - if proprietary_display_path: - if elem not in self.proprietary_display_app_containers: - self.proprietary_display_app_containers.append( elem ) - else: - if elem not in self.display_app_containers: - self.display_app_containers.append( elem ) - elif not deactivate: - # A new tool shed repository that contains proprietary datatypes is being installed, and since installation - # is occurring after the datatypes registry has been initialized, its contents cannot be overridden by new - # introduced conflicting data types. - self.log.warning( "Ignoring conflicting datatype with extension '%s' from %s." % ( extension, config ) ) - except Exception, e: - if deactivate: - self.log.warning( "Error deactivating datatype with extension '%s': %s" % ( extension, str( e ) ) ) + if dtype is not None: + try: + fields = dtype.split( ':' ) + datatype_module = fields[ 0 ] + datatype_class_name = fields[ 1 ] + except Exception, e: + self.log.exception( 'Error parsing datatype definition for dtype %s: %s' % ( str( dtype ), str( e ) ) ) + ok = False + if ok: + datatype_class = None + if proprietary_path and proprietary_datatype_module and datatype_class_name: + # We need to change the value of sys.path, so do it in a way that is thread-safe. + lock = threading.Lock() + lock.acquire( True ) + try: + imported_module = __import_module( proprietary_path, + proprietary_datatype_module, + datatype_class_name ) + if imported_module not in self.imported_modules: + self.imported_modules.append( imported_module ) + if hasattr( imported_module, datatype_class_name ): + datatype_class = getattr( imported_module, datatype_class_name ) + except Exception, e: + full_path = os.path.join( proprietary_path, proprietary_datatype_module ) + self.log.debug( "Exception importing proprietary code file %s: %s" % ( str( full_path ), str( e ) ) ) + finally: + lock.release() + # Either the above exception was thrown because the proprietary_datatype_module is not derived from a class + # in the repository, or we are loading Galaxy's datatypes. In either case we'll look in the registry. + if datatype_class is None: + try: + # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution. + fields = datatype_module.split( '.' ) + module = __import__( fields.pop( 0 ) ) + for mod in fields: + module = getattr( module, mod ) + datatype_class = getattr( module, datatype_class_name ) + self.log.debug( 'Retrieved datatype module %s from the datatype registry.' % str( datatype_module ) ) + except Exception, e: + self.log.exception( 'Error importing datatype module %s: %s' % ( str( datatype_module ), str( e ) ) ) + ok = False + elif type_extension is not None: + try: + datatype_class = self.datatypes_by_extension[ type_extension ].__class__ + except Exception, e: + self.log.exception( 'Error determining datatype_class for type_extension %s: %s' % ( str( type_extension ), str( e ) ) ) + ok = False + if ok: + if not deactivate: + # A new tool shed repository that contains custom datatypes is being installed, and since installation is + # occurring after the datatypes registry has been initialized at server startup, its contents cannot be + # overridden by new introduced conflicting data types unless the value of override is True. + if extension in self.datatypes_by_extension: + # Because of the way that the value of can_process_datatype was set above, we know that the value of + # override is True. + self.log.warning( "Overriding conflicting datatype with extension '%s', using datatype from %s." % \ + ( str( extension ), str( config ) ) ) + if make_subclass: + datatype_class = type( datatype_class_name, ( datatype_class, ), {} ) + self.datatypes_by_extension[ extension ] = datatype_class() + if mimetype is None: + # Use default mimetype per datatype specification. + mimetype = self.datatypes_by_extension[ extension ].get_mime() + self.mimetypes_by_extension[ extension ] = mimetype + if datatype_class.track_type: + self.available_tracks.append( extension ) + if display_in_upload and extension not in self.upload_file_formats: + self.upload_file_formats.append( extension ) + # Max file size cut off for setting optional metadata. + self.datatypes_by_extension[ extension ].max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None ) + for converter in elem.findall( 'converter' ): + # Build the list of datatype converters which will later be loaded into the calling app's toolbox. + converter_config = converter.get( 'file', None ) + target_datatype = converter.get( 'target_datatype', None ) + depends_on = converter.get( 'depends_on', None ) + if depends_on is not None and target_datatype is not None: + if extension not in self.converter_deps: + self.converter_deps[ extension ] = {} + self.converter_deps[ extension ][ target_datatype ] = depends_on.split( ',' ) + if converter_config and target_datatype: + if proprietary_converter_path: + self.proprietary_converters.append( ( converter_config, extension, target_datatype ) ) + else: + self.converters.append( ( converter_config, extension, target_datatype ) ) + # Add composite files. + for composite_file in elem.findall( 'composite_file' ): + name = composite_file.get( 'name', None ) + if name is None: + self.log.warning( "You must provide a name for your composite_file (%s)." % composite_file ) + optional = composite_file.get( 'optional', False ) + mimetype = composite_file.get( 'mimetype', None ) + self.datatypes_by_extension[ extension ].add_composite_file( name, optional=optional, mimetype=mimetype ) + for display_app in elem.findall( 'display' ): + if proprietary_display_path: + if elem not in self.proprietary_display_app_containers: + self.proprietary_display_app_containers.append( elem ) + else: + if elem not in self.display_app_containers: + self.display_app_containers.append( elem ) + # Processing the new datatype elem is now complete, so make sure the element defining it is retained by appending + # the new datatype to the in-memory list of datatype elems to enable persistence. + self.datatype_elems.append( elem ) else: - self.log.warning( "Error loading datatype with extension '%s': %s" % ( extension, str( e ) ) ) - # Load datatype sniffers from the config - sniffers = root.find( 'sniffers' ) - if sniffers: - for elem in sniffers.findall( 'sniffer' ): - # Keep an in-memory list of sniffer elems to enable persistence. - if elem not in self.sniffer_elems: - self.sniffer_elems.append( elem ) - dtype = elem.get( 'type', None ) - if dtype: - try: - fields = dtype.split( ":" ) - datatype_module = fields[0] - datatype_class_name = fields[1] - module = None - #if imported_modules: - if handling_proprietary_datatypes: - # See if one of the imported modules contains the datatype class name. - for imported_module in self.imported_modules: - if hasattr( imported_module, datatype_class_name ): - module = imported_module - break - if module is None: + if extension is not None: + if dtype is not None or type_extension is not None: + if extension in self.datatypes_by_extension: + if not override: + # Do not load the datatype since it conflicts with an existing datatype which we are not supposed + # to override. + self.log.warning( "Ignoring conflicting datatype with extension '%s' from %s." % ( extension, config ) ) + # Load datatype sniffers from the config - we'll do this even if one or more datatypes were not properly processed in the config + # since sniffers are not tightly coupled with datatypes. + self.load_datatype_sniffers( root, + deactivate=deactivate, + handling_proprietary_datatypes=handling_proprietary_datatypes, + override=override ) + self.upload_file_formats.sort() + # Persist the xml form of the registry into a temporary file so that it can be loaded from the command line by tools and + # set_metadata processing. + self.to_xml_file() + self.set_default_values() + + def append_to_sniff_order(): + # Just in case any supported data types are not included in the config's sniff_order section. + for ext in self.datatypes_by_extension: + datatype = self.datatypes_by_extension[ ext ] + included = False + for atype in self.sniff_order: + if isinstance( atype, datatype.__class__ ): + included = True + break + if not included: + self.sniff_order.append( datatype ) + append_to_sniff_order() + + def load_datatype_sniffers( self, root, deactivate=False, handling_proprietary_datatypes=False, override=False ): + """ + Process the sniffers element from a parsed a datatypes XML file located at root_dir/config (if processing the Galaxy + distributed config) or contained within an installed Tool Shed repository. If deactivate is True, an installed Tool + Shed repository that includes custom sniffers is being deactivated or uninstalled, so appropriate loaded sniffers will + be removed from the registry. The value of override will be False when a Tool Shed repository is being installed. + Since installation is occurring after the datatypes registry has been initialized at server startup, it's contents + cannot be overridden by newly introduced conflicting sniffers. + """ + sniffer_elem_classes = [ e.attrib[ 'type' ] for e in self.sniffer_elems ] + sniffers = root.find( 'sniffers' ) + if sniffers: + for elem in sniffers.findall( 'sniffer' ): + # Keep a status of the process steps to enable stopping the process of handling the sniffer if necessary. + ok = True + dtype = elem.get( 'type', None ) + if dtype is not None: + try: + fields = dtype.split( ":" ) + datatype_module = fields[ 0 ] + datatype_class_name = fields[ 1 ] + module = None + except Exception, e: + self.log.exception( 'Error determining datatype class or module for dtype %s: %s' % ( str( dtype ), str( e ) ) ) + ok = False + if ok: + if handling_proprietary_datatypes: + # See if one of the imported modules contains the datatype class name. + for imported_module in self.imported_modules: + if hasattr( imported_module, datatype_class_name ): + module = imported_module + break + if module is None: + try: # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution. module = __import__( datatype_module ) for comp in datatype_module.split( '.' )[ 1: ]: module = getattr( module, comp ) - aclass = getattr( module, datatype_class_name )() - if deactivate: - if elem in self.sniffer_elems: - self.sniffer_elems.remove( elem ) - for sniffer_class in self.sniff_order: - if sniffer_class.__class__ == aclass.__class__: - self.sniff_order.remove( sniffer_class ) - break - self.log.debug( "Deactivated sniffer for datatype '%s'" % dtype ) - else: - # See if we have a conflicting sniffer already loaded. - conflict = False - for conflict_loc, sniffer_class in enumerate( self.sniff_order ): - if sniffer_class.__class__ == aclass.__class__: - # We have a conflicting sniffer, so replace the one previously loaded. - conflict = True + except Exception, e: + self.log.exception( "Error importing datatype class for '%s': %s" % ( str( dtype ), str( e ) ) ) + ok = False + if ok: + try: + aclass = getattr( module, datatype_class_name )() + except Exception, e: + self.log.exception( 'Error calling method %s from class %s: %s', str( datatype_class_name ), str( module ), str( e ) ) + ok = False + if ok: + if deactivate: + # We are deactivating or uninstalling an installed Tool Shed repository, so eliminate the appropriate sniffers. + sniffer_class = elem.get( 'type', None ) + if sniffer_class is not None: + for index, s_e_c in enumerate( sniffer_elem_classes ): + if sniffer_class == s_e_c: + del self.sniffer_elems[ index ] + sniffer_elem_classes = [ e.attrib[ 'type' ] for e in self.sniffer_elems ] + self.log.debug( "Removed sniffer element for datatype '%s'" % str( dtype ) ) + break + for sniffer_class in self.sniff_order: + if sniffer_class.__class__ == aclass.__class__: + self.sniff_order.remove( sniffer_class ) + self.log.debug( "Removed sniffer class for datatype '%s' from sniff order" % str( dtype ) ) + break + else: + # We are loading new sniffer, so see if we have a conflicting sniffer already loaded. + conflict = False + for conflict_loc, sniffer_class in enumerate( self.sniff_order ): + if sniffer_class.__class__ == aclass.__class__: + # We have a conflicting sniffer, so replace the one previously loaded. + conflict = True + if override: + del self.sniff_order[ conflict_loc ] + self.log.debug( "Removed conflicting sniffer for datatype '%s'" % dtype ) + break + if conflict: if override: - del self.sniff_order[ conflict_loc ] - self.log.debug( "Replaced conflicting sniffer for datatype '%s'" % dtype ) - break - if conflict: - if override: + self.sniff_order.append( aclass ) + self.log.debug( "Loaded sniffer for datatype '%s'" % dtype ) + else: self.sniff_order.append( aclass ) self.log.debug( "Loaded sniffer for datatype '%s'" % dtype ) - else: - self.sniff_order.append( aclass ) - self.log.debug( "Loaded sniffer for datatype '%s'" % dtype ) - except Exception, exc: - if deactivate: - self.log.warning( "Error deactivating sniffer for datatype '%s': %s" % ( dtype, str( exc ) ) ) - else: - self.log.warning( "Error appending sniffer for datatype '%s' to sniff_order: %s" % ( dtype, str( exc ) ) ) - self.upload_file_formats.sort() - # Persist the xml form of the registry into a temporary file so that it - # can be loaded from the command line by tools and set_metadata processing. - self.to_xml_file() - # Default values. - if not self.datatypes_by_extension: - self.datatypes_by_extension = { - 'ab1' : binary.Ab1(), - 'axt' : sequence.Axt(), - 'bam' : binary.Bam(), - 'bed' : interval.Bed(), - 'coverage' : coverage.LastzCoverage(), - 'customtrack' : interval.CustomTrack(), - 'csfasta' : sequence.csFasta(), - 'fasta' : sequence.Fasta(), - 'eland' : tabular.Eland(), - 'fastq' : sequence.Fastq(), - 'fastqsanger' : sequence.FastqSanger(), - 'gtf' : interval.Gtf(), - 'gff' : interval.Gff(), - 'gff3' : interval.Gff3(), - 'genetrack' : tracks.GeneTrack(), - 'interval' : interval.Interval(), - 'laj' : images.Laj(), - 'lav' : sequence.Lav(), - 'maf' : sequence.Maf(), - 'pileup' : tabular.Pileup(), - 'qualsolid' : qualityscore.QualityScoreSOLiD(), - 'qualsolexa' : qualityscore.QualityScoreSolexa(), - 'qual454' : qualityscore.QualityScore454(), - 'sam' : tabular.Sam(), - 'scf' : binary.Scf(), - 'sff' : binary.Sff(), - 'tabular' : tabular.Tabular(), - 'taxonomy' : tabular.Taxonomy(), - 'txt' : data.Text(), - 'wig' : interval.Wiggle(), - 'xml' : xml.GenericXml(), - } - self.mimetypes_by_extension = { - 'ab1' : 'application/octet-stream', - 'axt' : 'text/plain', - 'bam' : 'application/octet-stream', - 'bed' : 'text/plain', - 'customtrack' : 'text/plain', - 'csfasta' : 'text/plain', - 'eland' : 'application/octet-stream', - 'fasta' : 'text/plain', - 'fastq' : 'text/plain', - 'fastqsanger' : 'text/plain', - 'gtf' : 'text/plain', - 'gff' : 'text/plain', - 'gff3' : 'text/plain', - 'interval' : 'text/plain', - 'laj' : 'text/plain', - 'lav' : 'text/plain', - 'maf' : 'text/plain', - 'memexml' : 'application/xml', - 'pileup' : 'text/plain', - 'qualsolid' : 'text/plain', - 'qualsolexa' : 'text/plain', - 'qual454' : 'text/plain', - 'sam' : 'text/plain', - 'scf' : 'application/octet-stream', - 'sff' : 'application/octet-stream', - 'tabular' : 'text/plain', - 'taxonomy' : 'text/plain', - 'txt' : 'text/plain', - 'wig' : 'text/plain', - 'xml' : 'application/xml', - } - # super supertype fix for input steps in workflows. - if 'data' not in self.datatypes_by_extension: - self.datatypes_by_extension['data'] = data.Data() - self.mimetypes_by_extension['data'] = 'application/octet-stream' - # Default values - the order in which we attempt to determine data types is critical - # because some formats are much more flexibly defined than others. - if len(self.sniff_order) < 1: - self.sniff_order = [ - binary.Bam(), - binary.Sff(), - xml.GenericXml(), - sequence.Maf(), - sequence.Lav(), - sequence.csFasta(), - qualityscore.QualityScoreSOLiD(), - qualityscore.QualityScore454(), - sequence.Fasta(), - sequence.Fastq(), - interval.Wiggle(), - images.Html(), - sequence.Axt(), - interval.Bed(), - interval.CustomTrack(), - interval.Gtf(), - interval.Gff(), - interval.Gff3(), - tabular.Pileup(), - interval.Interval(), - tabular.Sam(), - tabular.Eland() - ] - def append_to_sniff_order(): - # Just in case any supported data types are not included in the config's sniff_order section. - for ext in self.datatypes_by_extension: - datatype = self.datatypes_by_extension[ext] - included = False - for atype in self.sniff_order: - if isinstance(atype, datatype.__class__): - included = True - break - if not included: - self.sniff_order.append(datatype) - append_to_sniff_order() + # Processing the new sniffer elem is now complete, so make sure the element defining it is loaded if necessary. + sniffer_class = elem.get( 'type', None ) + if sniffer_class is not None: + if sniffer_class not in sniffer_elem_classes: + self.sniffer_elems.append( elem ) def get_datatype_class_by_name( self, name ): """ @@ -414,25 +419,28 @@ class Registry( object ): # #return datatype - def get_available_tracks(self): + def get_available_tracks( self ): return self.available_tracks - def get_mimetype_by_extension(self, ext, default = 'application/octet-stream' ): + + def get_mimetype_by_extension( self, ext, default='application/octet-stream' ): """Returns a mimetype based on an extension""" try: - mimetype = self.mimetypes_by_extension[ext] + mimetype = self.mimetypes_by_extension[ ext ] except KeyError: #datatype was never declared mimetype = default - self.log.warning('unknown mimetype in data factory %s' % ext) + self.log.warning( 'unknown mimetype in data factory %s' % str( ext ) ) return mimetype - def get_datatype_by_extension(self, ext ): + + def get_datatype_by_extension( self, ext ): """Returns a datatype based on an extension""" try: - builder = self.datatypes_by_extension[ext] + builder = self.datatypes_by_extension[ ext ] except KeyError: builder = data.Text() return builder - def change_datatype(self, data, ext): + + def change_datatype( self, data, ext ): data.extension = ext # call init_meta and copy metadata from itself. The datatype # being converted *to* will handle any metadata copying and @@ -441,13 +449,15 @@ class Registry( object ): data.set_size() data.init_meta( copy_from=data ) return data - def old_change_datatype(self, data, ext): + + def old_change_datatype( self, data, ext ): """Creates and returns a new datatype based on an existing data and an extension""" - newdata = factory(ext)(id=data.id) + newdata = factory( ext )( id=data.id ) for key, value in data.__dict__.items(): - setattr(newdata, key, value) + setattr( newdata, key, value ) newdata.ext = ext return newdata + def load_datatype_converters( self, toolbox, installed_repository_dict=None, deactivate=False ): """ If deactivate is False, add datatype converters from self.converters or self.proprietary_converters @@ -461,9 +471,9 @@ class Registry( object ): # Load converters defined by local datatypes_conf.xml. converters = self.converters for elem in converters: - tool_config = elem[0] - source_datatype = elem[1] - target_datatype = elem[2] + tool_config = elem[ 0 ] + source_datatype = elem[ 1 ] + target_datatype = elem[ 2 ] if installed_repository_dict: converter_path = installed_repository_dict[ 'converter_path' ] else: @@ -490,7 +500,8 @@ class Registry( object ): if converter.id in toolbox.tools_by_id: del toolbox.tools_by_id[ converter.id ] if source_datatype in self.datatype_converters: - del self.datatype_converters[ source_datatype ][ target_datatype ] + if target_datatype in self.datatype_converters[ source_datatype ]: + del self.datatype_converters[ source_datatype ][ target_datatype ] self.log.debug( "Deactivated converter: %s", converter.id ) else: toolbox.tools_by_id[ converter.id ] = converter @@ -503,6 +514,7 @@ class Registry( object ): self.log.exception( "Error deactivating converter from (%s): %s" % ( converter_path, str( e ) ) ) else: self.log.exception( "Error loading converter (%s): %s" % ( converter_path, str( e ) ) ) + def load_display_applications( self, installed_repository_dict=None, deactivate=False ): """ If deactivate is False, add display applications from self.display_app_containers or @@ -577,6 +589,7 @@ class Registry( object ): if current_app is None and isinstance( d_type1, type( d_type2 ) ): self.log.debug( "Adding inherited display application '%s' to datatype '%s'" % ( display_app.id, extension ) ) d_type1.add_display_application( display_app ) + def load_external_metadata_tool( self, toolbox ): """Adds a tool which is used to set external metadata""" # We need to be able to add a job to the queue to set metadata. The queue will currently only accept jobs with an associated @@ -585,6 +598,9 @@ class Registry( object ): tool_xml_text = """ + + samtools + $__SET_EXTERNAL_METADATA_COMMAND_LINE__ @@ -601,25 +617,128 @@ class Registry( object ): toolbox.tools_by_id[ set_meta_tool.id ] = set_meta_tool self.set_external_metadata_tool = set_meta_tool self.log.debug( "Loaded external metadata tool: %s", self.set_external_metadata_tool.id ) - def get_converters_by_datatype(self, ext): + + def set_default_values( self ): + # Default values. + if not self.datatypes_by_extension: + self.datatypes_by_extension = { + 'ab1' : binary.Ab1(), + 'axt' : sequence.Axt(), + 'bam' : binary.Bam(), + 'bed' : interval.Bed(), + 'coverage' : coverage.LastzCoverage(), + 'customtrack' : interval.CustomTrack(), + 'csfasta' : sequence.csFasta(), + 'fasta' : sequence.Fasta(), + 'eland' : tabular.Eland(), + 'fastq' : sequence.Fastq(), + 'fastqsanger' : sequence.FastqSanger(), + 'gtf' : interval.Gtf(), + 'gff' : interval.Gff(), + 'gff3' : interval.Gff3(), + 'genetrack' : tracks.GeneTrack(), + 'interval' : interval.Interval(), + 'laj' : images.Laj(), + 'lav' : sequence.Lav(), + 'maf' : sequence.Maf(), + 'pileup' : tabular.Pileup(), + 'qualsolid' : qualityscore.QualityScoreSOLiD(), + 'qualsolexa' : qualityscore.QualityScoreSolexa(), + 'qual454' : qualityscore.QualityScore454(), + 'sam' : tabular.Sam(), + 'scf' : binary.Scf(), + 'sff' : binary.Sff(), + 'tabular' : tabular.Tabular(), + 'taxonomy' : tabular.Taxonomy(), + 'txt' : data.Text(), + 'wig' : interval.Wiggle(), + 'xml' : xml.GenericXml(), + } + self.mimetypes_by_extension = { + 'ab1' : 'application/octet-stream', + 'axt' : 'text/plain', + 'bam' : 'application/octet-stream', + 'bed' : 'text/plain', + 'customtrack' : 'text/plain', + 'csfasta' : 'text/plain', + 'eland' : 'application/octet-stream', + 'fasta' : 'text/plain', + 'fastq' : 'text/plain', + 'fastqsanger' : 'text/plain', + 'gtf' : 'text/plain', + 'gff' : 'text/plain', + 'gff3' : 'text/plain', + 'interval' : 'text/plain', + 'laj' : 'text/plain', + 'lav' : 'text/plain', + 'maf' : 'text/plain', + 'memexml' : 'application/xml', + 'pileup' : 'text/plain', + 'qualsolid' : 'text/plain', + 'qualsolexa' : 'text/plain', + 'qual454' : 'text/plain', + 'sam' : 'text/plain', + 'scf' : 'application/octet-stream', + 'sff' : 'application/octet-stream', + 'tabular' : 'text/plain', + 'taxonomy' : 'text/plain', + 'txt' : 'text/plain', + 'wig' : 'text/plain', + 'xml' : 'application/xml', + } + # super supertype fix for input steps in workflows. + if 'data' not in self.datatypes_by_extension: + self.datatypes_by_extension[ 'data' ] = data.Data() + self.mimetypes_by_extension[ 'data' ] = 'application/octet-stream' + # Default values - the order in which we attempt to determine data types is critical + # because some formats are much more flexibly defined than others. + if len( self.sniff_order ) < 1: + self.sniff_order = [ + binary.Bam(), + binary.Sff(), + xml.GenericXml(), + sequence.Maf(), + sequence.Lav(), + sequence.csFasta(), + qualityscore.QualityScoreSOLiD(), + qualityscore.QualityScore454(), + sequence.Fasta(), + sequence.Fastq(), + interval.Wiggle(), + images.Html(), + sequence.Axt(), + interval.Bed(), + interval.CustomTrack(), + interval.Gtf(), + interval.Gff(), + interval.Gff3(), + tabular.Pileup(), + interval.Interval(), + tabular.Sam(), + tabular.Eland() + ] + + def get_converters_by_datatype( self, ext ): """Returns available converters by source type""" converters = odict() - source_datatype = type(self.get_datatype_by_extension(ext)) + source_datatype = type( self.get_datatype_by_extension( ext ) ) for ext2, dict in self.datatype_converters.items(): - converter_datatype = type(self.get_datatype_by_extension(ext2)) - if issubclass(source_datatype, converter_datatype): - converters.update(dict) + converter_datatype = type( self.get_datatype_by_extension( ext2 ) ) + if issubclass( source_datatype, converter_datatype ): + converters.update( dict ) #Ensure ext-level converters are present if ext in self.datatype_converters.keys(): - converters.update(self.datatype_converters[ext]) + converters.update( self.datatype_converters[ ext ] ) return converters - def get_converter_by_target_type(self, source_ext, target_ext): + + def get_converter_by_target_type( self, source_ext, target_ext ): """Returns a converter based on source and target datatypes""" - converters = self.get_converters_by_datatype(source_ext) + converters = self.get_converters_by_datatype( source_ext ) if target_ext in converters.keys(): - return converters[target_ext] + return converters[ target_ext ] return None - def find_conversion_destination_for_dataset_by_extensions( self, dataset, accepted_formats, converter_safe = True ): + + def find_conversion_destination_for_dataset_by_extensions( self, dataset, accepted_formats, converter_safe=True ): """Returns ( target_ext, existing converted dataset )""" for convert_ext in self.get_converters_by_datatype( dataset.ext ): if self.get_datatype_by_extension( convert_ext ).matches_any( accepted_formats ): @@ -632,8 +751,10 @@ class Registry( object ): ret_data = None return ( convert_ext, ret_data ) return ( None, None ) + def get_composite_extensions( self ): return [ ext for ( ext, d_type ) in self.datatypes_by_extension.iteritems() if d_type.composite_type is not None ] + def get_upload_metadata_params( self, context, group, tool ): """Returns dict of case value:inputs for metadata conditional for upload tool""" rval = {} @@ -649,16 +770,17 @@ class Registry( object ): if 'auto' not in rval and 'txt' in rval: #need to manually add 'auto' datatype rval[ 'auto' ] = rval[ 'txt' ] return rval + @property def integrated_datatypes_configs( self ): if self.xml_filename and os.path.isfile( self.xml_filename ): return self.xml_filename self.to_xml_file() return self.xml_filename + def to_xml_file( self ): if self.xml_filename is not None: - # If persisted previously, attempt to remove - # the temporary file in which we were written. + # If persisted previously, attempt to remove the temporary file in which we were written. try: os.unlink( self.xml_filename ) except: diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index f63593a7915..87e3cf0b82a 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -2,12 +2,14 @@ Sequence classes """ -import data +from . import data import gzip +import json import logging import os import re import string + from cgi import escape from galaxy import eggs, util @@ -16,8 +18,6 @@ from galaxy.datatypes.checkers import is_gzip from galaxy.datatypes.sniff import get_test_fname, get_headers from galaxy.datatypes.metadata import MetadataElement -eggs.require("simplejson") -import simplejson try: eggs.require( "bx-python" ) @@ -44,8 +44,8 @@ class SequenceSplitLocations( data.Text ): def set_peek( self, dataset, is_multi_byte=False ): if not dataset.dataset.purged: try: - parsed_data = simplejson.load(open(dataset.file_name)) - # dataset.peek = simplejson.dumps(data, sort_keys=True, indent=4) + parsed_data = json.load(open(dataset.file_name)) + # dataset.peek = json.dumps(data, sort_keys=True, indent=4) dataset.peek = data.get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte ) dataset.blurb = '%d sections' % len(parsed_data['sections']) except Exception, e: @@ -60,7 +60,7 @@ class SequenceSplitLocations( data.Text ): def sniff( self, filename ): if os.path.getsize(filename) < 50000: try: - data = simplejson.load(open(filename)) + data = json.load(open(filename)) sections = data['sections'] for section in sections: if 'start' not in section or 'end' not in section or 'sequences' not in section: @@ -155,7 +155,7 @@ class Sequence( data.Text ): do_slow_split = classmethod(do_slow_split) def do_fast_split( cls, input_datasets, toc_file_datasets, subdir_generator_function, split_params): - data = simplejson.load(open(toc_file_datasets[0].file_name)) + data = json.load(open(toc_file_datasets[0].file_name)) sections = data['sections'] total_sequences = long(0) for section in sections: @@ -191,7 +191,7 @@ class Sequence( data.Text ): toc = toc_file_datasets[ds_no] split_data['args']['toc_file'] = toc.file_name f = open(os.path.join(dir, 'split_info_%s.json' % base_name), 'w') - simplejson.dump(split_data, f) + json.dump(split_data, f) f.close() start_sequence += sequences_per_file[part_no] return directories @@ -557,7 +557,7 @@ class Fastq ( Sequence ): sequence_count = long(args['num_sequences']) if 'toc_file' in args: - toc_file = simplejson.load(open(args['toc_file'], 'r')) + toc_file = json.load(open(args['toc_file'], 'r')) commands = Sequence.get_split_commands_with_toc(input_name, output_name, toc_file, start_sequence, sequence_count) else: commands = Sequence.get_split_commands_sequential(is_gzip(input_name), input_name, output_name, start_sequence, sequence_count) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 110937f1c73..9f2a1aa3d04 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -275,7 +275,7 @@ class Tabular( data.Text ): cursor = f.read(1) return to_json_string( { 'ck_data': util.unicodify( ck_data ), 'ck_index': ck_index + 1 } ) - def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None): + def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None, **kwd): preview = util.string_as_bool( preview ) if chunk: return self.get_chunk(trans, dataset, chunk) @@ -652,7 +652,7 @@ class Vcf( Tabular ): MetadataElement( name="columns", default=10, desc="Number of columns", readonly=True, visible=False ) MetadataElement( name="column_types", default=['str','int','str','str','str','int','str','list','str','str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False ) - MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, multiple=True, visible=False ) + MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, optional=True, multiple=True, visible=False ) MetadataElement( name="sample_names", default=[], desc="Sample names", readonly=True, visible=False, optional=True, no_value=[] ) def sniff( self, filename ): diff --git a/lib/galaxy/datatypes/util/gff_util.py b/lib/galaxy/datatypes/util/gff_util.py index cc8f20a1e9a..764631582cc 100644 --- a/lib/galaxy/datatypes/util/gff_util.py +++ b/lib/galaxy/datatypes/util/gff_util.py @@ -75,7 +75,7 @@ class GFFFeature( GFFInterval ): # Preference for name: GTF, GFF3, GFF. for attr_name in [ # GTF: - 'transcript_id', 'gene_id', + 'gene_id', 'transcript_id', # GFF3: 'ID', 'id', # GFF (TODO): diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py index 6fd4deb7a0f..75c9aba18bd 100644 --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -1,34 +1,71 @@ """ Custom exceptions for Galaxy """ + +from galaxy import eggs +eggs.require( "Paste" ) + from paste import httpexceptions +from ..exceptions import error_codes + class MessageException( Exception ): """ - Exception to make throwing errors from deep in controllers easier + Exception to make throwing errors from deep in controllers easier. """ - def __init__( self, err_msg, type="info" ): - self.err_msg = err_msg + # status code to be set when used with API. + status_code = 400 + # Error code information embedded into API json responses. + err_code = error_codes.UNKNOWN + + def __init__( self, err_msg=None, type="info", **extra_error_info ): + self.err_msg = err_msg or self.err_code.default_error_message self.type = type + self.extra_error_info = extra_error_info + def __str__( self ): return self.err_msg + class ItemDeletionException( MessageException ): pass + class ItemAccessibilityException( MessageException ): - pass + status_code = 403 + err_code = error_codes.USER_CANNOT_ACCESS_ITEM + class ItemOwnershipException( MessageException ): - pass + status_code = 403 + err_code = error_codes.USER_DOES_NOT_OWN_ITEM + + +class DuplicatedSlugException( MessageException ): + status_code = 400 + err_code = error_codes.USER_SLUG_DUPLICATE + + +class ObjectAttributeInvalidException( MessageException ): + status_code = 400 + err_code = error_codes.USER_OBJECT_ATTRIBUTE_INVALID + + +class ObjectAttributeMissingException( MessageException ): + status_code = 400 + err_code = error_codes.USER_OBJECT_ATTRIBUTE_MISSING + class ActionInputError( MessageException ): def __init__( self, err_msg, type="error" ): super( ActionInputError, self ).__init__( err_msg, type ) -class ObjectNotFound( Exception ): + +class ObjectNotFound( MessageException ): """ Accessed object was not found """ - pass + status_code = 404 + err_code = error_codes.USER_OBJECT_NOT_FOUND + class ObjectInvalid( Exception ): """ Accessed object store ID is invalid """ diff --git a/lib/galaxy/exceptions/error_codes.json b/lib/galaxy/exceptions/error_codes.json new file mode 100644 index 00000000000..99f93fae53b --- /dev/null +++ b/lib/galaxy/exceptions/error_codes.json @@ -0,0 +1,57 @@ +[ + { + "name": "UNKNOWN", + "code": 0, + "message": "Unknown error occurred while processing request." + }, + { + "name": "USER_CANNOT_RUN_AS", + "code": 400001, + "message": "User does not have permissions to run jobs as another user." + }, + { + "name": "USER_INVALID_RUN_AS", + "code": 400002, + "message": "Invalid run_as request - run_as user does not exist." + }, + { + "name": "USER_INVALID_JSON", + "code": 400003, + "message": "Your request did not appear to be valid JSON, please consult the API documentation." + }, + { + "name": "USER_OBJECT_ATTRIBUTE_INVALID", + "code": 400004, + "message": "Attempted to create or update object with invalid attribute value." + }, + { + "name": "USER_OBJECT_ATTRIBUTE_MISSING", + "code": 400005, + "message": "Attempted to create object without required attribute." + }, + { + "name": "USER_SLUG_DUPLICATE", + "code": 400006, + "message": "Slug must be unique per user." + }, + { + "name": "USER_NO_API_KEY", + "code": 403001, + "message": "API Authentication Required for this request" + }, + { + "name": "USER_CANNOT_ACCESS_ITEM", + "code": 403002, + "message": "User cannot access specified item." + }, + { + "name": "USER_DOES_NOT_OWN_ITEM", + "code": 403003, + "message": "User does not own specified item." + }, + { + "name": "USER_OBJECT_NOT_FOUND", + "code": 404001, + "message": "No such object found." + } +] diff --git a/lib/galaxy/exceptions/error_codes.py b/lib/galaxy/exceptions/error_codes.py new file mode 100644 index 00000000000..4afe67bea80 --- /dev/null +++ b/lib/galaxy/exceptions/error_codes.py @@ -0,0 +1,32 @@ +from pkg_resources import resource_string +from json import loads + +# Error codes are provided as a convience to Galaxy API clients, but at this +# time they do represent part of the more stable interface. They can change +# without warning between releases. +UNKNOWN_ERROR_MESSAGE = "Unknown error occurred while processing request." + + +class ErrorCode( object ): + + def __init__( self, code, default_error_message ): + self.code = code + self.default_error_message = default_error_message or UNKNOWN_ERROR_MESSAGE + + def __str__( self ): + return str( self.default_error_message ) + + def __int__( self ): + return int( self.code ) + + @staticmethod + def from_dict( entry ): + name = entry.get("name") + code = entry.get("code") + message = entry.get("message") + return ( name, ErrorCode( code, message ) ) + +error_codes_json = resource_string( __name__, 'error_codes.json' ) +for entry in loads( error_codes_json ): + name, error_code_obj = ErrorCode.from_dict( entry ) + globals()[ name ] = error_code_obj diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 42fa0270e1a..30fe5833a4a 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -14,7 +14,6 @@ import re import shutil import subprocess import sys -import threading import traceback from galaxy import model, util from galaxy.datatypes import metadata @@ -39,21 +38,6 @@ DATABASE_MAX_STRING_SIZE_PRETTY = util.DATABASE_MAX_STRING_SIZE_PRETTY # and should eventually become API'd TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json' -class Sleeper( object ): - """ - Provides a 'sleep' method that sleeps for a number of seconds *unless* - the notify method is called (from a different thread). - """ - def __init__( self ): - self.condition = threading.Condition() - def sleep( self, seconds ): - self.condition.acquire() - self.condition.wait( seconds ) - self.condition.release() - def wake( self ): - self.condition.acquire() - self.condition.notify() - self.condition.release() class JobDestination( Bunch ): """ @@ -74,6 +58,7 @@ class JobDestination( Bunch ): if self.tags is not None: self['tags'] = [ x.strip() for x in self.tags.split(',') ] + class JobToolConfiguration( Bunch ): """ Provides details on what handler and destination a tool should use @@ -87,18 +72,21 @@ class JobToolConfiguration( Bunch ): self['params'] = dict() super(JobToolConfiguration, self).__init__(**kwds) + class JobConfiguration( object ): """A parser and interface to advanced job management features. These features are configured in the job configuration, by default, ``job_conf.xml`` """ DEFAULT_NWORKERS = 4 + def __init__(self, app): """Parse the job configuration XML. """ self.app = app self.runner_plugins = [] self.handlers = {} + self.handler_runner_plugins = {} self.default_handler_id = None self.destinations = {} self.destination_tags = {} @@ -151,6 +139,10 @@ class JobConfiguration( object ): else: log.debug("Read definition for handler '%s'" % id) self.handlers[id] = (id,) + for plugin in handler.findall('plugin'): + if id not in self.handler_runner_plugins: + self.handler_runner_plugins[id] = [] + self.handler_runner_plugins[id].append( plugin.get('id') ) if handler.get('tags', None) is not None: for tag in [ x.strip() for x in handler.get('tags').split(',') ]: if tag in self.handlers: @@ -188,17 +180,17 @@ class JobConfiguration( object ): self.tools[id].append(JobToolConfiguration(**dict(tool.items()))) self.tools[id][-1]['params'] = self.__get_params(tool) - types = dict(registered_user_concurrent_jobs = int, - anonymous_user_concurrent_jobs = int, - walltime = str, - output_size = int) + types = dict(registered_user_concurrent_jobs=int, + anonymous_user_concurrent_jobs=int, + walltime=str, + output_size=int) - self.limits = Bunch(registered_user_concurrent_jobs = None, - anonymous_user_concurrent_jobs = None, - walltime = None, - walltime_delta = None, - output_size = None, - concurrent_jobs = {}) + self.limits = Bunch(registered_user_concurrent_jobs=None, + anonymous_user_concurrent_jobs=None, + walltime=None, + walltime_delta=None, + output_size=None, + concurrent_jobs={}) # Parse job limits limits = root.find('limits') @@ -269,12 +261,12 @@ class JobConfiguration( object ): self.default_destination_id = self.app.config.default_cluster_job_runner # Set the job limits - self.limits = Bunch(registered_user_concurrent_jobs = self.app.config.registered_user_job_limit, - anonymous_user_concurrent_jobs = self.app.config.anonymous_user_job_limit, - walltime = self.app.config.job_walltime, - walltime_delta = self.app.config.job_walltime_delta, - output_size = self.app.config.output_size_limit, - concurrent_jobs = {}) + self.limits = Bunch(registered_user_concurrent_jobs=self.app.config.registered_user_job_limit, + anonymous_user_concurrent_jobs=self.app.config.anonymous_user_job_limit, + walltime=self.app.config.job_walltime, + walltime_delta=self.app.config.job_walltime_delta, + output_size=self.app.config.output_size_limit, + concurrent_jobs={}) log.debug('Done loading job configuration') @@ -433,13 +425,19 @@ class JobConfiguration( object ): """ return self.destinations.get(id_or_tag, None) - def get_job_runner_plugins(self): + def get_job_runner_plugins(self, handler_id): """Load all configured job runner plugins :returns: list of job runner plugins """ rval = {} - for runner in self.runner_plugins: + if handler_id in self.handler_runner_plugins: + plugins_to_load = [ rp for rp in self.runner_plugins if rp['id'] in self.handler_runner_plugins[handler_id] ] + log.info( "Handler '%s' will load specified runner plugins: %s", handler_id, ', '.join( [ rp['id'] for rp in plugins_to_load ] ) ) + else: + plugins_to_load = self.runner_plugins + log.info( "Handler '%s' will load all configured runner plugins", handler_id ) + for runner in plugins_to_load: class_names = [] module = None id = runner['id'] @@ -490,7 +488,7 @@ class JobConfiguration( object ): try: rval[id] = runner_class( self.app, runner[ 'workers' ], **runner.get( 'kwds', {} ) ) except TypeError: - log.warning( "Job runner '%s:%s' has not been converted to a new-style runner" % ( module_name, class_name ) ) + log.exception( "Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load" % ( module_name, class_name ) ) rval[id] = runner_class( self.app ) log.debug( "Loaded job runner '%s:%s' as '%s'" % ( module_name, class_name, id ) ) return rval @@ -549,6 +547,7 @@ class JobConfiguration( object ): else: log.warning("Legacy destination with id '%s' could not be converted: Unknown runner plugin: %s" % (id, destination.runner)) + class JobWrapper( object ): """ Wraps a 'model.Job' with convenience methods for running processes and @@ -643,7 +642,7 @@ class JobWrapper( object ): Prepare the job to run by creating the working directory and the config files. """ - self.sa_session.expunge_all() #this prevents the metadata reverting that has been seen in conjunction with the PBS job runner + self.sa_session.expunge_all() # this prevents the metadata reverting that has been seen in conjunction with the PBS job runner if not os.path.exists( self.working_directory ): os.mkdir( self.working_directory ) @@ -702,12 +701,9 @@ class JobWrapper( object ): self.command_line = self.tool.build_command_line( param_dict ) # FIXME: for now, tools get Galaxy's lib dir in their path if self.command_line and self.command_line.startswith( 'python' ): - self.galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root + self.galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root # Shell fragment to inject dependencies - if self.app.config.use_tool_dependencies: - self.dependency_shell_commands = self.tool.build_dependency_shell_commands() - else: - self.dependency_shell_commands = None + self.dependency_shell_commands = self.tool.build_dependency_shell_commands() # We need command_line persisted to the db in order for Galaxy to re-queue the job # if the server was stopped and restarted before the job finished job.command_line = self.command_line @@ -738,7 +734,7 @@ class JobWrapper( object ): job.traceback = traceback.format_exc() # Get the exception and let the tool attempt to generate # a better message - etype, evalue, tb = sys.exc_info() + etype, evalue, tb = sys.exc_info() m = self.tool.handle_job_failure_exception( evalue ) if m: message = m @@ -793,8 +789,8 @@ class JobWrapper( object ): # If the job was deleted, call tool specific fail actions (used for e.g. external metadata) and clean up if self.tool: self.tool.job_failed( self, message, exception ) - if self.app.config.cleanup_job == 'always' or (self.app.config.cleanup_job == 'onsuccess' and job.state == job.states.DELETED): - self.cleanup() + delete_files = self.app.config.cleanup_job == 'always' or (self.app.config.cleanup_job == 'onsuccess' and job.state == job.states.DELETED) + self.cleanup( delete_files=delete_files ) def pause( self, job=None, message=None ): if job is None: @@ -809,7 +805,7 @@ class JobWrapper( object ): job.state = job.states.PAUSED self.sa_session.add( job ) - def change_state( self, state, info = False ): + def change_state( self, state, info=False ): job = self.get_job() self.sa_session.refresh( job ) for dataset_assoc in job.output_datasets + job.output_library_datasets: @@ -835,7 +831,7 @@ class JobWrapper( object ): log.warning('set_runner() is deprecated, use set_job_destination()') self.set_job_destination(self.job_destination, external_id) - def set_job_destination(self, job_destination, external_id=None ): + def set_job_destination( self, job_destination, external_id=None ): """ Persist job destination params in the database for recovery. @@ -887,11 +883,10 @@ class JobWrapper( object ): # We set final_job_state to use for dataset management, but *don't* set # job.state until after dataset collection to prevent history issues - if job.states.ERROR != job.state: - if ( self.check_tool_output( stdout, stderr, tool_exit_code, job )): - final_job_state = job.states.OK - else: - final_job_state = job.states.ERROR + if ( self.check_tool_output( stdout, stderr, tool_exit_code, job ) ): + final_job_state = job.states.OK + else: + final_job_state = job.states.ERROR if self.version_string_cmd: version_filename = self.get_version_string_path() @@ -915,11 +910,11 @@ class JobWrapper( object ): job.state = final_job_state return self.fail( "Job %s's output dataset(s) could not be read" % job.id ) - job_context = ExpressionContext( dict( stdout = job.stdout, stderr = job.stderr ) ) + job_context = ExpressionContext( dict( stdout=job.stdout, stderr=job.stderr ) ) for dataset_assoc in job.output_datasets + job.output_library_datasets: context = self.get_dataset_finish_context( job_context, dataset_assoc.dataset.dataset ) #should this also be checking library associations? - can a library item be added from a history before the job has ended? - lets not allow this to occur - for dataset in dataset_assoc.dataset.dataset.history_associations + dataset_assoc.dataset.dataset.library_associations: #need to update all associated output hdas, i.e. history was shared with job running + for dataset in dataset_assoc.dataset.dataset.history_associations + dataset_assoc.dataset.dataset.library_associations: # need to update all associated output hdas, i.e. history was shared with job running trynum = 0 while trynum < self.app.config.retry_job_output_collection: try: @@ -930,9 +925,9 @@ class JobWrapper( object ): except ( OSError, ObjectNotFound ), e: trynum += 1 log.warning( 'Error accessing %s, will retry: %s', dataset.dataset.file_name, e ) - time.sleep( 2 ) + time.sleep( 2 ) dataset.blurb = 'done' - dataset.peek = 'no peek' + dataset.peek = 'no peek' dataset.info = (dataset.info or '') if context['stdout'].strip(): #Ensure white space between entries @@ -960,7 +955,7 @@ class JobWrapper( object ): #it would be quicker to just copy the metadata from the originating output dataset, #but somewhat trickier (need to recurse up the copied_from tree), for now we'll call set_meta() if ( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and self.app.config.retry_metadata_internally ): - dataset.datatype.set_meta( dataset, overwrite = False ) #call datatype.set_meta directly for the initial set_meta call during dataset creation + dataset.datatype.set_meta( dataset, overwrite=False ) # call datatype.set_meta directly for the initial set_meta call during dataset creation elif not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and job.states.ERROR != final_job_state: dataset._state = model.Dataset.states.FAILED_METADATA else: @@ -1030,19 +1025,22 @@ class JobWrapper( object ): out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] ) inp_data.update( [ ( da.name, da.dataset ) for da in job.input_library_datasets ] ) out_data.update( [ ( da.name, da.dataset ) for da in job.output_library_datasets ] ) - param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] ) # why not re-use self.param_dict here? ##dunno...probably should, this causes tools.parameters.basic.UnvalidatedValue to be used in following methods instead of validated and transformed values during i.e. running workflows + param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] ) # why not re-use self.param_dict here? ##dunno...probably should, this causes tools.parameters.basic.UnvalidatedValue to be used in following methods instead of validated and transformed values during i.e. running workflows param_dict = self.tool.params_from_strings( param_dict, self.app ) # Check for and move associated_files self.tool.collect_associated_files(out_data, self.working_directory) gitd = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first() if gitd: - self.tool.collect_associated_files({'' : gitd}, self.working_directory) + self.tool.collect_associated_files({'': gitd}, self.working_directory) # Create generated output children and primary datasets and add to param_dict - collected_datasets = {'children':self.tool.collect_child_datasets(out_data, self.working_directory),'primary':self.tool.collect_primary_datasets(out_data, self.working_directory)} - param_dict.update({'__collected_datasets__':collected_datasets}) + collected_datasets = { + 'children': self.tool.collect_child_datasets(out_data, self.working_directory), + 'primary': self.tool.collect_primary_datasets(out_data, self.working_directory) + } + param_dict.update({'__collected_datasets__': collected_datasets}) # Certain tools require tasks to be completed after job execution # ( this used to be performed in the "exec_after_process" hook, but hooks are deprecated ). - self.tool.exec_after_process( self.queue.app, inp_data, out_data, param_dict, job = job ) + self.tool.exec_after_process( self.queue.app, inp_data, out_data, param_dict, job=job ) # Call 'exec_after_process' hook self.tool.call_hook( 'exec_after_process', self.queue.app, inp_data=inp_data, out_data=out_data, param_dict=param_dict, @@ -1068,22 +1066,26 @@ class JobWrapper( object ): self.sa_session.flush() log.debug( 'job %d ended' % self.job_id ) - if self.app.config.cleanup_job == 'always' or ( not stderr and self.app.config.cleanup_job == 'onsuccess' ): - self.cleanup() + delete_files = self.app.config.cleanup_job == 'always' or ( job.state == job.states.OK and self.app.config.cleanup_job == 'onsuccess' ) + self.cleanup( delete_files=delete_files ) def check_tool_output( self, stdout, stderr, tool_exit_code, job ): return check_output( self.tool, stdout, stderr, tool_exit_code, job ) - def cleanup( self ): - # remove temporary files + def cleanup( self, delete_files=True ): + # At least one of these tool cleanup actions (job import), is needed + # for thetool to work properly, that is why one might want to run + # cleanup but not delete files. try: - for fname in self.extra_filenames: - os.remove( fname ) - self.external_output_metadata.cleanup_external_metadata( self.sa_session ) + if delete_files: + for fname in self.extra_filenames: + os.remove( fname ) + self.external_output_metadata.cleanup_external_metadata( self.sa_session ) galaxy.tools.imp_exp.JobExportHistoryArchiveWrapper( self.job_id ).cleanup_after_job( self.sa_session ) galaxy.tools.imp_exp.JobImportHistoryArchiveWrapper( self.app, self.job_id ).cleanup_after_job() galaxy.tools.genome_index.GenomeIndexToolWrapper( self.job_id ).postprocessing( self.sa_session, self.app ) - self.app.object_store.delete(self.get_job(), base_dir='job_work', entire_dir=True, dir_only=True, extra_dir=str(self.job_id)) + if delete_files: + self.app.object_store.delete(self.get_job(), base_dir='job_work', entire_dir=True, dir_only=True, extra_dir=str(self.job_id)) except: log.exception( "Unable to cleanup job %d" % self.job_id ) @@ -1133,7 +1135,7 @@ class JobWrapper( object ): def get_input_fnames( self ): job = self.get_job() filenames = [] - for da in job.input_datasets + job.input_library_datasets: #da is JobToInputDatasetAssociation object + for da in job.input_datasets + job.input_library_datasets: # da is JobToInputDatasetAssociation object if da.dataset: filenames.extend(self.get_input_dataset_fnames(da.dataset)) return filenames @@ -1155,11 +1157,12 @@ class JobWrapper( object ): def compute_outputs( self ) : class DatasetPath( object ): - def __init__( self, dataset_id, real_path, false_path = None, mutable = True ): + def __init__( self, dataset_id, real_path, false_path=None, mutable=True ): self.dataset_id = dataset_id self.real_path = real_path self.false_path = false_path self.mutable = mutable + def __str__( self ): if self.false_path is None: return self.real_path @@ -1176,13 +1179,13 @@ class JobWrapper( object ): self.output_hdas_and_paths = {} for name, hda in [ ( da.name, da.dataset ) for da in job.output_datasets + job.output_library_datasets ]: false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % hda.dataset.id ) ) - dsp = DatasetPath( hda.dataset.id, hda.dataset.file_name, false_path, mutable = hda.dataset.external_filename is None ) + dsp = DatasetPath( hda.dataset.id, hda.dataset.file_name, false_path, mutable=hda.dataset.external_filename is None ) self.output_paths.append( dsp ) self.output_hdas_and_paths[name] = hda, dsp if special: false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % special.dataset.id ) ) else: - results = [ ( da.name, da.dataset, DatasetPath( da.dataset.dataset.id, da.dataset.file_name, mutable = da.dataset.dataset.external_filename is None ) ) for da in job.output_datasets + job.output_library_datasets ] + results = [ ( da.name, da.dataset, DatasetPath( da.dataset.dataset.id, da.dataset.file_name, mutable=da.dataset.dataset.external_filename is None ) ) for da in job.output_datasets + job.output_library_datasets ] self.output_paths = [t[2] for t in results] self.output_hdas_and_paths = dict([(t[0], t[1:]) for t in results]) if special: @@ -1257,13 +1260,13 @@ class JobWrapper( object ): datatypes_config = self.app.datatypes_registry.integrated_datatypes_configs return self.external_output_metadata.setup_external_metadata( [ output_dataset_assoc.dataset for output_dataset_assoc in job.output_datasets ], self.sa_session, - exec_dir = exec_dir, - tmp_dir = tmp_dir, - dataset_files_path = dataset_files_path, - config_root = config_root, - config_file = config_file, - datatypes_config = datatypes_config, - job_metadata = os.path.join( self.working_directory, TOOL_PROVIDED_JOB_METADATA_FILE ), + exec_dir=exec_dir, + tmp_dir=tmp_dir, + dataset_files_path=dataset_files_path, + config_root=config_root, + config_file=config_file, + datatypes_config=datatypes_config, + job_metadata=os.path.join( self.working_directory, TOOL_PROVIDED_JOB_METADATA_FILE ), **kwds ) @property @@ -1337,7 +1340,7 @@ class JobWrapper( object ): just copy these files directly to the ulimate destination. """ return output_path - + @property def requires_setting_metadata( self ): if self.tool: @@ -1410,7 +1413,7 @@ class TaskWrapper(JobWrapper): out_data.update( [ ( da.name, da.dataset ) for da in job.output_library_datasets ] ) # DBTODO New method for generating command line for a task? # These can be passed on the command line if wanted as $userId $userEmail - if job.history and job.history.user: # check for anonymous user! + if job.history and job.history.user: # check for anonymous user! userId = '%d' % job.history.user.id userEmail = str(job.history.user.email) else: @@ -1449,12 +1452,9 @@ class TaskWrapper(JobWrapper): self.command_line = self.command_line.replace(k, v) # FIXME: for now, tools get Galaxy's lib dir in their path if self.command_line and self.command_line.startswith( 'python' ): - self.galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root + self.galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root # Shell fragment to inject dependencies - if self.app.config.use_tool_dependencies: - self.dependency_shell_commands = self.tool.build_dependency_shell_commands() - else: - self.dependency_shell_commands = None + self.dependency_shell_commands = self.tool.build_dependency_shell_commands() # We need command_line persisted to the db in order for Galaxy to re-queue the job # if the server was stopped and restarted before the job finished task.command_line = self.command_line @@ -1474,7 +1474,7 @@ class TaskWrapper(JobWrapper): self.status = 'error' # How do we want to handle task failure? Fail the job and let it clean up? - def change_state( self, state, info = False ): + def change_state( self, state, info=False ): task = self.get_task() self.sa_session.refresh( task ) if info: @@ -1523,8 +1523,8 @@ class TaskWrapper(JobWrapper): # if the job was deleted, don't finish it if task.state == task.states.DELETED: # Job was deleted by an administrator - if self.app.config.cleanup_job in ( 'always', 'onsuccess' ): - self.cleanup() + delete_files = self.app.config.cleanup_job in ( 'always', 'onsuccess' ) + self.cleanup( delete_files=delete_files ) return elif task.state == task.states.ERROR: self.fail( task.info ) @@ -1590,11 +1590,14 @@ class NoopQueue( object ): """ def put( self, *args, **kwargs ): return + def put_stop( self, *args ): return + def shutdown( self ): return + class ParallelismInfo(object): """ Stores the information (if any) for running multiple instances of the tool in parallel @@ -1606,7 +1609,7 @@ class ParallelismInfo(object): items = tag.iteritems() else: items = tag.attrib.items() - self.attributes = dict([item for item in items if item[0] != 'method' ]) + self.attributes = dict( [ item for item in items if item[ 0 ] != 'method' ]) if len(self.attributes) == 0: # legacy basic mode - provide compatible defaults self.attributes['split_size'] = 20 diff --git a/lib/galaxy/jobs/actions/post.py b/lib/galaxy/jobs/actions/post.py index e26732fa7f8..cba19328a6c 100644 --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -12,7 +12,7 @@ def get_form_template(action_type, title, content, help, on_output = True ): form = """ if (pja.action_type == "%s"){ p_str = "
        %s
        on " + pja.output_name + "\ -
        "; +
        "; %s p_str += "
        %s
        "; }""" % (action_type, title, content, help) @@ -20,7 +20,7 @@ def get_form_template(action_type, title, content, help, on_output = True ): form = """ if (pja.action_type == "%s"){ p_str = "
        %s \ -
        "; +
        "; %s p_str += "
        %s
        "; }""" % (action_type, title, content, help) diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index 969ad21049f..7b9bd1fd3e4 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -1,8 +1,11 @@ from os import getcwd from os.path import abspath +CAPTURE_RETURN_CODE = "return_code=$?" +YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"' -def build_command( job, job_wrapper, include_metadata=False, include_work_dir_outputs=True ): + +def build_command( runner, job_wrapper, include_metadata=False, include_work_dir_outputs=True, remote_command_params={} ): """ Compose the sequence of commands necessary to execute a job. This will currently include: @@ -13,64 +16,125 @@ def build_command( job, job_wrapper, include_metadata=False, include_work_dir_ou - commands to set metadata (if include_metadata is True) """ - commands = job_wrapper.get_command_line() + commands_builder = CommandsBuilder(job_wrapper.get_command_line()) # All job runners currently handle this case which should never occur - if not commands: + if not commands_builder.commands: return None - # Remove trailing semi-colon so we can start hacking up this command. - # TODO: Refactor to compose a list and join with ';', would be more clean. - commands = commands.rstrip("; ") + __handle_version_command(commands_builder, job_wrapper) + __handle_task_splitting(commands_builder, job_wrapper) + __handle_dependency_resolution(commands_builder, job_wrapper, remote_command_params) + if include_work_dir_outputs: + __handle_work_dir_outputs(commands_builder, job_wrapper, runner, remote_command_params) + + if include_metadata and job_wrapper.requires_setting_metadata: + __handle_metadata(commands_builder, job_wrapper, runner, remote_command_params) + + return commands_builder.build() + + +def __handle_version_command(commands_builder, job_wrapper): # Prepend version string if job_wrapper.version_string_cmd: - commands = "%s &> %s; " % ( job_wrapper.version_string_cmd, job_wrapper.get_version_string_path() ) + commands + version_command = "%s > %s 2>&1" % ( job_wrapper.version_string_cmd, job_wrapper.get_version_string_path() ) + commands_builder.prepend_command(version_command) + +def __handle_task_splitting(commands_builder, job_wrapper): # prepend getting input files (if defined) - if hasattr(job_wrapper, 'prepare_input_files_cmds') and job_wrapper.prepare_input_files_cmds is not None: - commands = "; ".join( job_wrapper.prepare_input_files_cmds + [ commands ] ) + if getattr(job_wrapper, 'prepare_input_files_cmds', None): + commands_builder.prepend_commands(job_wrapper.prepare_input_files_cmds) + + +def __handle_dependency_resolution(commands_builder, job_wrapper, remote_command_params): + local_dependency_resolution = remote_command_params.get("dependency_resolution", "local") == "local" # Prepend dependency injection - if job_wrapper.dependency_shell_commands: - commands = "; ".join( job_wrapper.dependency_shell_commands + [ commands ] ) + if job_wrapper.dependency_shell_commands and local_dependency_resolution: + commands_builder.prepend_commands(job_wrapper.dependency_shell_commands) - # Coping work dir outputs or setting metadata will mask return code of - # tool command. If these are used capture the return code and ensure - # the last thing that happens is an exit with return code. - capture_return_code_command = "; return_code=$?" - captured_return_code = False +def __handle_work_dir_outputs(commands_builder, job_wrapper, runner, remote_command_params): # Append commands to copy job outputs based on from_work_dir attribute. - if include_work_dir_outputs: - work_dir_outputs = job.get_work_dir_outputs( job_wrapper ) - if work_dir_outputs: - if not captured_return_code: - commands += capture_return_code_command - captured_return_code = True + work_dir_outputs_kwds = {} + if 'working_directory' in remote_command_params: + work_dir_outputs_kwds['job_working_directory'] = remote_command_params['working_directory'] + work_dir_outputs = runner.get_work_dir_outputs( job_wrapper, **work_dir_outputs_kwds ) + if work_dir_outputs: + commands_builder.capture_return_code() + copy_commands = map(__copy_if_exists_command, work_dir_outputs) + commands_builder.append_commands(copy_commands) - commands += "; " + "; ".join( [ "if [ -f %s ] ; then cp %s %s ; fi" % - ( source_file, source_file, destination ) for ( source_file, destination ) in work_dir_outputs ] ) +def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_params): # Append metadata setting commands, we don't want to overwrite metadata # that was copied over in init_meta(), as per established behavior - if include_metadata and job_wrapper.requires_setting_metadata: - metadata_command = job_wrapper.setup_external_metadata( - exec_dir=abspath( getcwd() ), - tmp_dir=job_wrapper.working_directory, - dataset_files_path=job.app.model.Dataset.file_path, - output_fnames=job_wrapper.get_output_fnames(), - set_extension=False, - kwds={ 'overwrite' : False } - ) or '' - metadata_command = metadata_command.strip() - if metadata_command: - if not captured_return_code: - commands += capture_return_code_command - captured_return_code = True - commands += "; cd %s; %s" % (abspath( getcwd() ), metadata_command) + metadata_kwds = remote_command_params.get('metadata_kwds', {}) + exec_dir = metadata_kwds.get( 'exec_dir', abspath( getcwd() ) ) + tmp_dir = metadata_kwds.get( 'tmp_dir', job_wrapper.working_directory ) + dataset_files_path = metadata_kwds.get( 'dataset_files_path', runner.app.model.Dataset.file_path ) + output_fnames = metadata_kwds.get( 'output_fnames', job_wrapper.get_output_fnames() ) + config_root = metadata_kwds.get( 'config_root', None ) + config_file = metadata_kwds.get( 'config_file', None ) + datatypes_config = metadata_kwds.get( 'datatypes_config', None ) + metadata_command = job_wrapper.setup_external_metadata( + exec_dir=exec_dir, + tmp_dir=tmp_dir, + dataset_files_path=dataset_files_path, + output_fnames=output_fnames, + set_extension=False, + config_root=config_root, + config_file=config_file, + datatypes_config=datatypes_config, + kwds={ 'overwrite' : False } + ) or '' + metadata_command = metadata_command.strip() + if metadata_command: + commands_builder.capture_return_code() + commands_builder.append_command("cd %s; %s" % (exec_dir, metadata_command)) - if captured_return_code: - commands += '; sh -c "exit $return_code"' - return commands +def __copy_if_exists_command(work_dir_output): + source_file, destination = work_dir_output + return "if [ -f %s ] ; then cp %s %s ; fi" % ( source_file, source_file, destination ) + + +class CommandsBuilder(object): + + def __init__(self, initial_command): + # Remove trailing semi-colon so we can start hacking up this command. + # TODO: Refactor to compose a list and join with ';', would be more clean. + commands = initial_command.rstrip("; ") + self.commands = commands + + # Coping work dir outputs or setting metadata will mask return code of + # tool command. If these are used capture the return code and ensure + # the last thing that happens is an exit with return code. + self.return_code_captured = False + + def prepend_command(self, command): + self.commands = "%s; %s" % (command, self.commands) + return self + + def prepend_commands(self, commands): + return self.prepend_command("; ".join(commands)) + + def append_command(self, command): + self.commands = "%s; %s" % (self.commands, command) + + def append_commands(self, commands): + self.append_command("; ".join(commands)) + + def capture_return_code(self): + if not self.return_code_captured: + self.return_code_captured = True + self.append_command(CAPTURE_RETURN_CODE) + + def build(self): + if self.return_code_captured: + self.append_command(YIELD_CAPTURED_CODE) + return self.commands + +__all__ = [build_command] diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index d738c90cce9..67ac354b6a5 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -11,7 +11,8 @@ from Queue import Queue, Empty from sqlalchemy.sql.expression import and_, or_, select, func from galaxy import model -from galaxy.jobs import Sleeper, JobWrapper, TaskWrapper, JobDestination +from galaxy.util.sleeper import Sleeper +from galaxy.jobs import JobWrapper, TaskWrapper, JobDestination log = logging.getLogger( __name__ ) @@ -106,7 +107,7 @@ class JobHandlerQueue( object ): if job.tool_id not in self.app.toolbox.tools_by_id: log.warning( "(%s) Tool '%s' removed from tool config, unable to recover job" % ( job.id, job.tool_id ) ) JobWrapper( job, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' ) - if job.job_runner_name is not None and job.job_runner_external_id is None: + elif job.job_runner_name is not None and job.job_runner_external_id is None: # This could happen during certain revisions of Galaxy where a runner URL was persisted before the job was dispatched to a runner. log.debug( "(%s) Job runner assigned but no external ID recorded, adding to the job handler queue" % job.id ) job.job_runner_name = None @@ -564,7 +565,7 @@ class JobHandlerStopQueue( object ): class DefaultJobDispatcher( object ): def __init__( self, app ): self.app = app - self.job_runners = self.app.job_config.get_job_runner_plugins() + self.job_runners = self.app.job_config.get_job_runner_plugins( self.app.config.server_name ) # Once plugins are loaded, all job destinations that were created from # URLs can have their URL params converted to the destination's param # dict by the plugin. diff --git a/lib/galaxy/jobs/manager.py b/lib/galaxy/jobs/manager.py index 054764eef04..a8e81cba883 100644 --- a/lib/galaxy/jobs/manager.py +++ b/lib/galaxy/jobs/manager.py @@ -11,7 +11,8 @@ import time from Queue import Empty, Queue from galaxy import model -from galaxy.jobs import handler, JobWrapper, NoopQueue, Sleeper +from galaxy.util.sleeper import Sleeper +from galaxy.jobs import handler, JobWrapper, NoopQueue from galaxy.util.json import from_json_string log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index e1cd9abbf61..7f8b7b320aa 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -15,19 +15,48 @@ import galaxy.jobs from galaxy.jobs.command_factory import build_command from galaxy import model from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size +from galaxy.util import in_directory from galaxy.jobs.runners.util.job_script import job_script log = logging.getLogger( __name__ ) STOP_SIGNAL = object() + +class RunnerParams( object ): + + def __init__( self, specs = None, params = None ): + self.specs = specs or dict() + self.params = params or dict() + for name, value in self.params.items(): + assert name in self.specs, 'Invalid job runner parameter for this plugin: %s' % name + if 'map' in self.specs[ name ]: + try: + self.params[ name ] = self.specs[ name ][ 'map' ]( value ) + except Exception, e: + raise Exception( 'Job runner parameter "%s" value "%s" could not be converted to the correct type: %s' % ( name, value, e ) ) + if 'valid' in self.specs[ name ]: + assert self.specs[ name ][ 'valid' ]( value ), 'Job runner parameter %s failed validation' % name + + def __getattr__( self, name ): + return self.params.get( name, self.specs[ name ][ 'default' ] ) + + __getitem__ = __getattr__ + + class BaseJobRunner( object ): - def __init__( self, app, nworkers ): + def __init__( self, app, nworkers, **kwargs ): """Start the job runner """ self.app = app self.sa_session = app.model.context self.nworkers = nworkers + runner_param_specs = dict( recheck_missing_job_retries = dict( map = int, valid = lambda x: x >= 0, default = 0 ) ) + if 'runner_param_specs' in kwargs: + runner_param_specs.update( kwargs.pop( 'runner_param_specs' ) ) + if kwargs: + log.debug( 'Loading %s with params: %s', self.runner_name, kwargs ) + self.runner_params = RunnerParams( specs = runner_param_specs, params = kwargs ) def _init_worker_threads(self): """Start ``nworkers`` worker threads. @@ -114,7 +143,7 @@ class BaseJobRunner( object ): job_wrapper.cleanup() return False elif job_state != model.Job.states.QUEUED: - log.info( "(%d) Job is in state %s, skipping execution" % ( job_id, job_state ) ) + log.info( "(%s) Job is in state %s, skipping execution" % ( job_id, job_state ) ) # cleanup may not be safe in all states return False @@ -146,23 +175,13 @@ class BaseJobRunner( object ): def build_command_line( self, job_wrapper, include_metadata=False, include_work_dir_outputs=True ): return build_command( self, job_wrapper, include_metadata=include_metadata, include_work_dir_outputs=include_work_dir_outputs ) - def get_work_dir_outputs( self, job_wrapper ): + def get_work_dir_outputs( self, job_wrapper, job_working_directory=None ): """ Returns list of pairs (source_file, destination) describing path to work_dir output file and ultimate destination. """ - - def in_directory( file, directory ): - """ - Return true, if the common prefix of both is equal to directory - e.g. /a/b/c/d.rst and directory is /a/b, the common prefix is /a/b - """ - - # Make both absolute. - directory = os.path.abspath( directory ) - file = os.path.abspath( file ) - - return os.path.commonprefix( [ file, directory ] ) == directory + if not job_working_directory: + job_working_directory = os.path.abspath( job_wrapper.working_directory ) # Set up dict of dataset id --> output path; output path can be real or # false depending on outputs_to_working_directory @@ -186,9 +205,9 @@ class BaseJobRunner( object ): if hda_tool_output and hda_tool_output.from_work_dir: # Copy from working dir to HDA. # TODO: move instead of copy to save time? - source_file = os.path.join( os.path.abspath( job_wrapper.working_directory ), hda_tool_output.from_work_dir ) + source_file = os.path.join( job_working_directory, hda_tool_output.from_work_dir ) destination = job_wrapper.get_output_destination( output_paths[ dataset.dataset_id ] ) - if in_directory( source_file, job_wrapper.working_directory ): + if in_directory( source_file, job_working_directory ): output_pairs.append( ( source_file, destination ) ) log.debug( "Copying %s to %s as directed by from_work_dir" % ( source_file, destination ) ) else: @@ -196,7 +215,7 @@ class BaseJobRunner( object ): log.exception( "from_work_dir specified a location not in the working directory: %s, %s" % ( source_file, job_wrapper.working_directory ) ) return output_pairs - def _handle_metadata_externally(self, job_wrapper): + def _handle_metadata_externally( self, job_wrapper, resolve_requirements=False ): """ Set metadata externally. Used by the local and lwr job runners where this shouldn't be attached to command-line to execute. @@ -210,7 +229,19 @@ class BaseJobRunner( object ): tmp_dir=job_wrapper.working_directory, #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior kwds={ 'overwrite' : False } ) + if resolve_requirements: + dependency_shell_commands = self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands() + if dependency_shell_commands: + if isinstance( dependency_shell_commands, list ): + dependency_shell_commands = "&&".join( dependency_shell_commands ) + external_metadata_script = "%s&&%s" % ( dependency_shell_commands, external_metadata_script ) log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) ) + if resolve_requirements: + dependency_shell_commands = self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands() + if dependency_shell_commands: + if isinstance( dependency_shell_commands, list ): + dependency_shell_commands = "&&".join( dependency_shell_commands ) + external_metadata_script = "%s&&%s" % ( dependency_shell_commands, external_metadata_script ) external_metadata_proc = subprocess.Popen( args=external_metadata_script, shell=True, env=os.environ, @@ -229,6 +260,10 @@ class BaseJobRunner( object ): options.update(**kwds) return job_script(**options) + def _complete_terminal_job( self, ajs, **kwargs ): + if ajs.job_wrapper.get_state() != model.Job.states.DELETED: + self.work_queue.put( ( self.finish_job, ajs ) ) + class AsynchronousJobState( object ): """ @@ -290,8 +325,8 @@ class AsynchronousJobRunner( BaseJobRunner ): to the correct methods (queue, finish, cleanup) at appropriate times.. """ - def __init__( self, app, nworkers ): - super( AsynchronousJobRunner, self ).__init__( app, nworkers ) + def __init__( self, app, nworkers, **kwargs ): + super( AsynchronousJobRunner, self ).__init__( app, nworkers, **kwargs ) # 'watched' and 'queue' are both used to keep track of jobs to watch. # 'queue' is used to add new watched jobs, and can be called from # any thread (usually by the 'queue_job' method). 'watched' must only diff --git a/lib/galaxy/jobs/runners/cli_job/torque.py b/lib/galaxy/jobs/runners/cli_job/torque.py index 7171421cd71..a7d16cdc2f7 100644 --- a/lib/galaxy/jobs/runners/cli_job/torque.py +++ b/lib/galaxy/jobs/runners/cli_job/torque.py @@ -35,7 +35,8 @@ cd %s echo $? > %s """ -argmap = { 'Execution_Time' : '-a', +argmap = { 'destination' : '-q', + 'Execution_Time' : '-a', 'Account_Name' : '-A', 'Checkpoint' : '-c', 'Error_Path' : '-e', diff --git a/lib/galaxy/jobs/runners/condor.py b/lib/galaxy/jobs/runners/condor.py index 7d25d703969..4b0e9402402 100644 --- a/lib/galaxy/jobs/runners/condor.py +++ b/lib/galaxy/jobs/runners/condor.py @@ -57,7 +57,7 @@ class CondorJobRunner( AsynchronousJobRunner ): galaxy_id_tag = job_wrapper.get_id_tag() # get destination params - query_params = submission_params(**job_destination.params) + query_params = submission_params(prefix="", **job_destination.params) galaxy_slots = query_params.get('request_cpus', None) if galaxy_slots: galaxy_slots_statement = 'GALAXY_SLOTS="%s"; export GALAXY_SLOTS_CONFIGURED="1"' % galaxy_slots diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index 1cf9ff159e0..57b335ef64a 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -2,13 +2,13 @@ Job control via the DRMAA API. """ +import json +import logging import os +import string +import subprocess import sys import time -import string -import logging -import subprocess -import simplejson as json from galaxy import eggs from galaxy import model @@ -16,27 +16,12 @@ from galaxy.jobs import JobDestination from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner eggs.require( "drmaa" ) -# We foolishly named this file the same as the name exported by the drmaa -# library... 'import drmaa' imports itself. -drmaa = __import__( "drmaa" ) log = logging.getLogger( __name__ ) __all__ = [ 'DRMAAJobRunner' ] -drmaa_state = { - drmaa.JobState.UNDETERMINED: 'process status cannot be determined', - drmaa.JobState.QUEUED_ACTIVE: 'job is queued and active', - drmaa.JobState.SYSTEM_ON_HOLD: 'job is queued and in system hold', - drmaa.JobState.USER_ON_HOLD: 'job is queued and in user hold', - drmaa.JobState.USER_SYSTEM_ON_HOLD: 'job is queued and in user and system hold', - drmaa.JobState.RUNNING: 'job is running', - drmaa.JobState.SYSTEM_SUSPENDED: 'job is system suspended', - drmaa.JobState.USER_SUSPENDED: 'job is user suspended', - drmaa.JobState.DONE: 'job finished normally', - drmaa.JobState.FAILED: 'job finished, but failed', -} - +drmaa = None DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification', 'jobName', 'email', 'project' ] @@ -48,8 +33,50 @@ class DRMAAJobRunner( AsynchronousJobRunner ): """ runner_name = "DRMAARunner" - def __init__( self, app, nworkers ): + def __init__( self, app, nworkers, **kwargs ): """Start the job runner""" + + global drmaa + + runner_param_specs = dict( + drmaa_library_path = dict( map = str, default = os.environ.get( 'DRMAA_LIBRARY_PATH', None ) ), + invalidjobexception_state = dict( map = str, valid = lambda x: x in ( model.Job.states.OK, model.Job.states.ERROR ), default = model.Job.states.OK ), + invalidjobexception_retries = dict( map = int, valid = lambda x: int >= 0, default = 0 ), + internalexception_state = dict( map = str, valid = lambda x: x in ( model.Job.states.OK, model.Job.states.ERROR ), default = model.Job.states.OK ), + internalexception_retries = dict( map = int, valid = lambda x: int >= 0, default = 0 ) ) + + if 'runner_param_specs' not in kwargs: + kwargs[ 'runner_param_specs' ] = dict() + kwargs[ 'runner_param_specs' ].update( runner_param_specs ) + + super( DRMAAJobRunner, self ).__init__( app, nworkers, **kwargs ) + + # This allows multiple drmaa runners (although only one per handler) in the same job config file + if 'drmaa_library_path' in kwargs: + log.info( 'Overriding DRMAA_LIBRARY_PATH due to runner plugin parameter: %s', self.runner_params.drmaa_library_path ) + os.environ['DRMAA_LIBRARY_PATH'] = self.runner_params.drmaa_library_path + + # We foolishly named this file the same as the name exported by the drmaa + # library... 'import drmaa' imports itself. + drmaa = __import__( "drmaa" ) + + # Subclasses may need access to state constants + self.drmaa_job_states = drmaa.JobState + + # Descriptive state strings pulled from the drmaa lib itself + self.drmaa_job_state_strings = { + drmaa.JobState.UNDETERMINED: 'process status cannot be determined', + drmaa.JobState.QUEUED_ACTIVE: 'job is queued and active', + drmaa.JobState.SYSTEM_ON_HOLD: 'job is queued and in system hold', + drmaa.JobState.USER_ON_HOLD: 'job is queued and in user hold', + drmaa.JobState.USER_SYSTEM_ON_HOLD: 'job is queued and in user and system hold', + drmaa.JobState.RUNNING: 'job is running', + drmaa.JobState.SYSTEM_SUSPENDED: 'job is system suspended', + drmaa.JobState.USER_SUSPENDED: 'job is user suspended', + drmaa.JobState.DONE: 'job finished normally', + drmaa.JobState.FAILED: 'job finished, but failed', + } + self.ds = drmaa.Session() self.ds.initialize() @@ -58,7 +85,6 @@ class DRMAAJobRunner( AsynchronousJobRunner ): self.external_killJob_script = app.config.drmaa_external_killjob_script self.userid = None - super( DRMAAJobRunner, self ).__init__( app, nworkers ) self._init_monitor_thread() self._init_worker_threads() @@ -137,8 +163,10 @@ class DRMAAJobRunner( AsynchronousJobRunner ): job_wrapper.cleanup() return - log.debug( "(%s) submitting file %s" % ( galaxy_id_tag, ajs.job_file ) ) - log.debug( "(%s) command is: %s" % ( galaxy_id_tag, command_line ) ) + log.debug( "(%s) submitting file %s", galaxy_id_tag, ajs.job_file ) + log.debug( "(%s) command is: %s", galaxy_id_tag, command_line ) + if native_spec: + log.debug( "(%s) native specification is: %s", galaxy_id_tag, native_spec ) # runJob will raise if there's a submit problem if self.external_runJob_script is None: @@ -175,6 +203,20 @@ class DRMAAJobRunner( AsynchronousJobRunner ): # Add to our 'queue' of jobs to monitor self.monitor_queue.put( ajs ) + def _complete_terminal_job( self, ajs, drmaa_state, **kwargs ): + """ + Handle a job upon its termination in the DRM. This method is meant to + be overridden by subclasses to improve post-mortem and reporting of + failures. + """ + if drmaa_state == drmaa.JobState.FAILED: + if ajs.job_wrapper.get_state() != model.Job.states.DELETED: + ajs.stop_job = False + ajs.fail_message = "The cluster DRM system terminated this job" + self.work_queue.put( ( self.fail_job, ajs ) ) + elif drmaa_state == drmaa.JobState.DONE: + super( DRMAAJobRunner, self )._complete_terminal_job( ajs ) + def check_watched_items( self ): """ Called by the monitor thread to look at each watched job and deal @@ -188,16 +230,30 @@ class DRMAAJobRunner( AsynchronousJobRunner ): try: assert external_job_id not in ( None, 'None' ), '(%s/%s) Invalid job id' % ( galaxy_id_tag, external_job_id ) state = self.ds.jobStatus( external_job_id ) - # InternalException was reported to be necessary on some DRMs, but - # this could cause failures to be detected as completion! Please - # report if you experience problems with this. - except ( drmaa.InvalidJobException, drmaa.InternalException ), e: - # we should only get here if an orphaned job was put into the queue at app startup - log.info( "(%s/%s) job left DRM queue with following message: %s" % ( galaxy_id_tag, external_job_id, e ) ) - self.work_queue.put( ( self.finish_job, ajs ) ) + except ( drmaa.InternalException, drmaa.InvalidJobException ), e: + if isinstance( e , drmaa.InvalidJobException ): + ecn = "InvalidJobException".lower() + else: + ecn = "InternalException".lower() + retry_param = ecn.lower() + '_retries' + state_param = ecn.lower() + '_state' + retries = getattr( ajs, retry_param, 0 ) + if self.runner_params[ retry_param ] > 0: + if retries < self.runner_params[ retry_param ]: + # will retry check on next iteration + setattr( ajs, retry_param, retries + 1 ) + continue + if self.runner_params[ state_param ] == model.Job.states.OK: + log.info( "(%s/%s) job left DRM queue with following message: %s", galaxy_id_tag, external_job_id, e ) + self.work_queue.put( ( self.finish_job, ajs ) ) + elif self.runner_params[ state_param ] == model.Job.states.ERROR: + log.info( "(%s/%s) job check resulted in %s after %s tries: %s", galaxy_id_tag, external_job_id, ecn, retries, e ) + self.work_queue.put( ( self.fail_job, ajs ) ) + else: + raise Exception( "%s is set to an invalid value (%s), this should not be possible. See galaxy.jobs.drmaa.__init__()", state_param, self.runner_params[ state_param ] ) continue except drmaa.DrmCommunicationException, e: - log.warning( "(%s/%s) unable to communicate with DRM: %s" % ( galaxy_id_tag, external_job_id, e )) + log.warning( "(%s/%s) unable to communicate with DRM: %s", galaxy_id_tag, external_job_id, e ) new_watched.append( ajs ) continue except Exception, e: @@ -208,19 +264,12 @@ class DRMAAJobRunner( AsynchronousJobRunner ): self.work_queue.put( ( self.fail_job, ajs ) ) continue if state != old_state: - log.debug( "(%s/%s) state change: %s" % ( galaxy_id_tag, external_job_id, drmaa_state[state] ) ) + log.debug( "(%s/%s) state change: %s" % ( galaxy_id_tag, external_job_id, self.drmaa_job_state_strings[state] ) ) if state == drmaa.JobState.RUNNING and not ajs.running: ajs.running = True ajs.job_wrapper.change_state( model.Job.states.RUNNING ) - if state == drmaa.JobState.FAILED: - if ajs.job_wrapper.get_state() != model.Job.states.DELETED: - ajs.stop_job = False - ajs.fail_message = "The cluster DRM system terminated this job" - self.work_queue.put( ( self.fail_job, ajs ) ) - continue - if state == drmaa.JobState.DONE: - if ajs.job_wrapper.get_state() != model.Job.states.DELETED: - self.work_queue.put( ( self.finish_job, ajs ) ) + if state in ( drmaa.JobState.FAILED, drmaa.JobState.DONE ): + self._complete_terminal_job( ajs, drmaa_state = state ) continue ajs.old_state = state new_watched.append( ajs ) @@ -302,7 +351,15 @@ class DRMAAJobRunner( AsynchronousJobRunner ): The external script will be run with sudo, and will setuid() to the specified user. Effectively, will QSUB as a different user (then the one used by Galaxy). """ - p = subprocess.Popen([ '/usr/bin/sudo', '-E', self.external_runJob_script, str(username), jobtemplate_filename ], + script_parts = self.external_runJob_script.split() + script = script_parts[0] + command = [ '/usr/bin/sudo', '-E', script] + for script_argument in script_parts[1:]: + command.append(script_argument) + + command.extend( [ str(username), jobtemplate_filename ] ) + log.info("Running command %s" % command) + p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdoutdata, stderrdata) = p.communicate() exitcode = p.returncode diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py index 0fd8707c921..620a541aed5 100644 --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -18,11 +18,13 @@ log = logging.getLogger( __name__ ) __all__ = [ 'LocalJobRunner' ] + class LocalJobRunner( BaseJobRunner ): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ runner_name = "LocalRunner" + def __init__( self, app, nworkers ): """Start the job runner """ @@ -52,9 +54,9 @@ class LocalJobRunner( BaseJobRunner ): slots = job_wrapper.job_destination.params.get( "local_slots", None ) command_line = command_line.lstrip( " ;" ) if slots: - command_line = 'export GALAXY_SLOTS="%d"; export GALAXY_SLOTS_CONFIGURED="1"; %s' % ( int( slots ), command_line ) + command_line = 'GALAXY_SLOTS="%d"; export GALAXY_SLOTS; GALAXY_SLOTS_CONFIGURED="1"; export GALAXY_SLOTS_CONFIGURED; %s' % ( int( slots ), command_line ) else: - command_line = 'export GALAXY_SLOTS="1"; %s' % command_line + command_line = 'GALAXY_SLOTS="1"; export GALAXY_SLOTS; %s' % command_line return command_line def queue_job( self, job_wrapper ): @@ -74,13 +76,13 @@ class LocalJobRunner( BaseJobRunner ): log.debug( '(%s) executing: %s' % ( job_id, command_line ) ) stdout_file = tempfile.NamedTemporaryFile( suffix='_stdout', dir=job_wrapper.working_directory ) stderr_file = tempfile.NamedTemporaryFile( suffix='_stderr', dir=job_wrapper.working_directory ) - proc = subprocess.Popen( args = command_line, - shell = True, - cwd = job_wrapper.working_directory, - stdout = stdout_file, - stderr = stderr_file, - env = self._environ, - preexec_fn = os.setpgrp ) + proc = subprocess.Popen( args=command_line, + shell=True, + cwd=job_wrapper.working_directory, + stdout=stdout_file, + stderr=stderr_file, + env=self._environ, + preexec_fn=os.setpgrp ) job_wrapper.set_job_destination(job_wrapper.job_destination, proc.pid) job_wrapper.change_state( model.Job.states.RUNNING ) job_start = datetime.datetime.now() @@ -89,7 +91,7 @@ class LocalJobRunner( BaseJobRunner ): while proc.poll() is None: i += 1 if (i % 20) == 0: - msg = job_wrapper.check_limits(runtime = datetime.datetime.now() - job_start) + msg = job_wrapper.check_limits(runtime=datetime.datetime.now() - job_start) if msg is not None: job_wrapper.fail(msg) log.debug('(%s) Terminating process group' % job_id) @@ -110,7 +112,7 @@ class LocalJobRunner( BaseJobRunner ): job_wrapper.fail( "failure running job", exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) return - self._handle_metadata_externally( job_wrapper ) + self._handle_metadata_externally( job_wrapper, resolve_requirements=True ) # Finish the job! try: job_wrapper.finish( stdout, stderr, exit_code ) @@ -122,7 +124,7 @@ class LocalJobRunner( BaseJobRunner ): #if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished job_ext_output_metadata = job.get_external_output_metadata() if job_ext_output_metadata: - pid = job_ext_output_metadata[0].job_runner_external_pid #every JobExternalOutputMetadata has a pid set, we just need to take from one of them + pid = job_ext_output_metadata[0].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them else: pid = job.get_job_runner_external_id() if pid in [ None, '' ]: @@ -140,14 +142,14 @@ class LocalJobRunner( BaseJobRunner ): return # give up sleep( 2 ) if not self._check_pid( pid ): - log.debug( "stop_job(): %s: PID %d successfully killed with signal %d" %( job.get_id(), pid, sig ) ) + log.debug( "stop_job(): %s: PID %d successfully killed with signal %d" % ( job.get_id(), pid, sig ) ) return else: - log.warning( "stop_job(): %s: PID %d refuses to die after signaling TERM/KILL" %( job.get_id(), pid ) ) + log.warning( "stop_job(): %s: PID %d refuses to die after signaling TERM/KILL" % ( job.get_id(), pid ) ) def recover( self, job, job_wrapper ): # local jobs can't be recovered - job_wrapper.change_state( model.Job.states.ERROR, info = "This job was killed when Galaxy was restarted. Please retry the job." ) + job_wrapper.change_state( model.Job.states.ERROR, info="This job was killed when Galaxy was restarted. Please retry the job." ) def _check_pid( self, pid ): try: @@ -157,7 +159,7 @@ class LocalJobRunner( BaseJobRunner ): if e.errno == errno.ESRCH: log.debug( "_check_pid(): PID %d is dead" % pid ) else: - log.warning( "_check_pid(): Got errno %s when attempting to check PID %d: %s" %( errno.errorcode[e.errno], pid, e.strerror ) ) + log.warning( "_check_pid(): Got errno %s when attempting to check PID %d: %s" % ( errno.errorcode[e.errno], pid, e.strerror ) ) return False def _terminate( self, proc ): @@ -165,4 +167,4 @@ class LocalJobRunner( BaseJobRunner ): sleep( 1 ) if proc.poll() is None: os.killpg( proc.pid, 9 ) - return proc.wait() # reap + return proc.wait() # reap diff --git a/lib/galaxy/jobs/runners/lwr.py b/lib/galaxy/jobs/runners/lwr.py index 83b9402da8b..640d8d82da9 100644 --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -3,7 +3,9 @@ import logging from galaxy import model from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner from galaxy.jobs import JobDestination +from galaxy.jobs.command_factory import build_command from galaxy.util import string_as_bool_or_none +from galaxy.util.bunch import Bunch import errno from time import sleep @@ -12,11 +14,17 @@ import os from .lwr_client import ClientManager, url_to_destination_params from .lwr_client import finish_job as lwr_finish_job from .lwr_client import submit_job as lwr_submit_job +from .lwr_client import ClientJobDescription +from .lwr_client import LwrOutputs +from .lwr_client import GalaxyOutputs log = logging.getLogger( __name__ ) __all__ = [ 'LwrJobRunner' ] +NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE = "LWR misconfiguration - LWR client configured to set metadata remotely, but remote LWR isn't properly configured with a galaxy_home directory." +NO_REMOTE_DATATYPES_CONFIG = "LWR client is configured to use remote datatypes configuration when setting metadata externally, but LWR is not configured with this information. Defaulting to datatypes_conf.xml." + class LwrJobRunner( AsynchronousJobRunner ): """ @@ -54,40 +62,32 @@ class LwrJobRunner( AsynchronousJobRunner ): return job_state def queue_job(self, job_wrapper): - command_line = '' job_destination = job_wrapper.job_destination - try: - job_wrapper.prepare() - if hasattr(job_wrapper, 'prepare_input_files_cmds') and job_wrapper.prepare_input_files_cmds is not None: - for cmd in job_wrapper.prepare_input_files_cmds: # run the commands to stage the input files - #log.debug( 'executing: %s' % cmd ) - if 0 != os.system(cmd): - raise Exception('Error running file staging command: %s' % cmd) - job_wrapper.prepare_input_files_cmds = None # prevent them from being used in-line - command_line = self.build_command_line( job_wrapper, include_metadata=False, include_work_dir_outputs=False ) - except: - job_wrapper.fail( "failure preparing job", exception=True ) - log.exception("failure running job %d" % job_wrapper.job_id) - return + command_line, client, remote_job_config = self.__prepare_job( job_wrapper, job_destination ) - # If we were able to get a command line, run the job if not command_line: - job_wrapper.finish( '', '' ) return try: - client = self.get_client_from_wrapper(job_wrapper) - output_files = self.get_output_files(job_wrapper) - input_files = job_wrapper.get_input_fnames() - working_directory = job_wrapper.working_directory - tool = job_wrapper.tool - config_files = job_wrapper.extra_filenames - job_id = lwr_submit_job(client, tool, command_line, config_files, input_files, output_files, working_directory) + dependency_resolution = LwrJobRunner.__dependency_resolution( client ) + remote_dependency_resolution = dependency_resolution == "remote" + requirements = job_wrapper.tool.requirements if remote_dependency_resolution else [] + client_job_description = ClientJobDescription( + command_line=command_line, + output_files=self.get_output_files(job_wrapper), + input_files=job_wrapper.get_input_fnames(), + working_directory=job_wrapper.working_directory, + tool=job_wrapper.tool, + config_files=job_wrapper.extra_filenames, + requirements=requirements, + version_file=job_wrapper.get_version_string_path(), + ) + job_id = lwr_submit_job(client, client_job_description, remote_job_config) log.info("lwr job submitted with job_id %s" % job_id) job_wrapper.set_job_destination( job_destination, job_id ) job_wrapper.change_state( model.Job.states.QUEUED ) - except: + except Exception: job_wrapper.fail( "failure running job", exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) return @@ -100,6 +100,52 @@ class LwrJobRunner( AsynchronousJobRunner ): lwr_job_state.job_destination = job_destination self.monitor_job(lwr_job_state) + def __prepare_job(self, job_wrapper, job_destination): + """ Build command-line and LWR client for this job. """ + command_line = None + client = None + remote_job_config = None + try: + job_wrapper.prepare() + self.__prepare_input_files_locally(job_wrapper) + client = self.get_client_from_wrapper(job_wrapper) + tool = job_wrapper.tool + remote_job_config = client.setup(tool.id, tool.version) + remote_metadata = LwrJobRunner.__remote_metadata( client ) + remote_work_dir_copy = LwrJobRunner.__remote_work_dir_copy( client ) + dependency_resolution = LwrJobRunner.__dependency_resolution( client ) + metadata_kwds = self.__build_metadata_configuration(client, job_wrapper, remote_metadata, remote_job_config) + remote_command_params = dict( + working_directory=remote_job_config['working_directory'], + metadata_kwds=metadata_kwds, + dependency_resolution=dependency_resolution, + ) + command_line = build_command( + self, + job_wrapper=job_wrapper, + include_metadata=remote_metadata, + include_work_dir_outputs=remote_work_dir_copy, + remote_command_params=remote_command_params, + ) + except Exception: + job_wrapper.fail( "failure preparing job", exception=True ) + log.exception("failure running job %d" % job_wrapper.job_id) + + # If we were able to get a command line, run the job + if not command_line: + job_wrapper.finish( '', '' ) + + return command_line, client, remote_job_config + + def __prepare_input_files_locally(self, job_wrapper): + """Run task splitting commands locally.""" + prepare_input_files_cmds = getattr(job_wrapper, 'prepare_input_files_cmds', None) + if prepare_input_files_cmds is not None: + for cmd in prepare_input_files_cmds: # run the commands to stage the input files + if 0 != os.system(cmd): + raise Exception('Error running file staging command: %s' % cmd) + job_wrapper.prepare_input_files_cmds = None # prevent them from being used in-line + def get_output_files(self, job_wrapper): output_fnames = job_wrapper.get_output_fnames() return [ str( o ) for o in output_fnames ] @@ -108,7 +154,10 @@ class LwrJobRunner( AsynchronousJobRunner ): job_id = job_wrapper.job_id if hasattr(job_wrapper, 'task_id'): job_id = "%s_%s" % (job_id, job_wrapper.task_id) - return self.get_client( job_wrapper.job_destination.params, job_id ) + params = job_wrapper.job_destination.params.copy() + for key, value in params.iteritems(): + params[key] = model.User.expand_user_properties( job_wrapper.get_job().user, value ) + return self.get_client( params, job_id ) def get_client_from_state(self, job_state): job_destination_params = job_state.job_destination.params @@ -127,34 +176,47 @@ class LwrJobRunner( AsynchronousJobRunner ): run_results = client.raw_check_complete() stdout = run_results.get('stdout', '') stderr = run_results.get('stderr', '') - + exit_code = run_results.get('returncode', None) + lwr_outputs = LwrOutputs(run_results) # Use LWR client code to transfer/copy files back # and cleanup job if needed. completed_normally = \ job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] cleanup_job = self.app.config.cleanup_job - work_dir_outputs = self.get_work_dir_outputs( job_wrapper ) + remote_work_dir_copy = LwrJobRunner.__remote_work_dir_copy( client ) + if not remote_work_dir_copy: + work_dir_outputs = self.get_work_dir_outputs( job_wrapper ) + else: + # They have already been copied over to look like regular outputs remotely, + # no need to handle them differently here. + work_dir_outputs = [] output_files = self.get_output_files( job_wrapper ) + galaxy_outputs = GalaxyOutputs( + working_directory=job_wrapper.working_directory, + work_dir_outputs=work_dir_outputs, + output_files=output_files, + version_file=job_wrapper.get_version_string_path(), + ) finish_args = dict( client=client, - working_directory=job_wrapper.working_directory, job_completed_normally=completed_normally, cleanup_job=cleanup_job, - work_dir_outputs=work_dir_outputs, - output_files=output_files ) + galaxy_outputs=galaxy_outputs, + lwr_outputs=lwr_outputs ) failed = lwr_finish_job( **finish_args ) if failed: job_wrapper.fail("Failed to find or download one or more job outputs from remote server.", exception=True) - except: + except Exception: message = "Failed to communicate with remote job server." job_wrapper.fail( message, exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) return - self._handle_metadata_externally( job_wrapper ) + if not LwrJobRunner.__remote_metadata( client ): + self._handle_metadata_externally( job_wrapper, resolve_requirements=True ) # Finish the job try: - job_wrapper.finish( stdout, stderr ) - except: + job_wrapper.finish( stdout, stderr, exit_code ) + except Exception: log.exception("Job wrapper finish method failed") job_wrapper.fail("Unable to finish job", exception=True) @@ -222,3 +284,71 @@ class LwrJobRunner( AsynchronousJobRunner ): job_state.old_state = True job_state.running = state == model.Job.states.RUNNING self.monitor_queue.put( job_state ) + + @staticmethod + def __dependency_resolution( lwr_client ): + dependency_resolution = lwr_client.destination_params.get( "dependency_resolution", "local" ) + if dependency_resolution not in ["none", "local", "remote"]: + raise Exception("Unknown dependency_resolution value encountered %s" % dependency_resolution) + return dependency_resolution + + @staticmethod + def __remote_metadata( lwr_client ): + remote_metadata = string_as_bool_or_none( lwr_client.destination_params.get( "remote_metadata", False ) ) + return remote_metadata + + @staticmethod + def __remote_work_dir_copy( lwr_client ): + # Right now remote metadata handling assumes from_work_dir outputs + # have been copied over before it runs. So do that remotely. This is + # not the default though because adding it to the command line is not + # cross-platform (no cp on Windows) and its un-needed work outside + # the context of metadata settting (just as easy to download from + # either place.) + return LwrJobRunner.__remote_metadata( lwr_client ) + + @staticmethod + def __use_remote_datatypes_conf( lwr_client ): + """ When setting remote metadata, use integrated datatypes from this + Galaxy instance or use the datatypes config configured via the remote + LWR. + + Both options are broken in different ways for same reason - datatypes + may not match. One can push the local datatypes config to the remote + server - but there is no guarentee these datatypes will be defined + there. Alternatively, one can use the remote datatype config - but + there is no guarentee that it will contain all the datatypes available + to this Galaxy. + """ + use_remote_datatypes = string_as_bool_or_none( lwr_client.destination_params.get( "use_remote_datatypes", False ) ) + return use_remote_datatypes + + def __build_metadata_configuration(self, client, job_wrapper, remote_metadata, remote_job_config): + metadata_kwds = {} + if remote_metadata: + remote_system_properties = remote_job_config.get("system_properties", {}) + remote_galaxy_home = remote_system_properties.get("galaxy_home", None) + if not remote_galaxy_home: + raise Exception(NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE) + metadata_kwds['exec_dir'] = remote_galaxy_home + outputs_directory = remote_job_config['outputs_directory'] + configs_directory = remote_job_config['configs_directory'] + outputs = [Bunch(false_path=os.path.join(outputs_directory, os.path.basename(path)), real_path=path) for path in self.get_output_files(job_wrapper)] + metadata_kwds['output_fnames'] = outputs + metadata_kwds['config_root'] = remote_galaxy_home + default_config_file = os.path.join(remote_galaxy_home, 'universe_wsgi.ini') + metadata_kwds['config_file'] = remote_system_properties.get('galaxy_config_file', default_config_file) + metadata_kwds['dataset_files_path'] = remote_system_properties.get('galaxy_dataset_files_path', None) + if LwrJobRunner.__use_remote_datatypes_conf( client ): + remote_datatypes_config = remote_system_properties.get('galaxy_datatypes_config_file', None) + if not remote_datatypes_config: + log.warn(NO_REMOTE_DATATYPES_CONFIG) + remote_datatypes_config = os.path.join(remote_galaxy_home, 'datatypes_conf.xml') + metadata_kwds['datatypes_config'] = remote_datatypes_config + else: + integrates_datatypes_config = self.app.datatypes_registry.integrated_datatypes_configs + # Ensure this file gets pushed out to the remote config dir. + job_wrapper.extra_filenames.append(integrates_datatypes_config) + + metadata_kwds['datatypes_config'] = os.path.join(configs_directory, os.path.basename(integrates_datatypes_config)) + return metadata_kwds diff --git a/lib/galaxy/jobs/runners/lwr_client/__init__.py b/lib/galaxy/jobs/runners/lwr_client/__init__.py index fbdb0a7d5f9..b47503acaf5 100644 --- a/lib/galaxy/jobs/runners/lwr_client/__init__.py +++ b/lib/galaxy/jobs/runners/lwr_client/__init__.py @@ -6,9 +6,24 @@ This module contains logic for interfacing with an external LWR server. """ -from .stager import submit_job, finish_job +from .staging.down import finish_job +from .staging.up import submit_job +from .staging import ClientJobDescription +from .staging import LwrOutputs +from .staging import GalaxyOutputs from .client import OutputNotFoundException from .manager import ClientManager from .destination import url_to_destination_params +from .path_mapper import PathMapper -__all__ = [ClientManager, OutputNotFoundException, url_to_destination_params, finish_job, submit_job] +__all__ = [ + ClientManager, + OutputNotFoundException, + url_to_destination_params, + finish_job, + submit_job, + ClientJobDescription, + LwrOutputs, + GalaxyOutputs, + PathMapper, +] diff --git a/lib/galaxy/jobs/runners/lwr_client/action_mapper.py b/lib/galaxy/jobs/runners/lwr_client/action_mapper.py index 186b6aeea83..3903f515610 100644 --- a/lib/galaxy/jobs/runners/lwr_client/action_mapper.py +++ b/lib/galaxy/jobs/runners/lwr_client/action_mapper.py @@ -1,12 +1,49 @@ -from simplejson import load +from json import load from os.path import abspath -from fnmatch import fnmatch +from os.path import dirname +from os.path import join +from os.path import basename +from os.path import sep +import fnmatch from re import compile - +from re import escape +from galaxy.util.bunch import Bunch +from .util import directory_files +from .util import unique_path_prefix DEFAULT_MAPPED_ACTION = 'transfer' # Not really clear to me what this should be, exception? DEFAULT_PATH_MAPPER_TYPE = 'prefix' +# Poor man's enum. +path_type = Bunch( + # Galaxy input datasets and extra files. + INPUT="input", + # Galaxy config and param files. + CONFIG="config", + # Files from tool's tool_dir (for now just wrapper if available). + TOOL="tool", + # Input work dir files - e.g. metadata files, task-split input files, etc.. + WORKDIR="workdir", + # Galaxy output datasets in their final home. + OUTPUT="output", + # Galaxy from_work_dir output paths and other files (e.g. galaxy.json) + OUTPUT_WORKDIR="output_workdir", + # Other fixed tool parameter paths (likely coming from tool data, but not + # nessecarily). Not sure this is the best name... + UNSTRUCTURED="unstructured", +) + + +ACTION_DEFAULT_PATH_TYPES = [ + path_type.INPUT, + path_type.CONFIG, + path_type.TOOL, + path_type.WORKDIR, + path_type.OUTPUT, + path_type.OUTPUT_WORKDIR, +] +ALL_PATH_TYPES = ACTION_DEFAULT_PATH_TYPES + [path_type.UNSTRUCTURED] + class FileActionMapper(object): """ @@ -15,38 +52,62 @@ class FileActionMapper(object): >>> json_string = r'''{"paths": [ \ {"path": "/opt/galaxy", "action": "none"}, \ {"path": "/galaxy/data", "action": "transfer"}, \ - {"path": "/cool/bamfiles/**/*.bam", "action": "copy", "type": "glob"}, \ - {"path": ".*/dataset_\\\\d+.dat", "action": "copy", "type": "regex"} \ + {"path": "/cool/bamfiles/**/*.bam", "action": "copy", "match_type": "glob"}, \ + {"path": ".*/dataset_\\\\d+.dat", "action": "copy", "match_type": "regex"} \ ]}''' >>> from tempfile import NamedTemporaryFile >>> from os import unlink - >>> f = NamedTemporaryFile(delete=False) - >>> f.write(json_string) - >>> f.close() - >>> class MockClient(): - ... default_file_action = 'none' - ... action_config_path = f.name - ... - >>> mapper = FileActionMapper(MockClient()) - >>> unlink(f.name) + >>> def mapper_for(default_action, config_contents): + ... f = NamedTemporaryFile(delete=False) + ... f.write(config_contents.encode('UTF-8')) + ... f.close() + ... mock_client = Bunch(default_file_action=default_action, action_config_path=f.name) + ... mapper = FileActionMapper(mock_client) + ... unlink(f.name) + ... return mapper + >>> mapper = mapper_for(default_action='none', config_contents=json_string) >>> # Test first config line above, implicit path prefix mapper - >>> mapper.action('/opt/galaxy/tools/filters/catWrapper.py', 'input') - ('none',) + >>> action = mapper.action('/opt/galaxy/tools/filters/catWrapper.py', 'input') + >>> action.action_type == u'none' + True + >>> action.staging_needed + False >>> # Test another (2nd) mapper, this one with a different action - >>> mapper.action('/galaxy/data/files/000/dataset_1.dat', 'input') - ('transfer',) + >>> action = mapper.action('/galaxy/data/files/000/dataset_1.dat', 'input') + >>> action.action_type == u'transfer' + True + >>> action.staging_needed + True >>> # Always at least copy work_dir outputs. - >>> mapper.action('/opt/galaxy/database/working_directory/45.sh', 'work_dir') - ('copy',) + >>> action = mapper.action('/opt/galaxy/database/working_directory/45.sh', 'workdir') + >>> action.action_type == u'copy' + True + >>> action.staging_needed + True >>> # Test glob mapper (matching test) - >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam', 'input') - ('copy',) + >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam', 'input').action_type == u'copy' + True >>> # Test glob mapper (non-matching test) - >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam.bai', 'input') - ('none',) + >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam.bai', 'input').action_type == u'none' + True >>> # Regex mapper test. - >>> mapper.action('/old/galaxy/data/dataset_10245.dat', 'input') - ('copy',) + >>> mapper.action('/old/galaxy/data/dataset_10245.dat', 'input').action_type == u'copy' + True + >>> # Doesn't map unstructured paths by default + >>> mapper.action('/old/galaxy/data/dataset_10245.dat', 'unstructured').action_type == u'none' + True + >>> input_only_mapper = mapper_for(default_action="none", config_contents=r'''{"paths": [ \ + {"path": "/", "action": "transfer", "path_types": "input"} \ + ] }''') + >>> input_only_mapper.action('/dataset_1.dat', 'input').action_type == u'transfer' + True + >>> input_only_mapper.action('/dataset_1.dat', 'output').action_type == u'none' + True + >>> unstructured_mapper = mapper_for(default_action="none", config_contents=r'''{"paths": [ \ + {"path": "/", "action": "transfer", "path_types": "*any*"} \ + ] }''') + >>> unstructured_mapper.action('/old/galaxy/data/dataset_10245.dat', 'unstructured').action_type == u'transfer' + True """ def __init__(self, client): @@ -59,27 +120,92 @@ class FileActionMapper(object): def __load_action_config(self, path): config = load(open(path, 'rb')) for path_config in config.get('paths', []): - map_type = path_config.get('type', DEFAULT_PATH_MAPPER_TYPE) + map_type = path_config.get('match_type', DEFAULT_PATH_MAPPER_TYPE) self.mappers.append(mappers[map_type](path_config)) - def action(self, path, type): - action = self.default_action + def action(self, path, type, mapper=None): + action_type = self.default_action if type in ACTION_DEFAULT_PATH_TYPES else "none" + file_lister = DEFAULT_FILE_LISTER normalized_path = abspath(path) - for mapper in self.mappers: - if mapper.matches(normalized_path): - action = mapper.action - break - if type in ["work_dir", "output_task"] and action == "none": + if not mapper: + for query_mapper in self.mappers: + if query_mapper.matches(normalized_path, type): + mapper = query_mapper + break + if mapper: + action_type = mapper.action_type + file_lister = mapper.file_lister + if type in ["workdir", "output_workdir"] and action_type == "none": ## We are changing the working_directory relative to what ## Galaxy would use, these need to be copied over. - action = "copy" - return (action,) + action_type = "copy" + action_class = actions.get(action_type, None) + if action_class is None: + message_template = "Unknown action_type encountered %s while trying to map path %s" + message_args = (action_type, path) + raise Exception(message_template % message_args) + return action_class(path, file_lister=file_lister) + + def unstructured_mappers(self): + """ Return mappers that will map 'unstructured' files (i.e. go beyond + mapping inputs, outputs, and config files). + """ + return filter(lambda m: path_type.UNSTRUCTURED in m.path_types, self.mappers) + + +class BaseAction(object): + + def __init__(self, path, file_lister=None): + self.path = path + self.file_lister = file_lister or DEFAULT_FILE_LISTER + + def unstructured_map(self): + unstructured_map = self.file_lister.unstructured_map(self.path) + # To ensure uniqueness, prepend unique prefix to each name + prefix = unique_path_prefix(self.path) + for path, name in unstructured_map.iteritems(): + unstructured_map[path] = join(prefix, name) + return unstructured_map + + +class NoneAction(BaseAction): + """ This action indicates the corresponding path does not require any + additional action. This should indicate paths that are available both on + the LWR client (i.e. Galaxy server) and remote LWR server with the same + paths. """ + action_type = "none" + staging_needed = False + + +class TransferAction(BaseAction): + """ This actions indicates that the LWR client should initiate an HTTP + transfer of the corresponding path to the remote LWR server before + launching the job. """ + action_type = "transfer" + staging_needed = True + + +class CopyAction(BaseAction): + """ This action indicates that the LWR client should execute a file system + copy of the corresponding path to the LWR staging directory prior to + launching the corresponding job. """ + action_type = "copy" + staging_needed = True class BasePathMapper(object): def __init__(self, config): - self.action = config.get('action', DEFAULT_MAPPED_ACTION) + self.action_type = config.get('action', DEFAULT_MAPPED_ACTION) + path_types_str = config.get('path_types', "*defaults*") + path_types_str = path_types_str.replace("*defaults*", ",".join(ACTION_DEFAULT_PATH_TYPES)) + path_types_str = path_types_str.replace("*any*", ",".join(ALL_PATH_TYPES)) + self.path_types = path_types_str.split(",") + self.file_lister = FileLister(config) + + def matches(self, path, path_type): + path_type_matches = path_type in self.path_types + return path_type_matches and self._path_matches(path) class PrefixPathMapper(BasePathMapper): @@ -88,9 +214,13 @@ class PrefixPathMapper(BasePathMapper): super(PrefixPathMapper, self).__init__(config) self.prefix_path = abspath(config['path']) - def matches(self, path): + def _path_matches(self, path): return path.startswith(self.prefix_path) + def to_pattern(self): + pattern_str = "(%s%s[^\s,\"\']+)" % (escape(self.prefix_path), escape(sep)) + return compile(pattern_str) + class GlobPathMapper(BasePathMapper): @@ -98,8 +228,11 @@ class GlobPathMapper(BasePathMapper): super(GlobPathMapper, self).__init__(config) self.glob_path = config['path'] - def matches(self, path): - return fnmatch(path, self.glob_path) + def _path_matches(self, path): + return fnmatch.fnmatch(path, self.glob_path) + + def to_pattern(self): + return compile(fnmatch.translate(self.glob_path)) class RegexPathMapper(BasePathMapper): @@ -108,9 +241,32 @@ class RegexPathMapper(BasePathMapper): super(RegexPathMapper, self).__init__(config) self.pattern = compile(config['path']) - def matches(self, path): + def _path_matches(self, path): return self.pattern.match(path) is not None + def to_pattern(self): + return self.pattern + + +class FileLister(object): + + def __init__(self, config): + self.depth = int(config.get("depth", "0")) + + def unstructured_map(self, path): + depth = self.depth + if self.depth == 0: + return {path: basename(path)} + else: + while depth > 0: + path = dirname(path) + depth -= 1 + return dict([(join(path, f), f) for f in directory_files(path)]) + +DEFAULT_FILE_LISTER = FileLister(dict(depth=0)) + +ACTION_CLASSES = [NoneAction, TransferAction, CopyAction] +actions = dict([(clazz.action_type, clazz) for clazz in ACTION_CLASSES]) mappers = { 'prefix': PrefixPathMapper, @@ -119,4 +275,4 @@ mappers = { } -__all__ = [FileActionMapper] +__all__ = [FileActionMapper, path_type] diff --git a/lib/galaxy/jobs/runners/lwr_client/client.py b/lib/galaxy/jobs/runners/lwr_client/client.py index 324391900ef..b29126f0f3d 100644 --- a/lib/galaxy/jobs/runners/lwr_client/client.py +++ b/lib/galaxy/jobs/runners/lwr_client/client.py @@ -1,7 +1,6 @@ import os import shutil -import simplejson -from simplejson import dumps +from json import dumps, loads from time import sleep from .destination import submit_params @@ -16,7 +15,7 @@ class parseJson(object): def __call__(self, func): def replacement(*args, **kwargs): response = func(*args, **kwargs) - return simplejson.loads(response) + return loads(response) return replacement @@ -50,7 +49,7 @@ class OutputNotFoundException(Exception): return "No remote output found for path %s" % self.path -class Client(object): +class JobClient(object): """ Objects of this client class perform low-level communication with a remote LWR server. @@ -82,16 +81,16 @@ class Client(object): args = {"job_id": self.job_id, "name": name, "input_type": input_type} return self._raw_execute('input_path', args) - def put_file(self, path, input_type, name=None, contents=None, action='transfer'): + def put_file(self, path, input_type, name=None, contents=None, action_type='transfer'): if not name: name = os.path.basename(path) args = {"job_id": self.job_id, "name": name, "input_type": input_type} input_path = path if contents: input_path = None - if action == 'transfer': + if action_type == 'transfer': return self._upload_file(args, contents, input_path) - elif action == 'copy': + elif action_type == 'copy': lwr_path = self._raw_execute('input_path', args) self._copy(path, lwr_path) return {'path': lwr_path} @@ -105,11 +104,14 @@ class Client(object): ## path. Use old paths. input_type = args['input_type'] action = { - 'input': 'upload_input', - 'input_extra': 'upload_extra_input', + # For backward compatibility just target upload_input_extra for all + # inputs, it allows nested inputs. Want to do away with distinction + # inputs and extra inputs. + 'input': 'upload_extra_input', 'config': 'upload_config_file', - 'work_dir': 'upload_working_directory_file', - 'tool': 'upload_tool_file' + 'workdir': 'upload_working_directory_file', + 'tool': 'upload_tool_file', + 'unstructured': 'upload_unstructured_file', }[input_type] del args['input_type'] return action @@ -119,7 +121,23 @@ class Client(object): return self._raw_execute("get_output_type", {"name": name, "job_id": self.job_id}) - def fetch_output(self, path, working_directory, action='transfer'): + # Deprecated + def fetch_output_legacy(self, path, working_directory, action_type='transfer'): + # Needs to determine if output is task/working directory or standard. + name = os.path.basename(path) + + output_type = self._get_output_type(name) + if output_type == "none": + # Just make sure the file was created. + if not os.path.exists(path): + raise OutputNotFoundException(path) + return + elif output_type in ["task"]: + path = os.path.join(working_directory, name) + + self.__populate_output_path(name, path, output_type, action_type) + + def fetch_output(self, path, name=None, check_exists_remotely=False, action_type='transfer'): """ Download an output dataset from the remote server. @@ -130,59 +148,48 @@ class Client(object): working_directory : str Local working_directory for the job. """ - name = os.path.basename(path) - output_type = self._get_output_type(name) - if output_type == "none": - # Just make sure the file was created. - if not os.path.exists(path): - raise OutputNotFoundException(path) - return + if not name: + # Extra files will send in the path. + name = os.path.basename(path) - output_path = self.__output_path(path, name, working_directory, output_type) - self.__populate_output_path(name, output_path, output_type, action) + output_type = "direct" # Task/from_work_dir outputs now handled with fetch_work_dir_output + self.__populate_output_path(name, path, output_type, action_type) - def __populate_output_path(self, name, output_path, output_type, action): - if action == 'transfer': + def __populate_output_path(self, name, output_path, output_type, action_type): + self.__ensure_directory(output_path) + if action_type == 'transfer': self.__raw_download_output(name, self.job_id, output_type, output_path) - elif action == 'copy': + elif action_type == 'copy': lwr_path = self._output_path(name, self.job_id, output_type)['path'] self._copy(lwr_path, output_path) - def __output_path(self, path, name, working_directory, output_type): - """ - Preconditions: output_type is not 'none'. - """ - if output_type == "direct": - output_path = path - elif output_type == "task": - output_path = os.path.join(working_directory, name) - else: - raise Exception("Unknown output_type returned from LWR server %s" % output_type) - return output_path - - def fetch_work_dir_output(self, source, working_directory, output_path, action='transfer'): + def fetch_work_dir_output(self, name, working_directory, output_path, action_type='transfer'): """ Download an output dataset specified with from_work_dir from the remote server. **Parameters** - source : str + name : str Path in job's working_directory to find output in. working_directory : str Local working_directory for the job. output_path : str Full path to output dataset. """ - output = open(output_path, "wb") - name = os.path.basename(source) - if action == 'transfer': - self.__raw_download_output(name, self.job_id, "work_dir", output) - elif action == 'copy': + self.__ensure_directory(output_path) + if action_type == 'transfer': + self.__raw_download_output(name, self.job_id, "work_dir", output_path) + else: # Even if action is none - LWR has a different work_dir so this needs to be copied. lwr_path = self._output_path(name, self.job_id, 'work_dir')['path'] self._copy(lwr_path, output_path) + def __ensure_directory(self, output_path): + output_path_directory = os.path.dirname(output_path) + if not os.path.exists(output_path_directory): + os.makedirs(output_path_directory) + @parseJson() def _output_path(self, name, job_id, output_type): return self._raw_execute("output_path", @@ -199,7 +206,7 @@ class Client(object): } self._raw_execute("download_output", output_params, output_path=output_path) - def launch(self, command_line): + def launch(self, command_line, requirements=[]): """ Run or queue up the execution of the supplied `command_line` on the remote server. @@ -213,6 +220,8 @@ class Client(object): submit_params = self._submit_params if submit_params: launch_params['params'] = dumps(submit_params) + if requirements: + launch_params['requirements'] = dumps([requirement.to_dict() for requirement in requirements]) return self._raw_execute("launch", launch_params) def kill(self): @@ -285,13 +294,13 @@ class Client(object): shutil.copyfile(source, destination) -class InputCachingClient(Client): +class InputCachingJobClient(JobClient): """ Beta client that cache's staged files to prevent duplication. """ def __init__(self, destination_params, job_id, job_manager_interface, client_cacher): - super(InputCachingClient, self).__init__(destination_params, job_id, job_manager_interface) + super(InputCachingJobClient, self).__init__(destination_params, job_id, job_manager_interface) self.client_cacher = client_cacher @parseJson() @@ -326,3 +335,55 @@ class InputCachingClient(Client): @parseJson() def file_available(self, path): return self._raw_execute("file_available", {"path": path}) + + +class ObjectStoreClient(object): + + def __init__(self, lwr_interface): + self.lwr_interface = lwr_interface + + @parseJson() + def exists(self, **kwds): + return self._raw_execute("object_store_exists", args=self.__data(**kwds)) + + @parseJson() + def file_ready(self, **kwds): + return self._raw_execute("object_store_file_ready", args=self.__data(**kwds)) + + @parseJson() + def create(self, **kwds): + return self._raw_execute("object_store_create", args=self.__data(**kwds)) + + @parseJson() + def empty(self, **kwds): + return self._raw_execute("object_store_empty", args=self.__data(**kwds)) + + @parseJson() + def size(self, **kwds): + return self._raw_execute("object_store_size", args=self.__data(**kwds)) + + @parseJson() + def delete(self, **kwds): + return self._raw_execute("object_store_delete", args=self.__data(**kwds)) + + @parseJson() + def get_data(self, **kwds): + return self._raw_execute("object_store_get_data", args=self.__data(**kwds)) + + @parseJson() + def get_filename(self, **kwds): + return self._raw_execute("object_store_get_filename", args=self.__data(**kwds)) + + @parseJson() + def update_from_file(self, **kwds): + return self._raw_execute("object_store_update_from_file", args=self.__data(**kwds)) + + @parseJson() + def get_store_usage_percent(self): + return self._raw_execute("object_store_get_store_usage_percent", args={}) + + def __data(self, **kwds): + return kwds + + def _raw_execute(self, command, args={}): + return self.lwr_interface.execute(command, args, data=None, input_path=None, output_path=None) diff --git a/lib/galaxy/jobs/runners/lwr_client/destination.py b/lib/galaxy/jobs/runners/lwr_client/destination.py index 6054b6e98fd..e4db9b47682 100644 --- a/lib/galaxy/jobs/runners/lwr_client/destination.py +++ b/lib/galaxy/jobs/runners/lwr_client/destination.py @@ -51,9 +51,10 @@ def submit_params(destination_params): >>> destination_params = {"private_token": "12345", "submit_native_specification": "-q batch"} >>> result = submit_params(destination_params) - >>> result.items() - [('native_specification', '-q batch')] + >>> result + {'native_specification': '-q batch'} """ - return dict([(key[len(SUBMIT_PREFIX):], value) - for key, value in (destination_params or {}).iteritems() + destination_params = destination_params or {} + return dict([(key[len(SUBMIT_PREFIX):], destination_params[key]) + for key in destination_params if key.startswith(SUBMIT_PREFIX)]) diff --git a/lib/galaxy/jobs/runners/lwr_client/manager.py b/lib/galaxy/jobs/runners/lwr_client/manager.py index 2a1d8c0693e..f54b5b902e6 100644 --- a/lib/galaxy/jobs/runners/lwr_client/manager.py +++ b/lib/galaxy/jobs/runners/lwr_client/manager.py @@ -5,10 +5,22 @@ except ImportError: from queue import Queue from threading import Thread from os import getenv -from urllib import urlencode -from StringIO import StringIO +try: + from urllib import urlencode +except ImportError: + from urllib.parse import urlencode +try: + from StringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO +try: + from six import text_type +except ImportError: + from galaxy.util import unicodify as text_type -from .client import Client, InputCachingClient +from .client import JobClient +from .client import InputCachingJobClient +from .client import ObjectStoreClient from .transport import get_transport from .util import TransferEventManager from .destination import url_to_destination_params @@ -27,10 +39,10 @@ class ClientManager(object): """ def __init__(self, **kwds): if 'job_manager' in kwds: - self.job_manager_interface_class = LocalJobManagerInterface + self.job_manager_interface_class = LocalLwrInterface self.job_manager_interface_args = dict(job_manager=kwds['job_manager'], file_cache=kwds['file_cache']) else: - self.job_manager_interface_class = HttpJobManagerInterface + self.job_manager_interface_class = HttpLwrInterface transport_type = kwds.get('transport_type', None) transport = get_transport(transport_type) self.job_manager_interface_args = dict(transport=transport) @@ -40,11 +52,11 @@ class ClientManager(object): if cache: log.info("Setting LWR client class to caching variant.") self.client_cacher = ClientCacher(**kwds) - self.client_class = InputCachingClient + self.client_class = InputCachingJobClient self.extra_client_kwds = {"client_cacher": self.client_cacher} else: log.info("Setting LWR client class to standard, non-caching variant.") - self.client_class = Client + self.client_class = JobClient self.extra_client_kwds = {} def get_client(self, destination_params, job_id): @@ -55,11 +67,35 @@ class ClientManager(object): return self.client_class(destination_params, job_id, job_manager_interface, **self.extra_client_kwds) def __parse_destination_params(self, destination_params): - if isinstance(destination_params, str) or isinstance(destination_params, unicode): + try: + unicode_type = unicode + except NameError: + unicode_type = str + if isinstance(destination_params, str) or isinstance(destination_params, unicode_type): destination_params = url_to_destination_params(destination_params) return destination_params +class ObjectStoreClientManager(object): + + def __init__(self, **kwds): + if 'object_store' in kwds: + self.interface_class = LocalLwrInterface + self.interface_args = dict(object_store=kwds['object_store']) + else: + self.interface_class = HttpLwrInterface + transport_type = kwds.get('transport_type', None) + transport = get_transport(transport_type) + self.interface_args = dict(transport=transport) + self.extra_client_kwds = {} + + def get_client(self, client_params): + interface_class = self.interface_class + interface_args = dict(destination_params=client_params, **self.interface_args) + interface = interface_class(**interface_args) + return ObjectStoreClient(interface) + + class JobManagerInteface(object): """ Abstract base class describes how client communicates with remote job @@ -76,7 +112,7 @@ class JobManagerInteface(object): """ -class HttpJobManagerInterface(object): +class HttpLwrInterface(object): def __init__(self, destination_params, transport): self.transport = transport @@ -92,16 +128,18 @@ class HttpJobManagerInterface(object): def __build_url(self, command, args): if self.private_key: args["private_key"] = self.private_key - data = urlencode(args) + arg_bytes = dict([(k, text_type(args[k]).encode('utf-8')) for k in args]) + data = urlencode(arg_bytes) url = self.remote_host + command + "?" + data return url -class LocalJobManagerInterface(object): +class LocalLwrInterface(object): - def __init__(self, destination_params, job_manager, file_cache): + def __init__(self, destination_params, job_manager=None, file_cache=None, object_store=None): self.job_manager = job_manager self.file_cache = file_cache + self.object_store = object_store def __app_args(self): ## Arguments that would be specified from LwrApp if running @@ -109,10 +147,12 @@ class LocalJobManagerInterface(object): return { 'manager': self.job_manager, 'file_cache': self.file_cache, + 'object_store': self.object_store, 'ip': None } def execute(self, command, args={}, data=None, input_path=None, output_path=None): + # If data set, should be unicode (on Python 2) or str (on Python 3). from lwr import routes from lwr.framework import build_func_args controller = getattr(routes, command) @@ -129,9 +169,9 @@ class LocalJobManagerInterface(object): def __build_body(self, data, input_path): if data is not None: - return StringIO(data) + return BytesIO(data.encode('utf-8')) elif input_path is not None: - return open(input_path, 'r') + return open(input_path, 'rb') else: return None @@ -188,4 +228,4 @@ def _environ_default_int(variable, default="0"): int_val = int(val) return int_val -__all__ = [ClientManager, HttpJobManagerInterface] +__all__ = [ClientManager, ObjectStoreClientManager, HttpLwrInterface] diff --git a/lib/galaxy/jobs/runners/lwr_client/path_mapper.py b/lib/galaxy/jobs/runners/lwr_client/path_mapper.py new file mode 100644 index 00000000000..b664daed4e3 --- /dev/null +++ b/lib/galaxy/jobs/runners/lwr_client/path_mapper.py @@ -0,0 +1,97 @@ +import os.path +from .action_mapper import FileActionMapper +from .action_mapper import path_type +from .util import PathHelper + +from galaxy.util import in_directory + + +class PathMapper(object): + """ Ties together a FileActionMapper and remote job configuration returned + by the LWR setup method to pre-determine the location of files for staging + on the remote LWR server. + + This is not useful when rewrite_paths (as has traditionally been done with + the LWR) because when doing that the LWR determines the paths as files are + uploaded. When rewrite_paths is disabled however, the destination of files + needs to be determined prior to transfer so an object of this class can be + used. + """ + + def __init__( + self, + client, + remote_job_config, + local_working_directory, + action_mapper=None, + ): + self.local_working_directory = local_working_directory + if not action_mapper: + action_mapper = FileActionMapper(client) + self.action_mapper = action_mapper + self.input_directory = remote_job_config["inputs_directory"] + self.output_directory = remote_job_config["outputs_directory"] + self.working_directory = remote_job_config["working_directory"] + self.unstructured_files_directory = remote_job_config["unstructured_files_directory"] + self.config_directory = remote_job_config["configs_directory"] + separator = remote_job_config["system_properties"]["separator"] + self.path_helper = PathHelper(separator) + + def remote_output_path_rewrite(self, local_path): + output_type = path_type.OUTPUT + if in_directory(local_path, self.local_working_directory): + output_type = path_type.OUTPUT_WORKDIR + remote_path = self.__remote_path_rewrite(local_path, output_type) + return remote_path + + def remote_input_path_rewrite(self, local_path): + remote_path = self.__remote_path_rewrite(local_path, path_type.INPUT) + return remote_path + + def remote_version_path_rewrite(self, local_path): + remote_path = self.__remote_path_rewrite(local_path, path_type.OUTPUT, name="COMMAND_VERSION") + return remote_path + + def check_for_arbitrary_rewrite(self, local_path): + if not os.path.exists(local_path): + return None, [] + + path = str(local_path) # Use false_path if needed. + action = self.action_mapper.action(path, path_type.UNSTRUCTURED) + if not action.staging_needed: + return None, [] + unique_names = action.unstructured_map() + name = unique_names[path] + remote_path = self.path_helper.remote_join(self.unstructured_files_directory, name) + return remote_path, unique_names + + def __remote_path_rewrite(self, dataset_path, dataset_path_type, name=None): + """ Return remote path of this file (if staging is required) else None. + """ + path = str(dataset_path) # Use false_path if needed. + action = self.action_mapper.action(path, dataset_path_type) + remote_path_rewrite = None + if action.staging_needed: + if name is None: + name = os.path.basename(path) + remote_directory = self.__remote_directory(dataset_path_type) + remote_path_rewrite = self.path_helper.remote_join(remote_directory, name) + return remote_path_rewrite + + def __action(self, dataset_path, dataset_path_type): + path = str(dataset_path) # Use false_path if needed. + action = self.action_mapper.action(path, dataset_path_type) + return action + + def __remote_directory(self, dataset_path_type): + if dataset_path_type in [path_type.OUTPUT]: + return self.output_directory + elif dataset_path_type in [path_type.WORKDIR, path_type.OUTPUT_WORKDIR]: + return self.working_directory + elif dataset_path_type in [path_type.INPUT]: + return self.input_directory + else: + message = "PathMapper cannot handle path type %s" % dataset_path_type + raise Exception(message) + +__all__ = [PathMapper] diff --git a/lib/galaxy/jobs/runners/lwr_client/staging/__init__.py b/lib/galaxy/jobs/runners/lwr_client/staging/__init__.py new file mode 100644 index 00000000000..ea79630077b --- /dev/null +++ b/lib/galaxy/jobs/runners/lwr_client/staging/__init__.py @@ -0,0 +1,120 @@ +from os.path import basename +from os.path import join +from os.path import dirname +from os import sep + +from ..util import PathHelper + +COMMAND_VERSION_FILENAME = "COMMAND_VERSION" + + +class ClientJobDescription(object): + """ A description of how client views job - command_line, inputs, etc.. + + **Parameters** + + command_line : str + The local command line to execute, this will be rewritten for the remote server. + config_files : list + List of Galaxy 'configfile's produced for this job. These will be rewritten and sent to remote server. + input_files : list + List of input files used by job. These will be transferred and references rewritten. + output_files : list + List of output_files produced by job. + tool_dir : str + Directory containing tool to execute (if a wrapper is used, it will be transferred to remote server). + working_directory : str + Local path created by Galaxy for running this job. + requirements : list + List of requirements for tool execution. + version_file : str + Path to version file expected on the client server + arbitrary_files : dict() + Additional non-input, non-tool, non-config, non-working directory files + to transfer before staging job. This is most likely data indices but + can be anything. For now these are copied into staging working + directory but this will be reworked to find a better, more robust + location. + rewrite_paths : boolean + Indicates whether paths should be rewritten in job inputs (command_line + and config files) while staging files). + """ + + def __init__( + self, + tool, + command_line, + config_files, + input_files, + output_files, + working_directory, + requirements, + version_file=None, + arbitrary_files=None, + rewrite_paths=True, + ): + self.tool = tool + self.command_line = command_line + self.config_files = config_files + self.input_files = input_files + self.output_files = output_files + self.working_directory = working_directory + self.requirements = requirements + self.version_file = version_file + self.rewrite_paths = rewrite_paths + self.arbitrary_files = arbitrary_files or {} + + +class GalaxyOutputs(object): + """ Abstraction describing the output datasets EXPECTED by the Galaxy job + runner client. """ + + def __init__(self, working_directory, work_dir_outputs, output_files, version_file): + self.working_directory = working_directory + self.work_dir_outputs = work_dir_outputs + self.output_files = output_files + self.version_file = version_file + + +class LwrOutputs(object): + """ Abstraction describing the output files PRODUCED by the remote LWR + server. """ + + def __init__(self, complete_response): + # Default to None instead of [] to distinguish between empty contents and it not set + # by the LWR - older LWR instances will not set these in complete response. + self.working_directory_contents = complete_response.get("working_directory_contents", None) + self.output_directory_contents = complete_response.get("outputs_directory_contents", None) + # Older (pre-2014) LWR servers will not include separator in response, + #so this should only be used when reasoning about outputs in + # subdirectories which was not previously supported. + self.path_helper = PathHelper(complete_response.get("system_properties", {}).get("separator", sep)) + + def has_output_file(self, output_file): + if self.output_directory_contents is None: + # Legacy LWR doesn't report this, return None indicating unsure if + # output was generated. + return None + else: + return basename(output_file) in self.output_directory_contents + + def has_output_directory_listing(self): + return self.output_directory_contents is not None + + def output_extras(self, output_file): + """ + Returns dict mapping local path to remote name. + """ + if not self.has_output_directory_listing(): + # Fetching $output.extra_files_path is not supported with legacy + # LWR (pre-2014) severs. + return {} + + output_directory = dirname(output_file) + + def local_path(name): + return join(output_directory, self.path_helper.local_name(name)) + + files_directory = "%s_files%s" % (basename(output_file)[0:-len(".dat")], self.path_helper.separator) + names = filter(lambda o: o.startswith(files_directory), self.output_directory_contents) + return dict(map(lambda name: (local_path(name), name), names)) diff --git a/lib/galaxy/jobs/runners/lwr_client/staging/down.py b/lib/galaxy/jobs/runners/lwr_client/staging/down.py new file mode 100644 index 00000000000..21c8c8c79bd --- /dev/null +++ b/lib/galaxy/jobs/runners/lwr_client/staging/down.py @@ -0,0 +1,127 @@ +from os.path import join +from os.path import relpath +from re import compile +from contextlib import contextmanager + +from ..staging import COMMAND_VERSION_FILENAME +from ..action_mapper import FileActionMapper + + +from logging import getLogger +log = getLogger(__name__) + +# All output files marked with from_work_dir attributes will copied or downloaded +# this pattern picks up attiditional files to copy back - such as those +# associated with multiple outputs and metadata configuration. Set to .* to just +# copy everything +COPY_FROM_WORKING_DIRECTORY_PATTERN = compile(r"primary_.*|galaxy.json|metadata_.*|dataset_\d+\.dat|dataset_\d+_files.+") + + +def finish_job(client, cleanup_job, job_completed_normally, galaxy_outputs, lwr_outputs): + """ Responsible for downloading results from remote server and cleaning up + LWR staging directory (if needed.) + """ + download_failure_exceptions = [] + if job_completed_normally: + downloader = ResultsDownloader(client, galaxy_outputs, lwr_outputs) + download_failure_exceptions = downloader.download() + return __clean(download_failure_exceptions, cleanup_job, client) + + +class ResultsDownloader(object): + + def __init__(self, client, galaxy_outputs, lwr_outputs): + self.client = client + self.galaxy_outputs = galaxy_outputs + self.lwr_outputs = lwr_outputs + self.action_mapper = FileActionMapper(client) + self.downloaded_working_directory_files = [] + self.exception_tracker = DownloadExceptionTracker() + self.output_files = galaxy_outputs.output_files + self.working_directory_contents = lwr_outputs.working_directory_contents or [] + + def download(self): + self.__download_working_directory_outputs() + self.__download_outputs() + self.__download_version_file() + self.__download_other_working_directory_files() + return self.exception_tracker.download_failure_exceptions + + def __download_working_directory_outputs(self): + working_directory = self.galaxy_outputs.working_directory + # Fetch explicit working directory outputs. + for source_file, output_file in self.galaxy_outputs.work_dir_outputs: + name = relpath(source_file, working_directory) + remote_name = self.lwr_outputs.path_helper.remote_name(name) + with self.exception_tracker(): + action = self.action_mapper.action(output_file, 'output_workdir') + self.client.fetch_work_dir_output(remote_name, working_directory, output_file, action_type=action.action_type) + self.downloaded_working_directory_files.append(remote_name) + # Remove from full output_files list so don't try to download directly. + self.output_files.remove(output_file) + + def __download_outputs(self): + # Legacy LWR not returning list of files, iterate over the list of + # expected outputs for tool. + for output_file in self.output_files: + # Fetch ouptut directly... + with self.exception_tracker(): + action = self.action_mapper.action(output_file, 'output') + output_generated = self.lwr_outputs.has_output_file(output_file) + working_directory = self.galaxy_outputs.working_directory + if output_generated is None: + self.client.fetch_output_legacy(output_file, working_directory, action_type=action.action_type) + elif output_generated: + self.client.fetch_output(output_file, action_type=action.action_type) + + for local_path, remote_name in self.lwr_outputs.output_extras(output_file).iteritems(): + with self.exception_tracker(): + action = self.action_mapper.action(local_path, 'output') + self.client.fetch_output(path=local_path, name=remote_name, action_type=action.action_type) + # else not output generated, do not attempt download. + + def __download_version_file(self): + version_file = self.galaxy_outputs.version_file + # output_directory_contents may be none for legacy LWR servers. + lwr_output_directory_contents = (self.lwr_outputs.output_directory_contents or []) + if version_file and COMMAND_VERSION_FILENAME in lwr_output_directory_contents: + action = self.action_mapper.action(version_file, 'output') + self.client.fetch_output(path=version_file, name=COMMAND_VERSION_FILENAME, action_type=action.action_type) + + def __download_other_working_directory_files(self): + working_directory = self.galaxy_outputs.working_directory + # Fetch remaining working directory outputs of interest. + for name in self.working_directory_contents: + if name in self.downloaded_working_directory_files: + continue + if COPY_FROM_WORKING_DIRECTORY_PATTERN.match(name): + with self.exception_tracker(): + output_file = join(working_directory, self.lwr_outputs.path_helper.local_name(name)) + action = self.action_mapper.action(output_file, 'output_workdir') + self.client.fetch_work_dir_output(name, working_directory, output_file, action_type=action.action_type) + self.downloaded_working_directory_files.append(name) + + +class DownloadExceptionTracker(object): + + def __init__(self): + self.download_failure_exceptions = [] + + @contextmanager + def __call__(self): + try: + yield + except Exception as e: + self.download_failure_exceptions.append(e) + + +def __clean(download_failure_exceptions, cleanup_job, client): + failed = (len(download_failure_exceptions) > 0) + if (not failed and cleanup_job != "never") or cleanup_job == "always": + try: + client.clean() + except: + log.warn("Failed to cleanup remote LWR job") + return failed + +__all__ = [finish_job] diff --git a/lib/galaxy/jobs/runners/lwr_client/stager.py b/lib/galaxy/jobs/runners/lwr_client/staging/up.py similarity index 51% rename from lib/galaxy/jobs/runners/lwr_client/stager.py rename to lib/galaxy/jobs/runners/lwr_client/staging/up.py index 2ceb30118f6..b80733bdda1 100644 --- a/lib/galaxy/jobs/runners/lwr_client/stager.py +++ b/lib/galaxy/jobs/runners/lwr_client/staging/up.py @@ -1,13 +1,215 @@ from os.path import abspath, basename, join, exists +from os.path import dirname +from os.path import relpath from os import listdir, sep from re import findall +from io import open -from .action_mapper import FileActionMapper +from ..staging import COMMAND_VERSION_FILENAME +from ..action_mapper import FileActionMapper +from ..action_mapper import path_type +from ..util import PathHelper +from ..util import directory_files from logging import getLogger log = getLogger(__name__) +def submit_job(client, client_job_description, job_config=None): + """ + """ + file_stager = FileStager(client, client_job_description, job_config) + rebuilt_command_line = file_stager.get_command_line() + job_id = file_stager.job_id + client.launch(rebuilt_command_line, requirements=client_job_description.requirements) + return job_id + + +class FileStager(object): + """ + Objects of the FileStager class interact with an LWR client object to + stage the files required to run jobs on a remote LWR server. + + **Parameters** + + client : JobClient + LWR client object. + client_job_description : client_job_description + Description of client view of job to stage and execute remotely. + """ + + def __init__(self, client, client_job_description, job_config): + """ + """ + self.client = client + self.command_line = client_job_description.command_line + self.config_files = client_job_description.config_files + self.input_files = client_job_description.input_files + self.output_files = client_job_description.output_files + self.tool_id = client_job_description.tool.id + self.tool_version = client_job_description.tool.version + self.tool_dir = abspath(client_job_description.tool.tool_dir) + self.working_directory = client_job_description.working_directory + self.version_file = client_job_description.version_file + self.arbitrary_files = client_job_description.arbitrary_files + self.rewrite_paths = client_job_description.rewrite_paths + + # Setup job inputs, these will need to be rewritten before + # shipping off to remote LWR server. + self.job_inputs = JobInputs(self.command_line, self.config_files) + + self.action_mapper = FileActionMapper(client) + self.transfer_tracker = TransferTracker(client, self.action_mapper, self.job_inputs, rewrite_paths=self.rewrite_paths) + + self.__handle_setup(job_config) + self.__initialize_referenced_tool_files() + if self.rewrite_paths: + self.__initialize_referenced_arbitrary_files() + + self.__upload_tool_files() + self.__upload_input_files() + self.__upload_working_directory_files() + self.__upload_arbitrary_files() + + if self.rewrite_paths: + self.__initialize_output_file_renames() + self.__initialize_task_output_file_renames() + self.__initialize_config_file_renames() + self.__initialize_version_file_rename() + + self.__handle_rewrites() + + self.__upload_rewritten_config_files() + + def __handle_setup(self, job_config): + if not job_config: + job_config = self.client.setup(self.tool_id, self.tool_version) + + self.new_working_directory = job_config['working_directory'] + self.new_outputs_directory = job_config['outputs_directory'] + # Default configs_directory to match remote working_directory to mimic + # behavior of older LWR servers. + self.new_configs_directory = job_config.get('configs_directory', self.new_working_directory) + self.remote_separator = self.__parse_remote_separator(job_config) + self.path_helper = PathHelper(self.remote_separator) + # If remote LWR server assigned job id, use that otherwise + # just use local job_id assigned. + galaxy_job_id = self.client.job_id + self.job_id = job_config.get('job_id', galaxy_job_id) + if self.job_id != galaxy_job_id: + # Remote LWR server assigned an id different than the + # Galaxy job id, update client to reflect this. + self.client.job_id = self.job_id + + def __parse_remote_separator(self, job_config): + separator = job_config.get("system_properties", {}).get("separator", None) + if not separator: # Legacy LWR + separator = job_config["path_separator"] # Poorly named + return separator + + def __initialize_referenced_tool_files(self): + self.referenced_tool_files = self.job_inputs.find_referenced_subfiles(self.tool_dir) + + def __initialize_referenced_arbitrary_files(self): + referenced_arbitrary_path_mappers = dict() + for mapper in self.action_mapper.unstructured_mappers(): + mapper_pattern = mapper.to_pattern() + # TODO: Make more sophisticated, allow parent directories, + # grabbing sibbling files based on patterns, etc... + paths = self.job_inputs.find_pattern_references(mapper_pattern) + for path in paths: + if path not in referenced_arbitrary_path_mappers: + referenced_arbitrary_path_mappers[path] = mapper + for path, mapper in referenced_arbitrary_path_mappers.iteritems(): + action = self.action_mapper.action(path, path_type.UNSTRUCTURED, mapper) + unstructured_map = action.unstructured_map() + self.arbitrary_files.update(unstructured_map) + + def __upload_tool_files(self): + for referenced_tool_file in self.referenced_tool_files: + self.transfer_tracker.handle_transfer(referenced_tool_file, path_type.TOOL) + + def __upload_arbitrary_files(self): + for path, name in self.arbitrary_files.iteritems(): + self.transfer_tracker.handle_transfer(path, path_type.UNSTRUCTURED, name=name) + + def __upload_input_files(self): + for input_file in self.input_files: + self.__upload_input_file(input_file) + self.__upload_input_extra_files(input_file) + + def __upload_input_file(self, input_file): + if self.__stage_input(input_file): + if exists(input_file): + self.transfer_tracker.handle_transfer(input_file, path_type.INPUT) + else: + message = "LWR: __upload_input_file called on empty or missing dataset." + \ + " So such file: [%s]" % input_file + log.debug(message) + + def __upload_input_extra_files(self, input_file): + files_path = "%s_files" % input_file[0:-len(".dat")] + if exists(files_path) and self.__stage_input(files_path): + for extra_file_name in directory_files(files_path): + extra_file_path = join(files_path, extra_file_name) + remote_name = self.path_helper.remote_name(relpath(extra_file_path, dirname(files_path))) + self.transfer_tracker.handle_transfer(extra_file_path, path_type.INPUT, name=remote_name) + + def __upload_working_directory_files(self): + # Task manager stages files into working directory, these need to be + # uploaded if present. + working_directory_files = listdir(self.working_directory) if exists(self.working_directory) else [] + for working_directory_file in working_directory_files: + path = join(self.working_directory, working_directory_file) + self.transfer_tracker.handle_transfer(path, 'workdir') + + def __initialize_version_file_rename(self): + version_file = self.version_file + if version_file: + remote_path = self.path_helper.remote_join(self.new_outputs_directory, COMMAND_VERSION_FILENAME) + self.transfer_tracker.register_rewrite(version_file, remote_path, path_type.OUTPUT) + + def __initialize_output_file_renames(self): + for output_file in self.output_files: + remote_path = self.path_helper.remote_join(self.new_outputs_directory, basename(output_file)) + self.transfer_tracker.register_rewrite(output_file, remote_path, path_type.OUTPUT) + + def __initialize_task_output_file_renames(self): + for output_file in self.output_files: + name = basename(output_file) + task_file = join(self.working_directory, name) + remote_path = self.path_helper.remote_join(self.new_working_directory, name) + self.transfer_tracker.register_rewrite(task_file, remote_path, path_type.OUTPUT_WORKDIR) + + def __initialize_config_file_renames(self): + for config_file in self.config_files: + remote_path = self.path_helper.remote_join(self.new_configs_directory, basename(config_file)) + self.transfer_tracker.register_rewrite(config_file, remote_path, path_type.CONFIG) + + def __handle_rewrites(self): + """ + For each file that has been transferred and renamed, updated + command_line and configfiles to reflect that rewrite. + """ + self.transfer_tracker.rewrite_input_paths() + + def __upload_rewritten_config_files(self): + for config_file, new_config_contents in self.job_inputs.config_files.items(): + self.client.put_file(config_file, input_type='config', contents=new_config_contents) + + def get_command_line(self): + """ + Returns the rewritten version of the command line to execute suitable + for remote host. + """ + return self.job_inputs.command_line + + def __stage_input(self, file_path): + # If we have disabled path rewriting, just assume everything needs to be transferred, + # else check to ensure the file is referenced before transferring it. + return (not self.rewrite_paths) or self.job_inputs.path_referenced(file_path) + + class JobInputs(object): """ Abstractions over dynamic inputs created for a given job (namely the command to @@ -24,22 +226,24 @@ class JobInputs(object): >>> import tempfile >>> tf = tempfile.NamedTemporaryFile() >>> def setup_inputs(tf): - ... open(tf.name, "w").write("world /path/to/input the rest") - ... inputs = JobInputs("hello /path/to/input", [tf.name]) + ... open(tf.name, "w").write(u"world /path/to/input the rest") + ... inputs = JobInputs(u"hello /path/to/input", [tf.name]) ... return inputs >>> inputs = setup_inputs(tf) - >>> inputs.rewrite_paths("/path/to/input", 'C:\\input') - >>> inputs.rewritten_command_line - 'hello C:\\\\input' - >>> inputs.rewritten_config_files[tf.name] - 'world C:\\\\input the rest' + >>> inputs.rewrite_paths(u"/path/to/input", u'C:\\input') + >>> inputs.command_line == u'hello C:\\\\input' + True + >>> inputs.config_files[tf.name] == u'world C:\\\\input the rest' + True >>> tf.close() >>> tf = tempfile.NamedTemporaryFile() >>> inputs = setup_inputs(tf) - >>> inputs.find_referenced_subfiles('/path/to') - ['/path/to/input'] + >>> inputs.find_referenced_subfiles('/path/to') == [u'/path/to/input'] + True >>> inputs.path_referenced('/path/to') True + >>> inputs.path_referenced(u'/path/to') + True >>> inputs.path_referenced('/path/to/input') True >>> inputs.path_referenced('/path/to/notinput') @@ -48,11 +252,17 @@ class JobInputs(object): """ def __init__(self, command_line, config_files): - self.rewritten_command_line = command_line - self.rewritten_config_files = {} + self.command_line = command_line + self.config_files = {} for config_file in config_files or []: config_contents = _read(config_file) - self.rewritten_config_files[config_file] = config_contents + self.config_files[config_file] = config_contents + + def find_pattern_references(self, pattern): + referenced_files = set() + for input_contents in self.__items(): + referenced_files.update(findall(pattern, input_contents)) + return list(referenced_files) def find_referenced_subfiles(self, directory): """ @@ -67,10 +277,7 @@ class JobInputs(object): """ pattern = r"(%s%s\S+)" % (directory, sep) - referenced_files = set() - for input_contents in self.__items(): - referenced_files.update(findall(pattern, input_contents)) - return list(referenced_files) + return self.find_pattern_references(pattern) def path_referenced(self, path): pattern = r"%s" % path @@ -89,50 +296,54 @@ class JobInputs(object): self.__rewrite_config_files(local_path, remote_path) def __rewrite_command_line(self, local_path, remote_path): - self.rewritten_command_line = self.rewritten_command_line.replace(local_path, remote_path) + self.command_line = self.command_line.replace(local_path, remote_path) def __rewrite_config_files(self, local_path, remote_path): - for config_file, rewritten_contents in self.rewritten_config_files.iteritems(): - self.rewritten_config_files[config_file] = rewritten_contents.replace(local_path, remote_path) + for config_file, contents in self.config_files.items(): + self.config_files[config_file] = contents.replace(local_path, remote_path) def __items(self): - items = [self.rewritten_command_line] - items.extend(self.rewritten_config_files.values()) + items = [self.command_line] + items.extend(self.config_files.values()) return items class TransferTracker(object): - def __init__(self, client, job_inputs): + def __init__(self, client, action_mapper, job_inputs, rewrite_paths): self.client = client - self.action_mapper = FileActionMapper(client) + self.action_mapper = action_mapper + self.job_inputs = job_inputs + self.rewrite_paths = rewrite_paths self.file_renames = {} def handle_transfer(self, path, type, name=None, contents=None): if contents: # If contents loaded in memory, no need to write out file and copy, # just transfer. - action = ('transfer', ) + action_type = 'transfer' else: if not exists(path): message = "handle_tranfer called on non-existent file - [%s]" % path log.warn(message) raise Exception(message) - action = self.__action(path, type) + action_type = self.__action(path, type).action_type - if action[0] in ['transfer', 'copy']: + if action_type in ['transfer', 'copy']: response = self.client.put_file(path, type, name=name, contents=contents) - self.register_rewrite(path, response['path'], type, force=True) - elif action[0] == 'none': + register = self.rewrite_paths or type == 'tool' # Even if inputs not rewritten, tool must be. + if register: + self.register_rewrite(path, response['path'], type, force=True) + elif action_type == 'none': # No action for this file. pass else: - raise Exception("Unknown action type (%s) encountered for path (%s)" % (action[0], path)) + raise Exception("Unknown action type (%s) encountered for path (%s)" % (action_type, path)) def register_rewrite(self, local_path, remote_path, type, force=False): action = self.__action(local_path, type) - if action[0] in ['transfer', 'copy'] or force: + if action.action_type in ['transfer', 'copy'] or force: self.file_renames[local_path] = remote_path def rewrite_input_paths(self): @@ -140,212 +351,23 @@ class TransferTracker(object): For each file that has been transferred and renamed, updated command_line and configfiles to reflect that rewrite. """ - for local_path, remote_path in self.file_renames.iteritems(): + for local_path, remote_path in self.file_renames.items(): self.job_inputs.rewrite_paths(local_path, remote_path) def __action(self, path, type): return self.action_mapper.action(path, type) -class FileStager(object): - """ - Objects of the FileStager class interact with an LWR client object to - stage the files required to run jobs on a remote LWR server. - - **Parameters** - - client : Client - LWR client object. - command_line : str - The local command line to execute, this will be rewritten for the remote server. - config_files : list - List of Galaxy 'configfile's produced for this job. These will be rewritten and sent to remote server. - input_files : list - List of input files used by job. These will be transferred and references rewritten. - output_files : list - List of output_files produced by job. - tool_dir : str - Directory containing tool to execute (if a wrapper is used, it will be transferred to remote server). - working_directory : str - Local path created by Galaxy for running this job. - - """ - - def __init__(self, client, tool, command_line, config_files, input_files, output_files, working_directory): - """ - """ - self.client = client - self.command_line = command_line - self.config_files = config_files - self.input_files = input_files - self.output_files = output_files - self.tool_id = tool.id - self.tool_version = tool.version - self.tool_dir = abspath(tool.tool_dir) - self.working_directory = working_directory - - # Setup job inputs, these will need to be rewritten before - # shipping off to remote LWR server. - self.job_inputs = JobInputs(self.command_line, self.config_files) - - self.transfer_tracker = TransferTracker(client, self.job_inputs) - - self.__handle_setup() - self.__initialize_referenced_tool_files() - self.__upload_tool_files() - self.__upload_input_files() - self.__upload_working_directory_files() - self.__initialize_output_file_renames() - self.__initialize_task_output_file_renames() - self.__initialize_config_file_renames() - self.__handle_rewrites() - self.__upload_rewritten_config_files() - - def __handle_setup(self): - job_config = self.client.setup(self.tool_id, self.tool_version) - - self.new_working_directory = job_config['working_directory'] - self.new_outputs_directory = job_config['outputs_directory'] - # Default configs_directory to match remote working_directory to mimic - # behavior of older LWR servers. - self.new_configs_drectory = job_config.get('configs_directory', self.new_working_directory) - self.remote_path_separator = job_config['path_separator'] - # If remote LWR server assigned job id, use that otherwise - # just use local job_id assigned. - galaxy_job_id = self.client.job_id - self.job_id = job_config.get('job_id', galaxy_job_id) - if self.job_id != galaxy_job_id: - # Remote LWR server assigned an id different than the - # Galaxy job id, update client to reflect this. - self.client.job_id = self.job_id - - def __initialize_referenced_tool_files(self): - self.referenced_tool_files = self.job_inputs.find_referenced_subfiles(self.tool_dir) - - def __upload_tool_files(self): - for referenced_tool_file in self.referenced_tool_files: - self.transfer_tracker.handle_transfer(referenced_tool_file, 'tool') - - def __upload_input_files(self): - for input_file in self.input_files: - self.__upload_input_file(input_file) - self.__upload_input_extra_files(input_file) - - def __upload_input_file(self, input_file): - if self.job_inputs.path_referenced(input_file): - if exists(input_file): - self.transfer_tracker.handle_transfer(input_file, 'input') - else: - message = "LWR: __upload_input_file called on empty or missing dataset." + \ - " So such file: [%s]" % input_file - log.debug(message) - - def __upload_input_extra_files(self, input_file): - # TODO: Determine if this is object store safe and what needs to be - # done if it is not. - files_path = "%s_files" % input_file[0:-len(".dat")] - if exists(files_path) and self.job_inputs.path_referenced(files_path): - for extra_file in listdir(files_path): - extra_file_path = join(files_path, extra_file) - relative_path = basename(files_path) - extra_file_relative_path = join(relative_path, extra_file) - self.transfer_tracker.handle_transfer(extra_file_path, 'input_extra', name=extra_file_relative_path) - - def __upload_working_directory_files(self): - # Task manager stages files into working directory, these need to be - # uploaded if present. - for working_directory_file in listdir(self.working_directory): - path = join(self.working_directory, working_directory_file) - self.transfer_tracker.handle_transfer(path, 'work_dir') - - def __initialize_output_file_renames(self): - for output_file in self.output_files: - remote_path = r'%s%s%s' % (self.new_outputs_directory, self.remote_path_separator, basename(output_file)) - self.transfer_tracker.register_rewrite(output_file, remote_path, 'output') - - def __initialize_task_output_file_renames(self): - for output_file in self.output_files: - name = basename(output_file) - task_file = join(self.working_directory, name) - remote_path = r'%s%s%s' % (self.new_working_directory, self.remote_path_separator, name) - self.transfer_tracker.register_rewrite(task_file, remote_path, 'output_task') - - def __initialize_config_file_renames(self): - for config_file in self.config_files: - remote_path = r'%s%s%s' % (self.new_configs_drectory, self.remote_path_separator, basename(config_file)) - self.transfer_tracker.register_rewrite(config_file, remote_path, 'config') - - def __handle_rewrites(self): - """ - For each file that has been transferred and renamed, updated - command_line and configfiles to reflect that rewrite. - """ - self.transfer_tracker.rewrite_input_paths() - - def __upload_rewritten_config_files(self): - for config_file, new_config_contents in self.job_inputs.rewritten_config_files.iteritems(): - self.client.put_file(config_file, input_type='config', contents=new_config_contents) - - def get_rewritten_command_line(self): - """ - Returns the rewritten version of the command line to execute suitable - for remote host. - """ - return self.job_inputs.rewritten_command_line - - -def finish_job(client, cleanup_job, job_completed_normally, working_directory, work_dir_outputs, output_files): - """ - """ - action_mapper = FileActionMapper(client) - download_failure_exceptions = [] - if job_completed_normally: - for source_file, output_file in work_dir_outputs: - try: - action = action_mapper.action(output_file, 'output') - client.fetch_work_dir_output(source_file, working_directory, output_file, action[0]) - except Exception, e: - download_failure_exceptions.append(e) - # Remove from full output_files list so don't try to download directly. - output_files.remove(output_file) - for output_file in output_files: - try: - action = action_mapper.action(output_file, 'output') - client.fetch_output(output_file, working_directory=working_directory, action=action[0]) - except Exception, e: - download_failure_exceptions.append(e) - return __clean(download_failure_exceptions, cleanup_job, client) - - -def __clean(download_failure_exceptions, cleanup_job, client): - failed = (len(download_failure_exceptions) > 0) - if not failed or cleanup_job == "always": - try: - client.clean() - except: - log.warn("Failed to cleanup remote LWR job") - return failed - - -def submit_job(client, tool, command_line, config_files, input_files, output_files, working_directory): - """ - """ - file_stager = FileStager(client, tool, command_line, config_files, input_files, output_files, working_directory) - rebuilt_command_line = file_stager.get_rewritten_command_line() - job_id = file_stager.job_id - client.launch(rebuilt_command_line) - return job_id - - def _read(path): """ Utility method to quickly read small files (config files and tool - wrappers) into memory as strings. + wrappers) into memory as bytes. """ - input = open(path, "r") + input = open(path, "r", encoding="utf-8") try: return input.read() finally: input.close() -__all__ = [submit_job, finish_job] + +__all__ = [submit_job] diff --git a/lib/galaxy/jobs/runners/lwr_client/transport/curl.py b/lib/galaxy/jobs/runners/lwr_client/transport/curl.py index ba714448873..72ee1d92d59 100644 --- a/lib/galaxy/jobs/runners/lwr_client/transport/curl.py +++ b/lib/galaxy/jobs/runners/lwr_client/transport/curl.py @@ -1,4 +1,7 @@ -from cStringIO import StringIO +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO try: from pycurl import Curl except: @@ -25,6 +28,8 @@ class PycurlTransport(object): c.setopt(c.INFILESIZE, filesize) if data: c.setopt(c.POST, 1) + if type(data).__name__ == 'unicode': + data = data.encode('UTF-8') c.setopt(c.POSTFIELDS, data) c.perform() if not output_path: diff --git a/lib/galaxy/jobs/runners/lwr_client/transport/standard.py b/lib/galaxy/jobs/runners/lwr_client/transport/standard.py index 66cd3ab6e06..1bd504ace40 100644 --- a/lib/galaxy/jobs/runners/lwr_client/transport/standard.py +++ b/lib/galaxy/jobs/runners/lwr_client/transport/standard.py @@ -2,22 +2,33 @@ LWR HTTP Client layer based on Python Standard Library (urllib2) """ from __future__ import with_statement +from os.path import getsize import mmap -import urllib2 +try: + from urllib2 import urlopen +except ImportError: + from urllib.request import urlopen +try: + from urllib2 import Request +except ImportError: + from urllib.request import Request class Urllib2Transport(object): def _url_open(self, request, data): - return urllib2.urlopen(request, data) + return urlopen(request, data) def execute(self, url, data=None, input_path=None, output_path=None): - request = urllib2.Request(url=url, data=data) + request = Request(url=url, data=data) input = None try: if input_path: - input = open(input_path, 'rb') - data = mmap.mmap(input.fileno(), 0, access=mmap.ACCESS_READ) + if getsize(input_path): + input = open(input_path, 'rb') + data = mmap.mmap(input.fileno(), 0, access=mmap.ACCESS_READ) + else: + data = b"" response = self._url_open(request, data) finally: if input: @@ -26,7 +37,7 @@ class Urllib2Transport(object): with open(output_path, 'wb') as output: while True: buffer = response.read(1024) - if buffer == "": + if not buffer: break output.write(buffer) return response diff --git a/lib/galaxy/jobs/runners/lwr_client/util.py b/lib/galaxy/jobs/runners/lwr_client/util.py index 6a610a58c02..da6acebfe3a 100644 --- a/lib/galaxy/jobs/runners/lwr_client/util.py +++ b/lib/galaxy/jobs/runners/lwr_client/util.py @@ -1,5 +1,85 @@ from threading import Lock, Event from weakref import WeakValueDictionary +from os import walk +from os import curdir +from os.path import relpath +from os.path import join +import os.path +import hashlib + + +def unique_path_prefix(path): + m = hashlib.md5() + m.update(path) + return m.hexdigest() + + +def directory_files(directory): + """ + + >>> from tempfile import mkdtemp + >>> from shutil import rmtree + >>> from os.path import join + >>> from os import makedirs + >>> tempdir = mkdtemp() + >>> with open(join(tempdir, "moo"), "w") as f: pass + >>> directory_files(tempdir) + ['moo'] + >>> subdir = join(tempdir, "cow", "sub1") + >>> makedirs(subdir) + >>> with open(join(subdir, "subfile1"), "w") as f: pass + >>> with open(join(subdir, "subfile2"), "w") as f: pass + >>> sorted(directory_files(tempdir)) + ['cow/sub1/subfile1', 'cow/sub1/subfile2', 'moo'] + >>> rmtree(tempdir) + """ + contents = [] + for path, _, files in walk(directory): + relative_path = relpath(path, directory) + for name in files: + # Return file1.txt, dataset_1_files/image.png, etc... don't + # include . in path. + if relative_path != curdir: + contents.append(join(relative_path, name)) + else: + contents.append(name) + return contents + + +class PathHelper(object): + ''' + + >>> import posixpath + >>> # Forcing local path to posixpath because LWR designed to be used with + >>> # posix client. + >>> posix_path_helper = PathHelper("/", local_path_module=posixpath) + >>> windows_slash = "\\\\" + >>> len(windows_slash) + 1 + >>> nt_path_helper = PathHelper(windows_slash, local_path_module=posixpath) + >>> posix_path_helper.remote_name("moo/cow") + 'moo/cow' + >>> nt_path_helper.remote_name("moo/cow") + 'moo\\\\cow' + >>> posix_path_helper.local_name("moo/cow") + 'moo/cow' + >>> nt_path_helper.local_name("moo\\\\cow") + 'moo/cow' + ''' + + def __init__(self, separator, local_path_module=os.path): + self.separator = separator + self.local_join = local_path_module.join + self.local_sep = local_path_module.sep + + def remote_name(self, local_name): + return self.remote_join(*local_name.split(self.local_sep)) + + def local_name(self, remote_name): + return self.local_join(*remote_name.split(self.separator)) + + def remote_join(self, *args): + return self.separator.join(args) class TransferEventManager(object): diff --git a/lib/galaxy/jobs/runners/pbs.py b/lib/galaxy/jobs/runners/pbs.py index e34677f63e2..6fd577a04f1 100644 --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -34,6 +34,8 @@ log = logging.getLogger( __name__ ) __all__ = [ 'PBSJobRunner' ] +CLUSTER_ERROR_MESSAGE = "Job cannot be completed due to a cluster error, please retry it later: %s" + # The last two lines execute the command and then retrieve the command's # exit code ($?) and write it to a file. pbs_symlink_template = """ @@ -47,7 +49,7 @@ mkdir -p %s """ PBS_ARGMAP = { - 'destination' : '-d', + 'destination' : '-q', 'Execution_Time' : '-a', 'Account_Name' : '-A', 'Checkpoint' : '-c', @@ -80,6 +82,7 @@ JOB_EXIT_STATUS = { -6: "job aborted on MOM init, chkpt, ok migrate", -7: "job restart failed", -8: "exec() of user command failed", + -11: "job maximum walltime exceeded", # Added by John, not from job.h. } @@ -210,6 +213,8 @@ class PBSJobRunner( AsynchronousJobRunner ): pbs_queue_name = None pbs_server_name = self.default_pbs_server pbs_options = [] + if '-q' in job_destination.params and 'destination' not in job_destination.params: + job_destination.params['destination'] = job_destination.params.pop('-q') if 'destination' in job_destination.params: if '@' in job_destination.params['destination']: # Destination includes a server @@ -412,8 +417,11 @@ class PBSJobRunner( AsynchronousJobRunner ): assert int( status.exit_status ) == 0 log.debug("(%s/%s) PBS job has completed successfully" % ( galaxy_job_id, job_id ) ) except AssertionError: - pbs_job_state.fail_message = 'Job cannot be completed due to a cluster error, please retry it later' - log.error( '(%s/%s) PBS job failed: %s' % ( galaxy_job_id, job_id, JOB_EXIT_STATUS.get( int( status.exit_status ), 'Unknown error: %s' % status.exit_status ) ) ) + exit_status = int( status.exit_status ) + error_message = JOB_EXIT_STATUS.get( exit_status, 'Unknown error: %s' % status.exit_status ) + pbs_job_state.fail_message = CLUSTER_ERROR_MESSAGE % error_message + log.error( '(%s/%s) PBS job failed: %s' % ( galaxy_job_id, job_id, error_message ) ) + pbs_job_state.stop_job = False self.work_queue.put( ( self.fail_job, pbs_job_state ) ) continue except AttributeError: diff --git a/lib/galaxy/jobs/runners/slurm.py b/lib/galaxy/jobs/runners/slurm.py new file mode 100644 index 00000000000..51b7df56c79 --- /dev/null +++ b/lib/galaxy/jobs/runners/slurm.py @@ -0,0 +1,57 @@ +""" +SLURM job control via the DRMAA API. +""" + +import time +import logging +import subprocess + +from galaxy import model +from galaxy.jobs.runners.drmaa import DRMAAJobRunner + +log = logging.getLogger( __name__ ) + +__all__ = [ 'SlurmJobRunner' ] + + +class SlurmJobRunner( DRMAAJobRunner ): + runner_name = "SlurmRunner" + + def _complete_terminal_job( self, ajs, drmaa_state, **kwargs ): + def __get_jobinfo(): + scontrol_out = subprocess.check_output( ( 'scontrol', '-o', 'show', 'job', ajs.job_id ) ) + return dict( [ out_param.split( '=', 1 ) for out_param in scontrol_out.split() ] ) + if drmaa_state == self.drmaa_job_states.FAILED: + try: + job_info = __get_jobinfo() + sleep = 1 + while job_info['JobState'] == 'COMPLETING': + log.debug( '(%s/%s) Waiting %s seconds for failed job to exit COMPLETING state for post-mortem', ajs.job_wrapper.get_id_tag(), ajs.job_id, sleep ) + time.sleep( sleep ) + sleep *= 2 + if sleep > 64: + ajs.fail_message = "This job failed and the system timed out while trying to determine the cause of the failure." + break + job_info = __get_jobinfo() + if job_info['JobState'] == 'TIMEOUT': + ajs.fail_message = "This job was terminated because it ran longer than the maximum allowed job run time." + elif job_info['JobState'] == 'NODE_FAIL': + log.warning( '(%s/%s) Job failed due to node failure, attempting resubmission', ajs.job_wrapper.get_id_tag(), ajs.job_id ) + ajs.job_wrapper.change_state( model.Job.states.QUEUED, info = 'Job was resubmitted due to node failure' ) + try: + self.queue_job( ajs.job_wrapper ) + return + except: + ajs.fail_message = "This job failed due to a cluster node failure, and an attempt to resubmit the job failed." + elif job_info['JobState'] == 'CANCELLED': + ajs.fail_message = "This job failed because it was cancelled by an administrator." + else: + ajs.fail_message = "This job failed for reasons that could not be determined." + ajs.fail_message += '\nPlease click the bug icon to report this problem if you need help.' + ajs.stop_job = False + self.work_queue.put( ( self.fail_job, ajs ) ) + except Exception, e: + log.exception( '(%s/%s) Unable to inspect failed slurm job using scontrol, job will be unconditionally failed: %s', ajs.job_wrapper.get_id_tag(), ajs.job_id, e ) + super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state = drmaa_state ) + elif drmaa_state == self.drmaa_job_states.DONE: + super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state = drmaa_state ) diff --git a/lib/galaxy/jobs/runners/tasks.py b/lib/galaxy/jobs/runners/tasks.py index 6a9f0db06ce..3d000dc8d36 100644 --- a/lib/galaxy/jobs/runners/tasks.py +++ b/lib/galaxy/jobs/runners/tasks.py @@ -1,11 +1,10 @@ import logging import subprocess -from Queue import Queue -import threading from galaxy import model -import os, errno +import os +import errno from time import sleep from galaxy.jobs import TaskWrapper @@ -15,11 +14,13 @@ log = logging.getLogger( __name__ ) __all__ = [ 'TaskedJobRunner' ] + class TaskedJobRunner( BaseJobRunner ): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ runner_name = "TaskRunner" + def __init__( self, app, nworkers ): """Start the job runner with 'nworkers' worker threads""" super( TaskedJobRunner, self ).__init__( app, nworkers ) @@ -124,14 +125,14 @@ class TaskedJobRunner( BaseJobRunner ): #this is terminate-able when output dataset/job is deleted #so that long running set_meta()s can be canceled without having to reboot the server if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and job_wrapper.output_paths: - external_metadata_script = job_wrapper.setup_external_metadata( output_fnames = job_wrapper.get_output_fnames(), - set_extension = True, - kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior + external_metadata_script = job_wrapper.setup_external_metadata( output_fnames=job_wrapper.get_output_fnames(), + set_extension=True, + kwds={ 'overwrite' : False } ) # we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) ) - external_metadata_proc = subprocess.Popen( args = external_metadata_script, - shell = True, - env = os.environ, - preexec_fn = os.setpgrp ) + external_metadata_proc = subprocess.Popen( args=external_metadata_script, + shell=True, + env=os.environ, + preexec_fn=os.setpgrp ) job_wrapper.external_output_metadata.set_job_runner_external_pid( external_metadata_proc.pid, self.sa_session ) external_metadata_proc.wait() log.debug( 'execution of external set_meta finished for job %d' % job_wrapper.job_id ) @@ -160,7 +161,7 @@ class TaskedJobRunner( BaseJobRunner ): else: #if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished if job.external_output_metadata: - pid = job.external_output_metadata[0].job_runner_external_pid #every JobExternalOutputMetadata has a pid set, we just need to take from one of them + pid = job.external_output_metadata[0].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them else: pid = job.job_runner_external_id if pid in [ None, '' ]: @@ -170,7 +171,7 @@ class TaskedJobRunner( BaseJobRunner ): def recover( self, job, job_wrapper ): # DBTODO Task Recovery, this should be possible. - job_wrapper.change_state( model.Job.states.ERROR, info = "This job was killed when Galaxy was restarted. Please retry the job." ) + job_wrapper.change_state( model.Job.states.ERROR, info="This job was killed when Galaxy was restarted. Please retry the job." ) def _cancel_job( self, job_wrapper, task_wrappers ): """ @@ -227,7 +228,7 @@ class TaskedJobRunner( BaseJobRunner ): if e.errno == errno.ESRCH: log.debug( "_check_pid(): PID %d is dead" % pid ) else: - log.warning( "_check_pid(): Got errno %s when attempting to check PID %d: %s" %( errno.errorcode[e.errno], pid, e.strerror ) ) + log.warning( "_check_pid(): Got errno %s when attempting to check PID %d: %s" % ( errno.errorcode[e.errno], pid, e.strerror ) ) return False def _stop_pid( self, pid, job_id ): @@ -254,7 +255,7 @@ class TaskedJobRunner( BaseJobRunner ): # avoid a two-second overhead using some other asynchronous method. sleep( 2 ) if not self._check_pid( pid ): - log.debug( "_stop_pid(): %s: PID %d successfully killed with signal %d" %( job_id, pid, sig ) ) + log.debug( "_stop_pid(): %s: PID %d successfully killed with signal %d" % ( job_id, pid, sig ) ) return else: - log.warning( "_stop_pid(): %s: PID %d refuses to die after signaling TERM/KILL" %( job_id, pid ) ) + log.warning( "_stop_pid(): %s: PID %d refuses to die after signaling TERM/KILL" % ( job_id, pid ) ) diff --git a/lib/galaxy/jobs/runners/util/condor/__init__.py b/lib/galaxy/jobs/runners/util/condor/__init__.py index ef019fb4aa7..1565d43f353 100644 --- a/lib/galaxy/jobs/runners/util/condor/__init__.py +++ b/lib/galaxy/jobs/runners/util/condor/__init__.py @@ -18,12 +18,12 @@ PROBLEM_PARSING_EXTERNAL_ID = \ SUBMIT_PARAM_PREFIX = "submit_" -def submission_params(**kwds): +def submission_params(prefix=SUBMIT_PARAM_PREFIX, **kwds): submission_params = {} for key, value in kwds.iteritems(): key = key.lower() - if key.startswith(SUBMIT_PARAM_PREFIX): - condor_key = key[len(SUBMIT_PARAM_PREFIX):] + if key.startswith(prefix): + condor_key = key[len(prefix):] submission_params[condor_key] = value return submission_params diff --git a/lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh b/lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh index 18b828e5c0e..3e016da0285 100644 --- a/lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh +++ b/lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh @@ -1,6 +1,9 @@ export GALAXY_SLOTS_CONFIGURED="1" -if [ -n "$SLURM_JOB_NUM_NODES" ]; then - GALAXY_SLOTS="$SLURM_JOB_NUM_NODES" +if [ -n "$SLURM_NTASKS" ]; then + # May want to multiply this by ${SLURM_CPUS_PER_TASK:-1}. + # SLURM_NTASKS is total tasks over all nodes so this is + # shouldn't be used for multi-node requests. + GALAXY_SLOTS="$SLURM_NTASKS" elif [ -n "$NSLOTS" ]; then GALAXY_SLOTS="$NSLOTS" elif [ -f "$PBS_NODEFILE" ]; then diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4445aa14c98..b497a17edbb 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6,7 +6,6 @@ the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ from galaxy import eggs -eggs.require("simplejson") eggs.require("pexpect") import codecs @@ -15,10 +14,11 @@ import logging import operator import os import pexpect -import simplejson +import json import socket import time from string import Template +from itertools import ifilter import galaxy.datatypes import galaxy.datatypes.registry @@ -26,16 +26,19 @@ import galaxy.security.passwords from galaxy.datatypes.metadata import MetadataCollection from galaxy.model.item_attrs import Dictifiable, UsesAnnotations from galaxy.security import get_permitted_actions -from galaxy.util import asbool, is_multi_byte, nice_size, Params, restore_text, send_mail +from galaxy.util import is_multi_byte, nice_size, Params, restore_text, send_mail +from galaxy.util import ready_name_for_url from galaxy.util.bunch import Bunch from galaxy.util.hash_util import new_secure_hash +from galaxy.util.directory_hash import directory_hash_id from galaxy.web.framework.helpers import to_unicode from galaxy.web.form_builder import (AddressField, CheckboxField, HistoryField, PasswordField, SelectField, TextArea, TextField, WorkflowField, WorkflowMappingField) from sqlalchemy.orm import object_session +from sqlalchemy.orm import joinedload from sqlalchemy.sql.expression import func -from tool_shed.util import common_util +from sqlalchemy import not_ log = logging.getLogger( __name__ ) @@ -43,6 +46,12 @@ datatypes_registry = galaxy.datatypes.registry.Registry() # Default Value Required for unit tests datatypes_registry.load_datatypes() +# When constructing filters with in for a fixed set of ids, maximum +# number of items to place in the IN statement. Different databases +# are going to have different limits so it is likely best to not let +# this be unlimited - filter in Python if over this limit. +MAX_IN_FILTER_LENGTH = 100 + class NoConverterException(Exception): def __init__(self, value): self.value = value @@ -63,6 +72,19 @@ def set_datatypes_registry( d_registry ): datatypes_registry = d_registry +class HasName: + + def get_display_name( self ): + """ + These objects have a name attribute can be either a string or a unicode + object. If string, convert to unicode object assuming 'utf-8' format. + """ + name = self.name + if isinstance(name, str): + name = unicode(name, 'utf-8') + return name + + class User( object, Dictifiable ): use_pbkdf2 = True """ @@ -203,8 +225,8 @@ class User( object, Dictifiable ): class Job( object, Dictifiable ): - dict_collection_visible_keys = [ 'id', 'state', 'exit_code' ] - dict_element_visible_keys = [ 'id', 'state', 'exit_code' ] + dict_collection_visible_keys = [ 'id', 'state', 'exit_code', 'update_time', 'create_time' ] + dict_element_visible_keys = [ 'id', 'state', 'exit_code', 'update_time', 'create_time' ] """ A job represents a request to run a tool given input datasets, tool @@ -410,30 +432,31 @@ class Job( object, Dictifiable ): dataset.info = 'Job output deleted by user before job completed' def to_dict( self, view='collection' ): rval = super( Job, self ).to_dict( view=view ) - rval['tool_name'] = self.tool_id - param_dict = dict( [ ( p.name, p.value ) for p in self.parameters ] ) - rval['params'] = param_dict + rval['tool_id'] = self.tool_id + if view == 'element': + param_dict = dict( [ ( p.name, p.value ) for p in self.parameters ] ) + rval['params'] = param_dict - input_dict = {} - for i in self.input_datasets: - if i.dataset is not None: - input_dict[i.name] = {"hda_id" : i.dataset.id} - for i in self.input_library_datasets: - if i.dataset is not None: - input_dict[i.name] = {"ldda_id" : i.dataset.id} - for k in input_dict: - if k in param_dict: - del param_dict[k] - rval['inputs'] = input_dict + input_dict = {} + for i in self.input_datasets: + if i.dataset is not None: + input_dict[i.name] = {"id" : i.dataset.id, "src" : "hda"} + for i in self.input_library_datasets: + if i.dataset is not None: + input_dict[i.name] = {"id" : i.dataset.id, "src" : "ldda"} + for k in input_dict: + if k in param_dict: + del param_dict[k] + rval['inputs'] = input_dict - output_dict = {} - for i in self.output_datasets: - if i.dataset is not None: - output_dict[i.name] = {"hda_id" : i.dataset.id} - for i in self.output_library_datasets: - if i.dataset is not None: - output_dict[i.name] = {"ldda_id" : i.dataset.id} - rval['outputs'] = output_dict + output_dict = {} + for i in self.output_datasets: + if i.dataset is not None: + output_dict[i.name] = {"id" : i.dataset.id, "src" : "hda"} + for i in self.output_library_datasets: + if i.dataset is not None: + output_dict[i.name] = {"id" : i.dataset.id, "src" : "ldda"} + rval['outputs'] = output_dict return rval @@ -625,6 +648,7 @@ class JobExternalOutputMetadata( object ): return self.library_dataset_dataset_association return None + class JobExportHistoryArchive( object ): def __init__( self, job=None, history=None, dataset=None, compressed=False, \ history_attrs_filename=None, datasets_attrs_filename=None, @@ -637,6 +661,33 @@ class JobExportHistoryArchive( object ): self.datasets_attrs_filename = datasets_attrs_filename self.jobs_attrs_filename = jobs_attrs_filename + @property + def up_to_date( self ): + """ Return False, if a new export should be generated for corresponding + history. + """ + job = self.job + return job.state not in [ Job.states.ERROR, Job.states.DELETED ] \ + and job.update_time > self.history.update_time + + @property + def ready( self ): + return self.job.state == Job.states.OK + + @property + def preparing( self ): + return self.job.state in [ Job.states.RUNNING, Job.states.QUEUED, Job.states.WAITING ] + + @property + def export_name( self ): + # Stream archive. + hname = ready_name_for_url( self.history.name ) + hname = "Galaxy-History-%s.tar" % ( hname ) + if self.compressed: + hname += ".gz" + return hname + + class JobImportHistoryArchive( object ): def __init__( self, job=None, history=None, archive_dir=None ): self.job = job @@ -707,7 +758,7 @@ class UserGroupAssociation( object ): self.user = user self.group = group -class History( object, Dictifiable, UsesAnnotations ): +class History( object, Dictifiable, UsesAnnotations, HasName ): dict_collection_visible_keys = ( 'id', 'name', 'published', 'deleted' ) dict_element_visible_keys = ( 'id', 'name', 'published', 'deleted', 'genome_build', 'purged' ) @@ -728,8 +779,7 @@ class History( object, Dictifiable, UsesAnnotations ): self.tags = [] def _next_hid( self ): - # TODO: override this with something in the database that ensures - # better integrity + # this is overriden in mapping.py db_next_hid() method if len( self.datasets ) == 0: return 1 else: @@ -815,16 +865,6 @@ class History( object, Dictifiable, UsesAnnotations ): # This needs to be a list return [ hda for hda in self.datasets if not hda.dataset.deleted ] - def get_display_name( self ): - """ - History name can be either a string or a unicode object. - If string, convert to unicode object assuming 'utf-8' format. - """ - history_name = self.name - if isinstance(history_name, str): - history_name = unicode(history_name, 'utf-8') - return history_name - def to_dict( self, view='collection', value_mapper = None ): # Get basic value. @@ -863,6 +903,11 @@ class History( object, Dictifiable, UsesAnnotations ): return changed + @property + def latest_export( self ): + exports = self.exports + return exports and exports[ 0 ] + @property def get_disk_size_bytes( self ): return self.get_disk_size( nice_size=False ) @@ -892,6 +937,49 @@ class History( object, Dictifiable, UsesAnnotations ): rval = galaxy.datatypes.data.nice_size( rval ) return rval + @property + def active_datasets_children_and_roles( self ): + if not hasattr(self, '_active_datasets_children_and_roles'): + db_session = object_session( self ) + query = db_session.query( HistoryDatasetAssociation ).filter( HistoryDatasetAssociation.table.c.history_id == self.id ). \ + filter( not_( HistoryDatasetAssociation.deleted ) ). \ + order_by( HistoryDatasetAssociation.table.c.hid.asc() ). \ + options( + joinedload("children"), + joinedload("dataset"), + joinedload("dataset.actions"), + joinedload("dataset.actions.role"), + ) + self._active_datasets_children_and_roles = query.all() + return self._active_datasets_children_and_roles + + def contents_iter( self, **kwds ): + """ + Fetch filtered list of contents of history. + """ + python_filter = None + db_session = object_session( self ) + assert db_session != None + query = db_session.query( HistoryDatasetAssociation ).filter( HistoryDatasetAssociation.table.c.history_id == self.id ) + query = query.order_by( HistoryDatasetAssociation.table.c.hid.asc() ) + deleted = galaxy.util.string_as_bool_or_none( kwds.get( 'deleted', None ) ) + if deleted is not None: + query = query.filter( HistoryDatasetAssociation.deleted == deleted ) + visible = galaxy.util.string_as_bool_or_none( kwds.get( 'visible', None ) ) + if visible is not None: + query = query.filter( HistoryDatasetAssociation.visible == visible ) + if 'ids' in kwds: + ids = kwds['ids'] + max_in_filter_length = kwds.get('max_in_filter_length', MAX_IN_FILTER_LENGTH) + if len(ids) < max_in_filter_length: + query = query.filter( HistoryDatasetAssociation.id.in_(ids) ) + else: + python_filter = lambda hda: hda.id in ids + if python_filter: + return ifilter(python_filter, query) + else: + return query + def copy_tags_from(self,target_user,source_history): for src_shta in source_history.tags: new_shta = src_shta.copy() @@ -1133,7 +1221,7 @@ class Dataset( object ): self.total_size = self.file_size or 0 if self.object_store.exists(self, extra_dir=self._extra_files_path or "dataset_%d_files" % self.id, dir_only=True): for root, dirs, files in os.walk( self.extra_files_path ): - self.total_size += sum( [ os.path.getsize( os.path.join( root, file ) ) for file in files ] ) + self.total_size += sum( [ os.path.getsize( os.path.join( root, file ) ) for file in files if os.path.exists( os.path.join( root, file ) ) ] ) def has_data( self ): """Detects whether there is any data""" return self.get_size() > 0 @@ -1553,7 +1641,7 @@ class DatasetInstance( object ): return msg -class HistoryDatasetAssociation( DatasetInstance, Dictifiable, UsesAnnotations ): +class HistoryDatasetAssociation( DatasetInstance, Dictifiable, UsesAnnotations, HasName ): """ Resource class that creates a relation between a dataset and a user history. """ @@ -1683,17 +1771,6 @@ class HistoryDatasetAssociation( DatasetInstance, Dictifiable, UsesAnnotations ) for assoc in self.implicitly_converted_parent_datasets: assoc.clear( purge = purge, delete_dataset = False ) - def get_display_name( self ): - """ - Return the name of this HDA in either ascii or utf-8 encoding. - """ - # Name can be either a string or a unicode object. - # If string, convert to unicode object assuming 'utf-8' format. - hda_name = self.name - if isinstance(hda_name, str): - hda_name = unicode(hda_name, 'utf-8') - return hda_name - def get_access_roles( self, trans ): """ Return The access roles associated with this HDA's dataset. @@ -1746,11 +1823,13 @@ class HistoryDatasetAssociation( DatasetInstance, Dictifiable, UsesAnnotations ) visible = hda.visible, state = hda.state, file_size = int( hda.get_size() ), + update_time = hda.update_time.isoformat(), data_type = hda.ext, genome_build = hda.dbkey, - misc_info = hda.info, + misc_info = hda.info.strip() if isinstance( hda.info, basestring ) else hda.info, misc_blurb = hda.blurb ) + # add tags string list tags_str_list = [] for tag in self.tags: tag_str = tag.user_tname @@ -1765,6 +1844,9 @@ class HistoryDatasetAssociation( DatasetInstance, Dictifiable, UsesAnnotations ) if hda.history is not None: rval['history_id'] = hda.history.id + if hda.extended_metadata is not None: + rval['extended_metadata'] = hda.extended_metadata.data + rval[ 'peek' ] = to_unicode( hda.display_peek() ) for name, spec in hda.metadata.spec.items(): val = hda.metadata.get( name ) @@ -1800,6 +1882,10 @@ class HistoryDatasetAssociation( DatasetInstance, Dictifiable, UsesAnnotations ) if new_val == old_val: continue + # special cases here + if key == 'deleted' and new_val is False and self.purged: + raise Exception( 'Cannot undelete a purged dataset' ) + self.__setattr__( key, new_val ) changed[ key ] = new_val @@ -1818,10 +1904,10 @@ class HistoryDatasetAssociationSubset( object ): self.subset = subset self.location = location -class Library( object, Dictifiable ): +class Library( object, Dictifiable, HasName ): permitted_actions = get_permitted_actions( filter='LIBRARY' ) dict_collection_visible_keys = ( 'id', 'name' ) - dict_element_visible_keys = ( 'id', 'deleted', 'name', 'description', 'synopsis' ) + dict_element_visible_keys = ( 'id', 'deleted', 'name', 'description', 'synopsis', 'root_folder_id' ) def __init__( self, name=None, description=None, synopsis=None, root_folder=None ): self.name = name or "Unnamed library" self.description = description @@ -1879,16 +1965,9 @@ class Library( object, Dictifiable ): if lp.action == trans.app.security_agent.permitted_actions.LIBRARY_ACCESS.action: roles.append( lp.role ) return roles - def get_display_name( self ): - # Library name can be either a string or a unicode object. If string, - # convert to unicode object assuming 'utf-8' format. - name = self.name - if isinstance( name, str ): - name = unicode( name, 'utf-8' ) - return name -class LibraryFolder( object, Dictifiable ): - dict_element_visible_keys = ( 'id', 'parent_id', 'name', 'description', 'item_count', 'genome_build' ) +class LibraryFolder( object, Dictifiable, HasName ): + dict_element_visible_keys = ( 'id', 'parent_id', 'name', 'description', 'item_count', 'genome_build', 'update_time' ) def __init__( self, name=None, description=None, item_count=0, order_id=None ): self.name = name or "Unnamed folder" self.description = description @@ -1952,13 +2031,7 @@ class LibraryFolder( object, Dictifiable ): def activatable_library_datasets( self ): # This needs to be a list return [ ld for ld in self.datasets if ld.library_dataset_dataset_association and not ld.library_dataset_dataset_association.dataset.deleted ] - def get_display_name( self ): - # Library folder name can be either a string or a unicode object. If string, - # convert to unicode object assuming 'utf-8' format. - name = self.name - if isinstance( name, str ): - name = unicode( name, 'utf-8' ) - return name + def to_dict( self, view='collection' ): rval = super( LibraryFolder, self ).to_dict( view=view ) info_association, inherited = self.get_info_association() @@ -2041,6 +2114,7 @@ class LibraryDataset( object ): rval = dict( id = self.id, ldda_id = ldda.id, + parent_library_id = self.folder.parent_library.id, folder_id = self.folder_id, model_class = self.__class__.__name__, name = ldda.name, @@ -2053,6 +2127,7 @@ class LibraryDataset( object ): genome_build = ldda.dbkey, misc_info = ldda.info, misc_blurb = ldda.blurb, + peek = ( lambda ldda: ldda.display_peek() if ldda.peek and ldda.peek != 'no peek' else None )( ldda ), template_data = template_data ) if ldda.dataset.uuid is None: rval['uuid'] = None @@ -2067,7 +2142,7 @@ class LibraryDataset( object ): rval['metadata_' + name] = val return rval -class LibraryDatasetDatasetAssociation( DatasetInstance ): +class LibraryDatasetDatasetAssociation( DatasetInstance, HasName ): def __init__( self, copied_from_history_dataset_association=None, copied_from_library_dataset_dataset_association=None, @@ -2174,6 +2249,7 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ): library_dataset_id = ldda.library_dataset_id, file_size = file_size, file_name = ldda.file_name, + update_time = ldda.update_time.isoformat(), data_type = ldda.ext, genome_build = ldda.dbkey, misc_info = ldda.info, @@ -2239,17 +2315,8 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ): template_data[template.name] = tmp_dict return template_data def templates_json( self, use_name=False ): - return simplejson.dumps( self.templates_dict( use_name=use_name ) ) + return json.dumps( self.templates_dict( use_name=use_name ) ) - def get_display_name( self ): - """ - LibraryDatasetDatasetAssociation name can be either a string or a unicode object. - If string, convert to unicode object assuming 'utf-8' format. - """ - ldda_name = self.name - if isinstance( ldda_name, str ): - ldda_name = unicode( ldda_name, 'utf-8' ) - return ldda_name class ExtendedMetadata( object ): def __init__(self, data): @@ -2385,8 +2452,8 @@ class UCI( object ): class StoredWorkflow( object, Dictifiable): - dict_collection_visible_keys = ( 'id', 'name', 'published' ) - dict_element_visible_keys = ( 'id', 'name', 'published' ) + dict_collection_visible_keys = ( 'id', 'name', 'published', 'deleted' ) + dict_element_visible_keys = ( 'id', 'name', 'published', 'deleted' ) def __init__( self ): self.id = None @@ -3138,7 +3205,8 @@ class UserOpenID( object ): self.session = session self.openid = openid -class Page( object ): +class Page( object, Dictifiable ): + dict_element_visible_keys = [ 'id', 'title', 'latest_revision_id', 'slug', 'published', 'importable', 'deleted' ] def __init__( self ): self.id = None self.user = None @@ -3149,12 +3217,27 @@ class Page( object ): self.importable = None self.published = None -class PageRevision( object ): + def to_dict( self, view='element' ): + rval = super( Page, self ).to_dict( view=view ) + rev = [] + for a in self.revisions: + rev.append(a.id) + rval['revision_ids'] = rev + return rval + +class PageRevision( object, Dictifiable ): + dict_element_visible_keys = [ 'id', 'page_id', 'title', 'content' ] def __init__( self ): self.user = None self.title = None self.content = None + def to_dict( self, view='element' ): + rval = super( PageRevision, self ).to_dict( view=view ) + rval['create_time'] = str(self.create_time) + rval['update_time'] = str(self.update_time) + return rval + class PageUserShareAssociation( object ): def __init__( self ): self.page = None @@ -3385,621 +3468,7 @@ class UserAction( object ): self.context = context class APIKeys( object ): - pass - - -class ToolShedRepository( object ): - dict_collection_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', - 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) - dict_element_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', - 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) - installation_status = Bunch( NEW='New', - CLONING='Cloning', - SETTING_TOOL_VERSIONS='Setting tool versions', - INSTALLING_REPOSITORY_DEPENDENCIES='Installing repository dependencies', - INSTALLING_TOOL_DEPENDENCIES='Installing tool dependencies', - LOADING_PROPRIETARY_DATATYPES='Loading proprietary datatypes', - INSTALLED='Installed', - DEACTIVATED='Deactivated', - ERROR='Error', - UNINSTALLED='Uninstalled' ) - states = Bunch( INSTALLING = 'running', - OK = 'ok', - WARNING = 'queued', - ERROR = 'error', - UNINSTALLED = 'deleted_new' ) - - def __init__( self, id=None, create_time=None, tool_shed=None, name=None, description=None, owner=None, installed_changeset_revision=None, - changeset_revision=None, ctx_rev=None, metadata=None, includes_datatypes=False, tool_shed_status=None, deleted=False, - uninstalled=False, dist_to_shed=False, status=None, error_message=None ): + def __init__( self, id=None, user_id=None, key=None): self.id = id - self.create_time = create_time - self.tool_shed = tool_shed - self.name = name - self.description = description - self.owner = owner - self.installed_changeset_revision = installed_changeset_revision - self.changeset_revision = changeset_revision - self.ctx_rev = ctx_rev - self.metadata = metadata - self.includes_datatypes = includes_datatypes - self.tool_shed_status = tool_shed_status - self.deleted = deleted - self.uninstalled = uninstalled - self.dist_to_shed = dist_to_shed - self.status = status - self.error_message = error_message - - def as_dict( self, value_mapper=None ): - return self.to_dict( view='element', value_mapper=value_mapper ) - - @property - def can_install( self ): - return self.status == self.installation_status.NEW - - @property - def can_reset_metadata( self ): - return self.status == self.installation_status.INSTALLED - - @property - def can_uninstall( self ): - return self.status != self.installation_status.UNINSTALLED - - @property - def can_deactivate( self ): - return self.status not in [ self.installation_status.DEACTIVATED, self.installation_status.UNINSTALLED ] - - @property - def can_reinstall_or_activate( self ): - return self.deleted - - def to_dict( self, view='collection', value_mapper=None ): - if value_mapper is None: - value_mapper = {} - rval = {} - try: - visible_keys = self.__getattribute__( 'dict_' + view + '_visible_keys' ) - except AttributeError: - raise Exception( 'Unknown API view: %s' % view ) - for key in visible_keys: - try: - rval[ key ] = self.__getattribute__( key ) - if key in value_mapper: - rval[ key ] = value_mapper.get( key, rval[ key ] ) - except AttributeError: - rval[ key ] = None - return rval - - def get_shed_config_filename( self ): - shed_config_filename = None - if self.metadata: - shed_config_filename = self.metadata.get( 'shed_config_filename', shed_config_filename ) - return shed_config_filename - - def get_shed_config_dict( self, app, default=None ): - """ - Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry - in the shed_tool_conf_dict. - """ - if not self.shed_config_filename: - self.guess_shed_config( app, default=default ) - if self.shed_config_filename: - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: - if self.shed_config_filename == shed_tool_conf_dict[ 'config_filename' ]: - return shed_tool_conf_dict - return default - - def get_tool_relative_path( self, app ): - shed_conf_dict = self.get_shed_config_dict( app ) - tool_path = None - relative_path = None - if shed_conf_dict: - tool_path = shed_conf_dict[ 'tool_path' ] - relative_path = os.path.join( self.tool_shed_path_name, 'repos', self.owner, self.name, self.installed_changeset_revision ) - return tool_path, relative_path - - def guess_shed_config( self, app, default=None ): - tool_ids = [] - metadata = self.metadata or {} - for tool in metadata.get( 'tools', [] ): - tool_ids.append( tool.get( 'guid' ) ) - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: - name = shed_tool_conf_dict[ 'config_filename' ] - for elem in shed_tool_conf_dict[ 'config_elems' ]: - if elem.tag == 'tool': - for sub_elem in elem.findall( 'id' ): - tool_id = sub_elem.text.strip() - if tool_id in tool_ids: - self.shed_config_filename = name - return shed_tool_conf_dict - elif elem.tag == "section": - for tool_elem in elem.findall( 'tool' ): - for sub_elem in tool_elem.findall( 'id' ): - tool_id = sub_elem.text.strip() - if tool_id in tool_ids: - self.shed_config_filename = name - return shed_tool_conf_dict - if self.includes_datatypes: - #we need to search by filepaths here, which is less desirable - tool_shed_url = self.tool_shed - if tool_shed_url.find( ':' ) > 0: - # Eliminate the port, if any, since it will result in an invalid directory name. - tool_shed_url = tool_shed_url.split( ':' )[ 0 ] - tool_shed = tool_shed_url.rstrip( '/' ) - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: - tool_path = shed_tool_conf_dict[ 'tool_path' ] - relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) - if os.path.exists( relative_path ): - self.shed_config_filename = shed_tool_conf_dict[ 'config_filename' ] - return shed_tool_conf_dict - return default - - @property - def has_readme_files( self ): - if self.metadata: - return 'readme_files' in self.metadata - return False - - @property - def has_repository_dependencies( self ): - if self.metadata: - repository_dependencies_dict = self.metadata.get( 'repository_dependencies', {} ) - repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) - # [["http://localhost:9009", "package_libgtextutils_0_6", "test", "e2003cbf18cd", "True", "True"]] - for rd_tup in repository_dependencies: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup ) - if not asbool( only_if_compiling_contained_td ): - return True - return False - - @property - def has_repository_dependencies_only_if_compiling_contained_td( self ): - if self.metadata: - repository_dependencies_dict = self.metadata.get( 'repository_dependencies', {} ) - repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) - # [["http://localhost:9009", "package_libgtextutils_0_6", "test", "e2003cbf18cd", "True", "True"]] - for rd_tup in repository_dependencies: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup ) - if not asbool( only_if_compiling_contained_td ): - return False - return True - return False - - @property - def in_error_state( self ): - return self.status == self.installation_status.ERROR - - @property - def includes_data_managers( self ): - if self.metadata: - return bool( len( self.metadata.get( 'data_manager', {} ).get( 'data_managers', {} ) ) ) - return False - - @property - def includes_tools( self ): - if self.metadata: - return 'tools' in self.metadata - return False - - @property - def includes_tools_for_display_in_tool_panel( self ): - if self.includes_tools: - tool_dicts = self.metadata[ 'tools' ] - for tool_dict in tool_dicts: - if tool_dict.get( 'add_to_tool_panel', True ): - return True - return False - - @property - def includes_tool_dependencies( self ): - if self.metadata: - return 'tool_dependencies' in self.metadata - return False - - @property - def includes_workflows( self ): - if self.metadata: - return 'workflows' in self.metadata - return False - - @property - def installed_repository_dependencies( self ): - """Return the repository's repository dependencies that are currently installed.""" - installed_required_repositories = [] - for required_repository in self.repository_dependencies: - if required_repository.status == self.installation_status.INSTALLED: - installed_required_repositories.append( required_repository ) - return installed_required_repositories - - @property - def installed_tool_dependencies( self ): - """Return the repository's tool dependencies that are currently installed.""" - installed_dependencies = [] - for tool_dependency in self.tool_dependencies: - if tool_dependency.status in [ ToolDependency.installation_status.INSTALLED, ToolDependency.installation_status.ERROR ]: - installed_dependencies.append( tool_dependency ) - return installed_dependencies - - @property - def is_deprecated_in_tool_shed( self ): - if self.tool_shed_status: - return galaxy.util.asbool( self.tool_shed_status.get( 'repository_deprecated', False ) ) - return False - - @property - def is_latest_installable_revision( self ): - if self.tool_shed_status: - return galaxy.util.asbool( self.tool_shed_status.get( 'latest_installable_revision', False ) ) - return False - - @property - def missing_repository_dependencies( self ): - """Return the repository's repository dependencies that are not currently installed, and may not ever have been installed.""" - missing_required_repositories = [] - for required_repository in self.repository_dependencies: - if required_repository.status not in [ self.installation_status.INSTALLED ]: - missing_required_repositories.append( required_repository ) - return missing_required_repositories - - @property - def missing_tool_dependencies( self ): - """Return the repository's tool dependencies that are not currently installed, and may not ever have been installed.""" - missing_dependencies = [] - for tool_dependency in self.tool_dependencies: - if tool_dependency.status not in [ ToolDependency.installation_status.INSTALLED ]: - missing_dependencies.append( tool_dependency ) - return missing_dependencies - - def repo_files_directory( self, app ): - repo_path = self.repo_path( app ) - if repo_path: - return os.path.join( repo_path, self.name ) - return None - - def repo_path( self, app ): - tool_shed_url = self.tool_shed - if tool_shed_url.find( ':' ) > 0: - # Eliminate the port, if any, since it will result in an invalid directory name. - tool_shed_url = tool_shed_url.split( ':' )[ 0 ] - tool_shed = tool_shed_url.rstrip( '/' ) - for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ): - tool_path = shed_tool_conf_dict[ 'tool_path' ] - relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) - if os.path.exists( relative_path ): - return relative_path - return None - - @property - def repository_dependencies( self ): - """ - Return all of this repository's repository dependencies, ignoring their attributes like prior_installation_required and - only_if_compiling_contained_td. - """ - required_repositories = [] - for rrda in self.required_repositories: - repository_dependency = rrda.repository_dependency - required_repository = repository_dependency.repository - if required_repository: - required_repositories.append( required_repository ) - return required_repositories - - @property - def repository_dependencies_being_installed( self ): - """Return the repository's repository dependencies that are currently being installed.""" - required_repositories_being_installed = [] - for required_repository in self.repository_dependencies: - if required_repository.status in [ self.installation_status.CLONING, - self.installation_status.INSTALLING_REPOSITORY_DEPENDENCIES, - self.installation_status.INSTALLING_TOOL_DEPENDENCIES, - self.installation_status.LOADING_PROPRIETARY_DATATYPES, - self.installation_status.SETTING_TOOL_VERSIONS ]: - required_repositories_being_installed.append( required_repository ) - return required_repositories_being_installed - - @property - def repository_dependencies_missing_or_being_installed( self ): - """Return the repository's repository dependencies that are either missing or currently being installed.""" - required_repositories_missing_or_being_installed = [] - for required_repository in self.repository_dependencies: - if required_repository.status in [ self.installation_status.ERROR, - self.installation_status.INSTALLING, - self.installation_status.NEVER_INSTALLED, - self.installation_status.UNINSTALLED ]: - required_repositories_missing_or_being_installed.append( required_repository ) - return required_repositories_missing_or_being_installed - - @property - def repository_dependencies_with_installation_errors( self ): - """Return the repository's repository dependencies that have installation errors.""" - required_repositories_with_installation_errors = [] - for required_repository in self.repository_dependencies: - if required_repository.status == self.installation_status.ERROR: - required_repositories_with_installation_errors.append( required_repository ) - return required_repositories_with_installation_errors - - @property - def requires_prior_installation_of( self ): - """ - Return a list of repository dependency tuples like (tool_shed, name, owner, changeset_revision, prior_installation_required) for this - repository's repository dependencies where prior_installation_required is True. By definition, repository dependencies are required to - be installed in order for this repository to function correctly. However, those repository dependencies that are defined for this - repository with prior_installation_required set to True place them in a special category in that the required repositories must be - installed before this repository is installed. Among other things, this enables these "special" repository dependencies to include - information that enables the successful intallation of this repository. This method is not used during the initial installation of - this repository, but only after it has been installed (metadata must be set for this repository in order for this method to be useful). - """ - required_rd_tups_that_must_be_installed = [] - if self.has_repository_dependencies: - rd_tups = self.metadata[ 'repository_dependencies' ][ 'repository_dependencies' ] - for rd_tup in rd_tups: - if len( rd_tup ) == 5: - tool_shed, name, owner, changeset_revision, prior_installation_required = rd_tup - if galaxy.util.asbool( prior_installation_required ): - required_rd_tups_that_must_be_installed.append( ( tool_shed, name, owner, changeset_revision, 'True', 'False' ) ) - elif len( rd_tup ) == 6: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = rd_tup - # The repository dependency will only be required to be previsously installed if it does not fall into the category of - # a repository that must be installed only so that it's contained tool dependency can be used for compiling the tool - # dependency of the dependent repository. - if not galaxy.util.asbool( only_if_compiling_contained_td ): - if galaxy.util.asbool( prior_installation_required ): - required_rd_tups_that_must_be_installed.append( ( tool_shed, name, owner, changeset_revision, 'True', 'False' ) ) - return required_rd_tups_that_must_be_installed - - @property - def revision_update_available( self ): - # This method should be named update_available, but since it is no longer possible to drop a table column using migration scripts - # with the sqlite database (see ~/galaxy/model/migrate/versions/0016_drop_update_available_col_add_tool_shed_status_col.py), we - # have to name it in such a way that it will not conflict with the eliminated tool_shed_repository.update_available column (which - # cannot be eliminated if using the sqlite database). - if self.tool_shed_status: - return galaxy.util.asbool( self.tool_shed_status.get( 'revision_update', False ) ) - return False - - def set_shed_config_filename( self, value ): - self.metadata[ 'shed_config_filename' ] = value - - shed_config_filename = property( get_shed_config_filename, set_shed_config_filename ) - - @property - def tool_dependencies_being_installed( self ): - dependencies_being_installed = [] - for tool_dependency in self.tool_dependencies: - if tool_dependency.status == ToolDependency.installation_status.INSTALLING: - dependencies_being_installed.append( tool_dependency ) - return dependencies_being_installed - - @property - def tool_dependencies_missing_or_being_installed( self ): - dependencies_missing_or_being_installed = [] - for tool_dependency in self.tool_dependencies: - if tool_dependency.status in [ ToolDependency.installation_status.ERROR, - ToolDependency.installation_status.INSTALLING, - ToolDependency.installation_status.NEVER_INSTALLED, - ToolDependency.installation_status.UNINSTALLED ]: - dependencies_missing_or_being_installed.append( tool_dependency ) - return dependencies_missing_or_being_installed - - @property - def tool_dependencies_with_installation_errors( self ): - dependencies_with_installation_errors = [] - for tool_dependency in self.tool_dependencies: - if tool_dependency.status == ToolDependency.installation_status.ERROR: - dependencies_with_installation_errors.append( tool_dependency ) - return dependencies_with_installation_errors - - @property - def tool_shed_path_name( self ): - tool_shed_url = self.tool_shed - if tool_shed_url.find( ':' ) > 0: - # Eliminate the port, if any, since it will result in an invalid directory name. - tool_shed_url = tool_shed_url.split( ':' )[ 0 ] - return tool_shed_url.rstrip( '/' ) - - @property - def tuples_of_repository_dependencies_needed_for_compiling_td( self ): - """ - Return tuples defining this repository's repository dependencies that are necessary only for compiling this repository's tool - dependencies. - """ - rd_tups_of_repositories_needed_for_compiling_td = [] - if self.metadata: - repository_dependencies = self.metadata.get( 'repository_dependencies', None ) - rd_tups = repository_dependencies[ 'repository_dependencies' ] - for rd_tup in rd_tups: - if len( rd_tup ) == 6: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = rd_tup - if galaxy.util.asbool( only_if_compiling_contained_td ): - rd_tups_of_repositories_needed_for_compiling_td.append( ( tool_shed, name, owner, changeset_revision, 'False', 'True' ) ) - return rd_tups_of_repositories_needed_for_compiling_td - - @property - def uninstalled_repository_dependencies( self ): - """Return the repository's repository dependencies that have been uninstalled.""" - uninstalled_required_repositories = [] - for required_repository in self.repository_dependencies: - if required_repository.status == self.installation_status.UNINSTALLED: - uninstalled_required_repositories.append( required_repository ) - return uninstalled_required_repositories - - @property - def uninstalled_tool_dependencies( self ): - """Return the repository's tool dependencies that have been uninstalled.""" - uninstalled_tool_dependencies = [] - for tool_dependency in self.tool_dependencies: - if tool_dependency.status == ToolDependency.installation_status.UNINSTALLED: - uninstalled_tool_dependencies.append( tool_dependency ) - return uninstalled_tool_dependencies - - @property - def upgrade_available( self ): - if self.tool_shed_status: - if self.is_deprecated_in_tool_shed: - # Only allow revision upgrades if the repository is not deprecated in the tool shed. - return False - return galaxy.util.asbool( self.tool_shed_status.get( 'revision_upgrade', False ) ) - return False - - -class RepositoryRepositoryDependencyAssociation( object ): - def __init__( self, tool_shed_repository_id=None, repository_dependency_id=None ): - self.tool_shed_repository_id = tool_shed_repository_id - self.repository_dependency_id = repository_dependency_id - - -class RepositoryDependency( object ): - def __init__( self, tool_shed_repository_id=None ): - self.tool_shed_repository_id = tool_shed_repository_id - - -class ToolDependency( object ): - installation_status = Bunch( NEVER_INSTALLED='Never installed', - INSTALLING='Installing', - INSTALLED='Installed', - ERROR='Error', - UNINSTALLED='Uninstalled' ) - states = Bunch( INSTALLING = 'running', - OK = 'ok', - WARNING = 'queued', - ERROR = 'error', - UNINSTALLED = 'deleted_new' ) - - def __init__( self, tool_shed_repository_id=None, name=None, version=None, type=None, status=None, error_message=None ): - self.tool_shed_repository_id = tool_shed_repository_id - self.name = name - self.version = version - self.type = type - self.status = status - self.error_message = error_message - - @property - def can_install( self ): - return self.status in [ self.installation_status.NEVER_INSTALLED, self.installation_status.UNINSTALLED ] - - @property - def can_uninstall( self ): - return self.status in [ self.installation_status.ERROR, self.installation_status.INSTALLED ] - - @property - def can_update( self ): - return self.status in [ self.installation_status.NEVER_INSTALLED, - self.installation_status.INSTALLED, - self.installation_status.ERROR, - self.installation_status.UNINSTALLED ] - - @property - def in_error_state( self ): - return self.status == self.installation_status.ERROR - - def installation_directory( self, app ): - if self.type == 'package': - return os.path.join( app.config.tool_dependency_dir, - self.name, - self.version, - self.tool_shed_repository.owner, - self.tool_shed_repository.name, - self.tool_shed_repository.installed_changeset_revision ) - if self.type == 'set_environment': - return os.path.join( app.config.tool_dependency_dir, - 'environment_settings', - self.name, - self.tool_shed_repository.owner, - self.tool_shed_repository.name, - self.tool_shed_repository.installed_changeset_revision ) - - -class ToolVersion( object, Dictifiable ): - dict_element_visible_keys = ( 'id', 'tool_shed_repository' ) - def __init__( self, id=None, create_time=None, tool_id=None, tool_shed_repository=None ): - self.id = id - self.create_time = create_time - self.tool_id = tool_id - self.tool_shed_repository = tool_shed_repository - def get_previous_version( self, app ): - sa_session = app.model.context.current - tva = sa_session.query( app.model.ToolVersionAssociation ) \ - .filter( app.model.ToolVersionAssociation.table.c.tool_id == self.id ) \ - .first() - if tva: - return sa_session.query( app.model.ToolVersion ) \ - .filter( app.model.ToolVersion.table.c.id == tva.parent_id ) \ - .first() - return None - def get_next_version( self, app ): - sa_session = app.model.context.current - tva = sa_session.query( app.model.ToolVersionAssociation ) \ - .filter( app.model.ToolVersionAssociation.table.c.parent_id == self.id ) \ - .first() - if tva: - return sa_session.query( app.model.ToolVersion ) \ - .filter( app.model.ToolVersion.table.c.id == tva.tool_id ) \ - .first() - return None - def get_versions( self, app ): - tool_versions = [] - # Prepend ancestors. - def __ancestors( app, tool_version ): - # Should we handle multiple parents at each level? - previous_version = tool_version.get_previous_version( app ) - if previous_version: - if previous_version not in tool_versions: - tool_versions.insert( 0, previous_version ) - __ancestors( app, previous_version ) - # Append descendants. - def __descendants( app, tool_version ): - # Should we handle multiple child siblings at each level? - next_version = tool_version.get_next_version( app ) - if next_version: - if next_version not in tool_versions: - tool_versions.append( next_version ) - __descendants( app, next_version ) - __ancestors( app, self ) - if self not in tool_versions: - tool_versions.append( self ) - __descendants( app, self ) - return tool_versions - def get_version_ids( self, app, reverse=False ): - if reverse: - version_ids = [] - for tool_version in self.get_versions( app ): - version_ids.insert( 0, tool_version.tool_id ) - return version_ids - return [ tool_version.tool_id for tool_version in self.get_versions( app ) ] - - def to_dict( self, view='element' ): - rval = super( ToolVersion, self ).to_dict( view=view ) - rval['tool_name'] = self.tool_id - for a in self.parent_tool_association: - rval['parent_tool_id'] = a.parent_id - for a in self.child_tool_association: - rval['child_tool_id'] = a.tool_id - return rval - -class ToolVersionAssociation( object ): - def __init__( self, id=None, tool_id=None, parent_id=None ): - self.id = id - self.tool_id = tool_id - self.parent_id = parent_id - -class MigrateTools( object ): - def __init__( self, repository_id=None, repository_path=None, version=None ): - self.repository_id = repository_id - self.repository_path = repository_path - self.version = version - -## ---- Utility methods ------------------------------------------------------- - -def directory_hash_id( id ): - s = str( id ) - l = len( s ) - # Shortcut -- ids 0-999 go under ../000/ - if l < 4: - return [ "000" ] - # Pad with zeros until a multiple of three - padded = ( ( 3 - len( s ) % 3 ) * "0" ) + s - # Drop the last three digits -- 1000 files per directory - padded = padded[:-3] - # Break into chunks of three - return [ padded[i*3:(i+1)*3] for i in range( len( padded ) // 3 ) ] + self.user_id = user_id + self.key = key diff --git a/lib/galaxy/model/base.py b/lib/galaxy/model/base.py new file mode 100644 index 00000000000..60574a98e02 --- /dev/null +++ b/lib/galaxy/model/base.py @@ -0,0 +1,39 @@ +""" +Shared model and mapping code between Galaxy and Tool Shed, trying to +generalize to generic database connections. +""" + +from sqlalchemy.orm import scoped_session, sessionmaker +from galaxy.util.bunch import Bunch +from inspect import getmembers, isclass + + +# TODO: Refactor this to be a proper class, not a bunch. +class ModelMapping(Bunch): + + def __init__(self, model_modules, engine): + self.engine = engine + context = scoped_session( sessionmaker( autoflush=False, autocommit=True ) ) + # For backward compatibility with "context.current" + # deprecated? + context.current = context + self.context = context + self.session = context + + model_classes = {} + for module in model_modules: + m_obs = getmembers(module, isclass) + m_obs = dict([m for m in m_obs if m[1].__module__ == module.__name__]) + model_classes.update(m_obs) + + super(ModelMapping, self).__init__(**model_classes) + + context.remove() + context.configure( bind=engine ) + + @property + def Session(self): + """ + For backward compat., deprecated. + """ + return self.context diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index 183484467f4..0096c7bc405 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -1,8 +1,6 @@ from sqlalchemy.types import * -import pkg_resources -pkg_resources.require("simplejson") -import simplejson +import json import pickle import copy import uuid @@ -19,8 +17,8 @@ import logging log = logging.getLogger( __name__ ) # Default JSON encoder and decoder -json_encoder = simplejson.JSONEncoder( sort_keys=True ) -json_decoder = simplejson.JSONDecoder( ) +json_encoder = json.JSONEncoder( sort_keys=True ) +json_decoder = json.JSONDecoder( ) def _sniffnfix_pg9_hex(value): """ diff --git a/lib/galaxy/model/item_attrs.py b/lib/galaxy/model/item_attrs.py index a75927d8b91..0fe94854d3d 100644 --- a/lib/galaxy/model/item_attrs.py +++ b/lib/galaxy/model/item_attrs.py @@ -3,6 +3,8 @@ from sqlalchemy.orm.collections import InstrumentedList # Cannot import galaxy.model b/c it creates a circular import graph. import galaxy import logging +import datetime + log = logging.getLogger( __name__ ) class RuntimeException( Exception ): @@ -182,6 +184,13 @@ class Dictifiable: except: if key in value_mapper: return value_mapper.get( key )( item ) + if type(item) == datetime.datetime: + return item.isoformat() + # Leaving this for future reference, though we may want a more + # generic way to handle special type mappings going forward. + # If the item is of a class that needs to be 'stringified' before being put into a JSON data structure + # elif type(item) in []: + # return str(item) return item # Create dict to represent item. diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 94f6a0859f4..b2fad22aa47 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -3,43 +3,27 @@ Details of how the data model objects are mapped onto the relational database are encapsulated here. """ -import datetime import logging import pkg_resources -import inspect -from sqlalchemy import and_, asc, Boolean, Column, create_engine, DateTime, desc, ForeignKey, Integer, MetaData, not_, Numeric, select, String, Table, TEXT, Unicode, UniqueConstraint +from sqlalchemy import and_, asc, Boolean, Column, DateTime, desc, ForeignKey, Integer, MetaData, not_, Numeric, select, String, Table, TEXT, Unicode, UniqueConstraint from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.orderinglist import ordering_list from sqlalchemy.types import BigInteger -from sqlalchemy.orm import backref, object_session, relation, scoped_session, sessionmaker, mapper, class_mapper +from sqlalchemy.orm import backref, object_session, relation, mapper, class_mapper from sqlalchemy.orm.collections import attribute_mapped_collection from galaxy import model -from galaxy.model.orm import dialect_to_egg +from galaxy.model.orm.engine_factory import build_engine +from galaxy.model.orm.now import now from galaxy.model.custom_types import JSONType, MetadataType, TrimmedString, UUIDType +from galaxy.model.base import ModelMapping from galaxy.security import GalaxyRBACAgent -from galaxy.util.bunch import Bunch - log = logging.getLogger( __name__ ) metadata = MetaData() -context = Session = scoped_session( sessionmaker( autoflush=False, autocommit=True ) ) -# For backward compatibility with "context.current" -context.current = Session - - -# NOTE REGARDING TIMESTAMPS: -# It is currently difficult to have the timestamps calculated by the -# database in a portable way, so we're doing it in the client. This -# also saves us from needing to postfetch on postgres. HOWEVER: it -# relies on the client's clock being set correctly, so if clustering -# web servers, use a time server to ensure synchronization - -# Return the current time in UTC without any timezone information -now = datetime.datetime.utcnow model.User.table = Table( "galaxy_user", metadata, Column( "id", Integer, primary_key=True), @@ -56,6 +40,7 @@ model.User.table = Table( "galaxy_user", metadata, Column( "active", Boolean, index=True, default=True, nullable=False ), Column( "activation_token", TrimmedString( 64 ), nullable=True, index=True ) ) + model.UserAddress.table = Table( "user_address", metadata, Column( "id", Integer, primary_key=True), Column( "create_time", DateTime, default=now ), @@ -125,7 +110,10 @@ model.HistoryDatasetAssociation.table = Table( "history_dataset_association", me Column( "designation", TrimmedString( 255 ) ), Column( "deleted", Boolean, index=True, default=False ), Column( "purged", Boolean, index=True, default=False ), - Column( "visible", Boolean ) ) + Column( "visible", Boolean ), + Column( "extended_metadata_id", Integer, + ForeignKey( "extended_metadata.id" ), index=True ) + ) model.Dataset.table = Table( "dataset", metadata, Column( "id", Integer, primary_key=True ), @@ -394,67 +382,6 @@ model.LibraryDatasetDatasetInfoAssociation.table = Table( 'library_dataset_datas Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ), Column( "deleted", Boolean, index=True, default=False ) ) -model.ToolShedRepository.table = Table( "tool_shed_repository", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "tool_shed", TrimmedString( 255 ), index=True ), - Column( "name", TrimmedString( 255 ), index=True ), - Column( "description" , TEXT ), - Column( "owner", TrimmedString( 255 ), index=True ), - Column( "installed_changeset_revision", TrimmedString( 255 ) ), - Column( "changeset_revision", TrimmedString( 255 ), index=True ), - Column( "ctx_rev", TrimmedString( 10 ) ), - Column( "metadata", JSONType, nullable=True ), - Column( "includes_datatypes", Boolean, index=True, default=False ), - Column( "tool_shed_status", JSONType, nullable=True ), - Column( "deleted", Boolean, index=True, default=False ), - Column( "uninstalled", Boolean, default=False ), - Column( "dist_to_shed", Boolean, default=False ), - Column( "status", TrimmedString( 255 ) ), - Column( "error_message", TEXT ) ) - -model.RepositoryRepositoryDependencyAssociation.table = Table( 'repository_repository_dependency_association', metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True ), - Column( "repository_dependency_id", Integer, ForeignKey( "repository_dependency.id" ), index=True ) ) - -model.RepositoryDependency.table = Table( "repository_dependency", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=False ) ) - -model.ToolDependency.table = Table( "tool_dependency", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=False ), - Column( "name", TrimmedString( 255 ) ), - Column( "version", TEXT ), - Column( "type", TrimmedString( 40 ) ), - Column( "status", TrimmedString( 255 ), nullable=False ), - Column( "error_message", TEXT ) ) - -model.ToolVersion.table = Table( "tool_version", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "tool_id", String( 255 ) ), - Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=True ) ) - -model.ToolVersionAssociation.table = Table( "tool_version_association", metadata, - Column( "id", Integer, primary_key=True ), - Column( "tool_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ), - Column( "parent_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ) ) - -model.MigrateTools.table = Table( "migrate_tools", metadata, - Column( "repository_id", TrimmedString( 255 ) ), - Column( "repository_path", TEXT ), - Column( "version", Integer ) ) - model.Job.table = Table( "job", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -1126,8 +1053,11 @@ model.APIKeys.table = Table( "api_keys", metadata, Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "key", TrimmedString( 32 ), index=True, unique=True ) ) + # With the tables defined we can define the mappers and setup the # relationships between the model objects. +def simple_mapping( model, **kwds ): + mapper( model, model.table, properties=kwds ) mapper( model.Sample, model.Sample.table, @@ -1276,64 +1206,66 @@ mapper( model.UserOpenID, model.UserOpenID.table, mapper( model.ValidationError, model.ValidationError.table ) -mapper( model.HistoryDatasetAssociation, model.HistoryDatasetAssociation.table, - properties=dict( - dataset=relation( - model.Dataset, - primaryjoin=( model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id ), lazy=False ), - # .history defined in History mapper - copied_from_history_dataset_association=relation( - model.HistoryDatasetAssociation, - primaryjoin=( model.HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id == model.HistoryDatasetAssociation.table.c.id ), - remote_side=[model.HistoryDatasetAssociation.table.c.id], - uselist=False ), - copied_to_history_dataset_associations=relation( - model.HistoryDatasetAssociation, - primaryjoin=( model.HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id == model.HistoryDatasetAssociation.table.c.id ) ), - copied_from_library_dataset_dataset_association=relation( - model.LibraryDatasetDatasetAssociation, - primaryjoin=( model.HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == model.LibraryDatasetDatasetAssociation.table.c.id ), - uselist=False ), - copied_to_library_dataset_dataset_associations=relation( - model.LibraryDatasetDatasetAssociation, - primaryjoin=( model.HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == model.LibraryDatasetDatasetAssociation.table.c.id ) ), - implicitly_converted_datasets=relation( - model.ImplicitlyConvertedDatasetAssociation, - primaryjoin=( model.ImplicitlyConvertedDatasetAssociation.table.c.hda_parent_id == model.HistoryDatasetAssociation.table.c.id ) ), - implicitly_converted_parent_datasets=relation( - model.ImplicitlyConvertedDatasetAssociation, - primaryjoin=( model.ImplicitlyConvertedDatasetAssociation.table.c.hda_id == model.HistoryDatasetAssociation.table.c.id ) ), - children=relation( - model.HistoryDatasetAssociation, - primaryjoin=( model.HistoryDatasetAssociation.table.c.parent_id == model.HistoryDatasetAssociation.table.c.id ), - backref=backref( "parent", primaryjoin=( model.HistoryDatasetAssociation.table.c.parent_id == model.HistoryDatasetAssociation.table.c.id ), remote_side=[model.HistoryDatasetAssociation.table.c.id], uselist=False ) ), - visible_children=relation( - model.HistoryDatasetAssociation, - primaryjoin=( ( model.HistoryDatasetAssociation.table.c.parent_id == model.HistoryDatasetAssociation.table.c.id ) & ( model.HistoryDatasetAssociation.table.c.visible == True ) ) ), - tags=relation( model.HistoryDatasetAssociationTagAssociation, order_by=model.HistoryDatasetAssociationTagAssociation.table.c.id, backref='history_tag_associations' ), - annotations=relation( model.HistoryDatasetAssociationAnnotationAssociation, order_by=model.HistoryDatasetAssociationAnnotationAssociation.table.c.id, backref="hdas" ), - ratings=relation( model.HistoryDatasetAssociationRatingAssociation, order_by=model.HistoryDatasetAssociationRatingAssociation.table.c.id, backref="hdas" ) ) - ) +simple_mapping( model.HistoryDatasetAssociation, + dataset=relation( + model.Dataset, + primaryjoin=( model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id ), lazy=False ), + # .history defined in History mapper + copied_from_history_dataset_association=relation( + model.HistoryDatasetAssociation, + primaryjoin=( model.HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id == model.HistoryDatasetAssociation.table.c.id ), + remote_side=[model.HistoryDatasetAssociation.table.c.id], + uselist=False ), + copied_to_history_dataset_associations=relation( + model.HistoryDatasetAssociation, + primaryjoin=( model.HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id == model.HistoryDatasetAssociation.table.c.id ) ), + copied_from_library_dataset_dataset_association=relation( + model.LibraryDatasetDatasetAssociation, + primaryjoin=( model.HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == model.LibraryDatasetDatasetAssociation.table.c.id ), + uselist=False ), + copied_to_library_dataset_dataset_associations=relation( + model.LibraryDatasetDatasetAssociation, + primaryjoin=( model.HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == model.LibraryDatasetDatasetAssociation.table.c.id ) ), + implicitly_converted_datasets=relation( + model.ImplicitlyConvertedDatasetAssociation, + primaryjoin=( model.ImplicitlyConvertedDatasetAssociation.table.c.hda_parent_id == model.HistoryDatasetAssociation.table.c.id ) ), + implicitly_converted_parent_datasets=relation( + model.ImplicitlyConvertedDatasetAssociation, + primaryjoin=( model.ImplicitlyConvertedDatasetAssociation.table.c.hda_id == model.HistoryDatasetAssociation.table.c.id ) ), + children=relation( + model.HistoryDatasetAssociation, + primaryjoin=( model.HistoryDatasetAssociation.table.c.parent_id == model.HistoryDatasetAssociation.table.c.id ), + backref=backref( "parent", primaryjoin=( model.HistoryDatasetAssociation.table.c.parent_id == model.HistoryDatasetAssociation.table.c.id ), remote_side=[model.HistoryDatasetAssociation.table.c.id], uselist=False ) ), + visible_children=relation( + model.HistoryDatasetAssociation, + primaryjoin=( ( model.HistoryDatasetAssociation.table.c.parent_id == model.HistoryDatasetAssociation.table.c.id ) & ( model.HistoryDatasetAssociation.table.c.visible == True ) ) ), + tags=relation( model.HistoryDatasetAssociationTagAssociation, order_by=model.HistoryDatasetAssociationTagAssociation.table.c.id, backref='history_tag_associations' ), + annotations=relation( model.HistoryDatasetAssociationAnnotationAssociation, order_by=model.HistoryDatasetAssociationAnnotationAssociation.table.c.id, backref="hdas" ), + ratings=relation( model.HistoryDatasetAssociationRatingAssociation, order_by=model.HistoryDatasetAssociationRatingAssociation.table.c.id, backref="hdas" ), + extended_metadata=relation( + model.ExtendedMetadata, + primaryjoin=( ( model.HistoryDatasetAssociation.table.c.extended_metadata_id == model.ExtendedMetadata.table.c.id ) ) + ) +) -mapper( model.Dataset, model.Dataset.table, - properties=dict( - history_associations=relation( - model.HistoryDatasetAssociation, - primaryjoin=( model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id ) ), - active_history_associations=relation( - model.HistoryDatasetAssociation, - primaryjoin=( ( model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id ) & ( model.HistoryDatasetAssociation.table.c.deleted == False ) & ( model.HistoryDatasetAssociation.table.c.purged == False ) ) ), - purged_history_associations=relation( - model.HistoryDatasetAssociation, - primaryjoin=( ( model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id ) & ( model.HistoryDatasetAssociation.table.c.purged == True ) ) ), - library_associations=relation( - model.LibraryDatasetDatasetAssociation, - primaryjoin=( model.Dataset.table.c.id == model.LibraryDatasetDatasetAssociation.table.c.dataset_id ) ), - active_library_associations=relation( - model.LibraryDatasetDatasetAssociation, - primaryjoin=( ( model.Dataset.table.c.id == model.LibraryDatasetDatasetAssociation.table.c.dataset_id ) & ( model.LibraryDatasetDatasetAssociation.table.c.deleted == False ) ) ), - tags=relation(model.DatasetTagAssociation, order_by=model.DatasetTagAssociation.table.c.id, backref='datasets') - ) ) +simple_mapping( model.Dataset, + history_associations=relation( + model.HistoryDatasetAssociation, + primaryjoin=( model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id ) ), + active_history_associations=relation( + model.HistoryDatasetAssociation, + primaryjoin=( ( model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id ) & ( model.HistoryDatasetAssociation.table.c.deleted == False ) & ( model.HistoryDatasetAssociation.table.c.purged == False ) ) ), + purged_history_associations=relation( + model.HistoryDatasetAssociation, + primaryjoin=( ( model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id ) & ( model.HistoryDatasetAssociation.table.c.purged == True ) ) ), + library_associations=relation( + model.LibraryDatasetDatasetAssociation, + primaryjoin=( model.Dataset.table.c.id == model.LibraryDatasetDatasetAssociation.table.c.dataset_id ) ), + active_library_associations=relation( + model.LibraryDatasetDatasetAssociation, + primaryjoin=( ( model.Dataset.table.c.id == model.LibraryDatasetDatasetAssociation.table.c.dataset_id ) & ( model.LibraryDatasetDatasetAssociation.table.c.deleted == False ) ) ), + tags=relation(model.DatasetTagAssociation, order_by=model.DatasetTagAssociation.table.c.id, backref='datasets') +) mapper( model.HistoryDatasetAssociationDisplayAtAuthorization, model.HistoryDatasetAssociationDisplayAtAuthorization.table, properties=dict( history_dataset_association = relation( model.HistoryDatasetAssociation ), @@ -1363,6 +1295,7 @@ mapper( model.ImplicitlyConvertedDatasetAssociation, model.ImplicitlyConvertedDa mapper( model.History, model.History.table, properties=dict( galaxy_sessions=relation( model.GalaxySessionToHistoryAssociation ), datasets=relation( model.HistoryDatasetAssociation, backref="history", order_by=asc(model.HistoryDatasetAssociation.table.c.hid) ), + exports=relation( model.JobExportHistoryArchive, primaryjoin=( model.JobExportHistoryArchive.table.c.history_id == model.History.table.c.id ), order_by=desc( model.JobExportHistoryArchive.table.c.id ) ), active_datasets=relation( model.HistoryDatasetAssociation, primaryjoin=( ( model.HistoryDatasetAssociation.table.c.history_id == model.History.table.c.id ) & not_( model.HistoryDatasetAssociation.table.c.deleted ) ), order_by=asc( model.HistoryDatasetAssociation.table.c.hid ), viewonly=True ), visible_datasets=relation( model.HistoryDatasetAssociation, primaryjoin=( ( model.HistoryDatasetAssociation.table.c.history_id == model.History.table.c.id ) & not_( model.HistoryDatasetAssociation.table.c.deleted ) & model.HistoryDatasetAssociation.table.c.visible ), order_by=asc( model.HistoryDatasetAssociation.table.c.hid ), viewonly=True ), @@ -1791,40 +1724,6 @@ mapper( model.Page, model.Page.table, ratings=relation( model.PageRatingAssociation, order_by=model.PageRatingAssociation.table.c.id, backref="pages" ) ) ) -mapper( model.ToolShedRepository, model.ToolShedRepository.table, - properties=dict( tool_versions=relation( model.ToolVersion, - primaryjoin=( model.ToolShedRepository.table.c.id == model.ToolVersion.table.c.tool_shed_repository_id ), - backref='tool_shed_repository' ), - tool_dependencies=relation( model.ToolDependency, - primaryjoin=( model.ToolShedRepository.table.c.id == model.ToolDependency.table.c.tool_shed_repository_id ), - order_by=model.ToolDependency.table.c.name, - backref='tool_shed_repository' ), - required_repositories=relation( model.RepositoryRepositoryDependencyAssociation, - primaryjoin=( model.ToolShedRepository.table.c.id == model.RepositoryRepositoryDependencyAssociation.table.c.tool_shed_repository_id ) ) ) ) - -mapper( model.RepositoryRepositoryDependencyAssociation, model.RepositoryRepositoryDependencyAssociation.table, - properties=dict( repository=relation( model.ToolShedRepository, - primaryjoin=( model.RepositoryRepositoryDependencyAssociation.table.c.tool_shed_repository_id == model.ToolShedRepository.table.c.id ) ), - repository_dependency=relation( model.RepositoryDependency, - primaryjoin=( model.RepositoryRepositoryDependencyAssociation.table.c.repository_dependency_id == model.RepositoryDependency.table.c.id ) ) ) ) - -mapper( model.RepositoryDependency, model.RepositoryDependency.table, - properties=dict( repository=relation( model.ToolShedRepository, - primaryjoin=( model.RepositoryDependency.table.c.tool_shed_repository_id == model.ToolShedRepository.table.c.id ) ) ) ) - -mapper( model.ToolDependency, model.ToolDependency.table ) - -mapper( model.ToolVersion, model.ToolVersion.table, - properties=dict( - parent_tool_association=relation( model.ToolVersionAssociation, - primaryjoin=( model.ToolVersion.table.c.id == model.ToolVersionAssociation.table.c.tool_id ) ), - child_tool_association=relation( model.ToolVersionAssociation, - primaryjoin=( model.ToolVersion.table.c.id == model.ToolVersionAssociation.table.c.parent_id ) ) - ) - ) - -mapper( model.ToolVersionAssociation, model.ToolVersionAssociation.table ) - # Set up proxy so that # Page.users_shared_with # returns a list of users that page is shared with. @@ -1861,90 +1760,63 @@ mapper( model.VisualizationUserShareAssociation, model.VisualizationUserShareAss ) ) # Tag tables. +simple_mapping( model.Tag, + children=relation(model.Tag, backref=backref( 'parent', remote_side=[model.Tag.table.c.id] ) ) +) -mapper( model.Tag, model.Tag.table, - properties=dict( children=relation(model.Tag, backref=backref( 'parent', remote_side=[model.Tag.table.c.id] ) ) - ) ) -mapper( model.HistoryTagAssociation, model.HistoryTagAssociation.table, - properties=dict( tag=relation(model.Tag, backref="tagged_histories"), user=relation( model.User ) ) - ) +def tag_mapping( tag_association_class, backref_name ): + simple_mapping( tag_association_class, tag=relation( model.Tag, backref=backref_name), user=relation( model.User ) ) -mapper( model.DatasetTagAssociation, model.DatasetTagAssociation.table, - properties=dict( tag=relation(model.Tag, backref="tagged_datasets"), user=relation( model.User ) ) - ) +tag_mapping( model.HistoryTagAssociation, "tagged_histories" ) -mapper( model.HistoryDatasetAssociationTagAssociation, model.HistoryDatasetAssociationTagAssociation.table, - properties=dict( tag=relation(model.Tag, backref="tagged_history_dataset_associations"), user=relation( model.User ) ) - ) +tag_mapping( model.DatasetTagAssociation, "tagged_datasets" ) -mapper( model.PageTagAssociation, model.PageTagAssociation.table, - properties=dict( tag=relation(model.Tag, backref="tagged_pages"), user=relation( model.User ) ) - ) +tag_mapping( model.HistoryDatasetAssociationTagAssociation, "tagged_history_dataset_associations" ) -mapper( model.StoredWorkflowTagAssociation, model.StoredWorkflowTagAssociation.table, - properties=dict( tag=relation(model.Tag, backref="tagged_workflows"), user=relation( model.User ) ) - ) +tag_mapping( model.PageTagAssociation, "tagged_pages" ) -mapper( model.WorkflowStepTagAssociation, model.WorkflowStepTagAssociation.table, - properties=dict( tag=relation(model.Tag, backref="tagged_workflow_steps"), user=relation( model.User ) ) - ) +tag_mapping( model.StoredWorkflowTagAssociation, "tagged_workflows" ) -mapper( model.VisualizationTagAssociation, model.VisualizationTagAssociation.table, - properties=dict( tag=relation(model.Tag, backref="tagged_visualizations"), user=relation( model.User ) ) - ) +tag_mapping( model.WorkflowStepTagAssociation, "tagged_workflow_steps" ) + +tag_mapping( model.VisualizationTagAssociation, "tagged_visualizations" ) + +tag_mapping( model.ToolTagAssociation, "tagged_tools" ) -mapper( model.ToolTagAssociation, model.ToolTagAssociation.table, - properties=dict( tag=relation(model.Tag, backref="tagged_tools"), user=relation( model.User ) ) - ) # Annotation tables. +def annotation_mapping( annotation_class, **kwds ): + kwds = dict( [ (key, relation( value ) ) for key, value in kwds.iteritems() ] ) + simple_mapping( annotation_class, **dict(user=relation( model.User ), **kwds ) ) -mapper( model.HistoryAnnotationAssociation, model.HistoryAnnotationAssociation.table, - properties=dict( history=relation( model.History ), user=relation( model.User ) ) - ) +annotation_mapping( model.HistoryAnnotationAssociation, history=model.History ) -mapper( model.HistoryDatasetAssociationAnnotationAssociation, model.HistoryDatasetAssociationAnnotationAssociation.table, - properties=dict( hda=relation( model.HistoryDatasetAssociation ), user=relation( model.User ) ) - ) +annotation_mapping( model.HistoryDatasetAssociationAnnotationAssociation, hda=model.HistoryDatasetAssociation ) -mapper( model.StoredWorkflowAnnotationAssociation, model.StoredWorkflowAnnotationAssociation.table, - properties=dict( stored_workflow=relation( model.StoredWorkflow ), user=relation( model.User ) ) - ) +annotation_mapping( model.StoredWorkflowAnnotationAssociation, stored_workflow=model.StoredWorkflow ) -mapper( model.WorkflowStepAnnotationAssociation, model.WorkflowStepAnnotationAssociation.table, - properties=dict( workflow_step=relation( model.WorkflowStep ), user=relation( model.User ) ) - ) +annotation_mapping( model.WorkflowStepAnnotationAssociation, workflow_step=model.WorkflowStep ) -mapper( model.PageAnnotationAssociation, model.PageAnnotationAssociation.table, - properties=dict( page=relation( model.Page ), user=relation( model.User ) ) - ) +annotation_mapping( model.PageAnnotationAssociation, page=model.Page ) + +annotation_mapping( model.VisualizationAnnotationAssociation, visualization=model.Visualization ) -mapper( model.VisualizationAnnotationAssociation, model.VisualizationAnnotationAssociation.table, - properties=dict( visualization=relation( model.Visualization ), user=relation( model.User ) ) - ) # Rating tables. +def rating_mapping( rating_class, **kwds ): + kwds = dict( [ (key, relation( value ) ) for key, value in kwds.iteritems() ] ) + simple_mapping( rating_class, **dict(user=relation( model.User ), **kwds ) ) -mapper( model.HistoryRatingAssociation, model.HistoryRatingAssociation.table, - properties=dict( history=relation( model.History ), user=relation( model.User ) ) - ) +rating_mapping( model.HistoryRatingAssociation, history=model.History ) -mapper( model.HistoryDatasetAssociationRatingAssociation, model.HistoryDatasetAssociationRatingAssociation.table, - properties=dict( hda=relation( model.HistoryDatasetAssociation ), user=relation( model.User ) ) - ) +rating_mapping( model.HistoryDatasetAssociationRatingAssociation, hda=model.HistoryDatasetAssociation ) -mapper( model.StoredWorkflowRatingAssociation, model.StoredWorkflowRatingAssociation.table, - properties=dict( stored_workflow=relation( model.StoredWorkflow ), user=relation( model.User ) ) - ) +rating_mapping( model.StoredWorkflowRatingAssociation, stored_workflow=model.StoredWorkflow ) -mapper( model.PageRatingAssociation, model.PageRatingAssociation.table, - properties=dict( page=relation( model.Page ), user=relation( model.User ) ) - ) +rating_mapping( model.PageRatingAssociation, page=model.Page ) -mapper( model.VisualizationRatingAssociation, model.VisualizationRatingAssociation.table, - properties=dict( visualization=relation( model.Visualization ), user=relation( model.User ) ) - ) +rating_mapping( model.VisualizationRatingAssociation, visualizaiton=model.Visualization ) #Data Manager tables mapper( model.DataManagerHistoryAssociation, model.DataManagerHistoryAssociation.table, @@ -1981,8 +1853,14 @@ class_mapper(model.LibraryDatasetDatasetAssociation).add_property( "creating_job def db_next_hid( self ): """ - Override __next_hid to generate from the database in a concurrency - safe way. + db_next_hid( self ) + + Override __next_hid to generate from the database in a concurrency safe way. + Loads the next history ID from the DB and returns it. + It also saves the future next_id into the DB. + + :rtype: int + :returns: the next history id """ conn = object_session( self ).connection() table = self.table @@ -1998,25 +1876,8 @@ def db_next_hid( self ): model.History._next_hid = db_next_hid -def guess_dialect_for_url( url ): - return (url.split(':', 1))[0] -def load_egg_for_url( url ): - # Load the appropriate db module - dialect = guess_dialect_for_url( url ) - try: - egg = dialect_to_egg[dialect] - try: - pkg_resources.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module's in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) - -def init( file_path, url, engine_options={}, create_tables=False, database_query_profiling_proxy=False, object_store=None, trace_logger=None, use_pbkdf2=True ): +def init( file_path, url, engine_options={}, create_tables=False, map_install_models=False, database_query_profiling_proxy=False, object_store=None, trace_logger=None, use_pbkdf2=True ): """Connect mappings to the database""" # Connect dataset to the file path model.Dataset.file_path = file_path @@ -2025,45 +1886,25 @@ def init( file_path, url, engine_options={}, create_tables=False, database_query # Use PBKDF2 password hashing? model.User.use_pbkdf2 = use_pbkdf2 # Load the appropriate db module - load_egg_for_url( url ) - # Should we use the logging proxy? - if database_query_profiling_proxy: - import galaxy.model.orm.logging_connection_proxy as logging_connection_proxy - proxy = logging_connection_proxy.LoggingProxy() - # If metlog is enabled, do micrologging - elif trace_logger: - import galaxy.model.orm.logging_connection_proxy as logging_connection_proxy - proxy = logging_connection_proxy.TraceLoggerProxy( trace_logger ) - else: - proxy = None - # Create the database engine - engine = create_engine( url, proxy=proxy, **engine_options ) + engine = build_engine( url, engine_options, database_query_profiling_proxy, trace_logger ) + # Connect the metadata to the database. metadata.bind = engine - # Clear any existing contextual sessions and reconfigure - Session.remove() - Session.configure( bind=engine ) + + model_modules = [model] + if map_install_models: + import galaxy.model.tool_shed_install.mapping + from galaxy.model import tool_shed_install + model_modules.append(tool_shed_install) + + result = ModelMapping(model_modules, engine=engine) + # Create tables if needed if create_tables: metadata.create_all() # metadata.engine.commit() - # Pack everything into a bunch -- inspecting should work better than - # grabbing everything in globals(), but it's possible that there's code - # working somewhere that expects random stuff we'd incidentally included. - m_obs = inspect.getmembers(model, inspect.isclass) - m_obs = dict([m for m in m_obs if m[1].__module__ == 'galaxy.model']) - result = Bunch( **m_obs ) - result.engine = engine - result.session = Session - # For backward compatibility with "model.context.current" - result.context = Session + result.create_tables = create_tables #load local galaxy security policy result.security_agent = GalaxyRBACAgent( result ) return result - -def get_suite(): - """Get unittest suite for this module""" - import unittest, mapping_tests - return unittest.makeSuite( mapping_tests.MappingTests ) - diff --git a/lib/galaxy/model/mapping_tests.py b/lib/galaxy/model/mapping_tests.py deleted file mode 100644 index 951da3b4c47..00000000000 --- a/lib/galaxy/model/mapping_tests.py +++ /dev/null @@ -1,54 +0,0 @@ -import unittest -import galaxy.model.mapping as mapping -from galaxy.model import directory_hash_id -import os.path - -class MappingTests( unittest.TestCase ): - def test_basic( self ): - # Start the database and connect the mapping - model = mapping.init( "/tmp", "sqlite:///:memory:", create_tables=True ) - assert model.engine is not None - # Make some changes and commit them - u = model.User( email="james@foo.bar.baz", password="password" ) - # gs = model.GalaxySession() - h1 = model.History( name="History 1", user=u) - #h1.queries.append( model.Query( "h1->q1" ) ) - #h1.queries.append( model.Query( "h1->q2" ) ) - h2 = model.History( name=( "H" * 1024 ) ) - model.session.add_all( ( u, h1, h2 ) ) - #q1 = model.Query( "h2->q1" ) - d1 = model.HistoryDatasetAssociation( extension="interval", metadata=dict(chromCol=1,startCol=2,endCol=3 ), history=h2, create_dataset=True, sa_session=model.session ) - #h2.queries.append( q1 ) - #h2.queries.append( model.Query( "h2->q2" ) ) - model.session.add( ( d1 ) ) - model.session.flush() - model.session.expunge_all() - # Check - users = model.session.query( model.User ).all() - assert len( users ) == 1 - assert users[0].email == "james@foo.bar.baz" - assert users[0].password == "password" - assert len( users[0].histories ) == 1 - assert users[0].histories[0].name == "History 1" - hists = model.session.query( model.History ).all() - assert hists[0].name == "History 1" - assert hists[1].name == ( "H" * 255 ) - assert hists[0].user == users[0] - assert hists[1].user is None - assert hists[1].datasets[0].metadata.chromCol == 1 - # The filename test has moved to objecstore - #id = hists[1].datasets[0].id - #assert hists[1].datasets[0].file_name == os.path.join( "/tmp", *directory_hash_id( id ) ) + ( "/dataset_%d.dat" % id ) - # Do an update and check - hists[1].name = "History 2b" - model.session.flush() - model.session.expunge_all() - hists = model.session.query( model.History ).all() - assert hists[0].name == "History 1" - assert hists[1].name == "History 2b" - # gvk TODO need to ad test for GalaxySessions, but not yet sure what they should look like. - -def get_suite(): - suite = unittest.TestSuite() - suite.addTest( MappingTests( "test_basic" ) ) - return suite diff --git a/lib/galaxy/model/migrate/check.py b/lib/galaxy/model/migrate/check.py index dff7951cf4c..dc6283e6fc0 100644 --- a/lib/galaxy/model/migrate/check.py +++ b/lib/galaxy/model/migrate/check.py @@ -57,7 +57,7 @@ def create_or_verify_database( url, galaxy_config_file, engine_options={}, app=N migrate_to_current_version( engine, db_schema ) meta = MetaData( bind=engine ) - if getattr( app.config, 'database_auto_migrate', False ): + if app and getattr( app.config, 'database_auto_migrate', False ): migrate() return diff --git a/lib/galaxy/model/migrate/versions/0118_add_hda_extended_metadata.py b/lib/galaxy/model/migrate/versions/0118_add_hda_extended_metadata.py new file mode 100644 index 00000000000..879f5ad8a41 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0118_add_hda_extended_metadata.py @@ -0,0 +1,48 @@ +""" +Add link from history_dataset_association to the extended_metadata table +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +from galaxy.model.custom_types import JSONType + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData() + +extended_metadata_hda_col = Column( "extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), nullable=True ) + + +def display_migration_details(): + print "This migration script adds a ExtendedMetadata links to HistoryDatasetAssociation tables" + +def upgrade(migrate_engine): + print __doc__ + metadata.bind = migrate_engine + metadata.reflect() + + try: + hda_table = Table( "history_dataset_association", metadata, autoload=True ) + extended_metadata_hda_col.create( hda_table ) + assert extended_metadata_hda_col is hda_table.c.extended_metadata_id + except Exception, e: + print str(e) + log.error( "Adding column 'extended_metadata_id' to history_dataset_association table failed: %s" % str( e ) ) + + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + + # Drop the HDA table's extended metadata ID column. + try: + hda_table = Table( "history_dataset_association", metadata, autoload=True ) + extended_metadata_id = hda_table.c.extended_metadata_id + extended_metadata_id.drop() + except Exception, e: + log.debug( "Dropping 'extended_metadata_id' column from history_dataset_association table failed: %s" % ( str( e ) ) ) + + diff --git a/lib/galaxy/model/orm/__init__.py b/lib/galaxy/model/orm/__init__.py index fe6b773cf3d..f7470334cab 100644 --- a/lib/galaxy/model/orm/__init__.py +++ b/lib/galaxy/model/orm/__init__.py @@ -4,13 +4,34 @@ pkg_resources.require( "SQLAlchemy >= 0.4" ) from sqlalchemy import * from sqlalchemy.orm import * from sqlalchemy.interfaces import * -import sqlalchemy.exc -from sqlalchemy.ext.orderinglist import ordering_list +import logging +log = logging.getLogger( __name__ ) + dialect_to_egg = { - "sqlite" : "pysqlite>=2", - "postgres" : "psycopg2", - "postgresql" : "psycopg2", - "mysql" : "MySQL_python" + "sqlite": "pysqlite>=2", + "postgres": "psycopg2", + "postgresql": "psycopg2", + "mysql": "MySQL_python" } + + +def load_egg_for_url( url ): + # Load the appropriate db module + dialect = __guess_dialect_for_url( url ) + try: + egg = dialect_to_egg[dialect] + try: + pkg_resources.require( egg ) + log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) + except: + # If the module's in the path elsewhere (i.e. non-egg), it'll still load. + log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) + except KeyError: + # Let this go, it could possibly work with db's we don't support + log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) + + +def __guess_dialect_for_url( url ): + return (url.split(':', 1))[0] diff --git a/lib/galaxy/model/orm/engine_factory.py b/lib/galaxy/model/orm/engine_factory.py new file mode 100644 index 00000000000..7ddaafd4861 --- /dev/null +++ b/lib/galaxy/model/orm/engine_factory.py @@ -0,0 +1,24 @@ +import logging +log = logging.getLogger( __name__ ) + +from sqlalchemy import create_engine +from galaxy.model.orm import load_egg_for_url + + +def build_engine(url, engine_options, database_query_profiling_proxy=False, trace_logger=None): + load_egg_for_url( url ) + + # Should we use the logging proxy? + if database_query_profiling_proxy: + import galaxy.model.orm.logging_connection_proxy as logging_connection_proxy + proxy = logging_connection_proxy.LoggingProxy() + # If metlog is enabled, do micrologging + elif trace_logger: + import galaxy.model.orm.logging_connection_proxy as logging_connection_proxy + proxy = logging_connection_proxy.TraceLoggerProxy( trace_logger ) + else: + proxy = None + + # Create the database engine + engine = create_engine( url, proxy=proxy, **engine_options ) + return engine diff --git a/lib/galaxy/model/orm/now.py b/lib/galaxy/model/orm/now.py new file mode 100644 index 00000000000..b6b4de8fa0d --- /dev/null +++ b/lib/galaxy/model/orm/now.py @@ -0,0 +1,13 @@ +from datetime import datetime + +# NOTE REGARDING TIMESTAMPS: +# It is currently difficult to have the timestamps calculated by the +# database in a portable way, so we're doing it in the client. This +# also saves us from needing to postfetch on postgres. HOWEVER: it +# relies on the client's clock being set correctly, so if clustering +# web servers, use a time server to ensure synchronization + +# Return the current time in UTC without any timezone information +now = datetime.utcnow + +__all__ = [now] diff --git a/lib/galaxy/model/orm/scripts.py b/lib/galaxy/model/orm/scripts.py new file mode 100644 index 00000000000..2078fd2beda --- /dev/null +++ b/lib/galaxy/model/orm/scripts.py @@ -0,0 +1,119 @@ +""" +Code to support database helper scripts (create_db.py, manage_db.py, etc...). +""" +import logging +import os.path +from ConfigParser import SafeConfigParser + +from galaxy import eggs + +eggs.require( "decorator" ) +eggs.require( "Tempita" ) +eggs.require( "SQLAlchemy" ) +eggs.require( "sqlalchemy_migrate" ) + +from galaxy.model.orm import dialect_to_egg + +import pkg_resources + +log = logging.getLogger( __name__ ) + +DEFAULT_CONFIG_FILE = 'universe_wsgi.ini' +DEFAULT_CONFIG_PREFIX = '' +DEFAULT_DATABASE = 'galaxy' + +DATABASE = { + "galaxy": + { + 'repo': 'lib/galaxy/model/migrate', + 'default_sqlite_file': './database/universe.sqlite', + }, + "tool_shed": + { + 'repo': 'lib/galaxy/webapps/tool_shed/model/migrate', + 'config_file': 'tool_shed_wsgi.ini', + 'default_sqlite_file': './database/community.sqlite', + }, + "install": + { + 'repo': 'lib/galaxy/model/tool_shed_install/migrate', + 'config_prefix': 'install_', + 'default_sqlite_file': './database/install.sqlite', + }, +} + + +def require_dialect_egg( db_url ): + dialect = ( db_url.split( ':', 1 ) )[0] + try: + egg = dialect_to_egg[dialect] + try: + pkg_resources.require( egg ) + log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) + except: + # If the module is in the path elsewhere (i.e. non-egg), it'll still load. + log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) + except KeyError: + # Let this go, it could possibly work with db's we don't support + log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) + + +def read_config_file_arg( argv, default ): + if '-c' in argv: + pos = argv.index( '-c' ) + argv.pop(pos) + config_file = argv.pop( pos ) + else: + config_file = default + return config_file + + +def get_config( argv, cwd=None ): + """ + Read sys.argv and parse out repository of migrations and database url. + + >>> from tempfile import mkdtemp + >>> config_dir = mkdtemp() + >>> def write_ini(path, property, value): + ... p = SafeConfigParser() + ... p.add_section('app:main') + ... p.set('app:main', property, value) + ... with open(os.path.join(config_dir, path), 'w') as f: p.write(f) + >>> write_ini('tool_shed_wsgi.ini', 'database_connection', 'sqlite:///pg/testdb1') + >>> config = get_config(['manage_db.py', 'tool_shed'], cwd=config_dir) + >>> config['repo'] + 'lib/galaxy/webapps/tool_shed/model/migrate' + >>> config['db_url'] + 'sqlite:///pg/testdb1' + >>> write_ini('universe_wsgi.ini', 'database_file', 'moo.sqlite') + >>> config = get_config(['manage_db.py'], cwd=config_dir) + >>> config['db_url'] + 'sqlite:///moo.sqlite?isolation_level=IMMEDIATE' + >>> config['repo'] + 'lib/galaxy/model/migrate' + """ + if argv and (argv[-1] in DATABASE): + database = argv.pop() # database name tool_shed, galaxy, or install. + else: + database = 'galaxy' + database_defaults = DATABASE[ database ] + + config_file = read_config_file_arg( argv, database_defaults.get( 'config_file', DEFAULT_CONFIG_FILE ) ) + repo = database_defaults[ 'repo' ] + config_prefix = database_defaults.get( 'config_prefix', DEFAULT_CONFIG_PREFIX ) + default_sqlite_file = database_defaults[ 'default_sqlite_file' ] + if cwd: + config_file = os.path.join( cwd, config_file ) + + cp = SafeConfigParser() + cp.read( config_file ) + + if cp.has_option( "app:main", "%sdatabase_connection" % config_prefix): + db_url = cp.get( "app:main", "%sdatabase_connection" % config_prefix ) + elif cp.has_option( "app:main", "%sdatabase_file" % config_prefix ): + db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % cp.get( "app:main", "database_file" ) + else: + db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % default_sqlite_file + + require_dialect_egg( db_url ) + return dict(db_url=db_url, repo=repo, config_file=config_file, database=database) diff --git a/lib/galaxy/model/search.py b/lib/galaxy/model/search.py index ec22c5f5f3f..5040a2814a3 100644 --- a/lib/galaxy/model/search.py +++ b/lib/galaxy/model/search.py @@ -32,10 +32,12 @@ eggs.require("Parsley") import parsley from galaxy.model import (HistoryDatasetAssociation, LibraryDatasetDatasetAssociation, -History, Library, LibraryFolder, LibraryDataset,StoredWorkflowTagAssociation, -StoredWorkflow, HistoryTagAssociation,HistoryDatasetAssociationTagAssociation, +History, Library, LibraryFolder, LibraryDataset, StoredWorkflowTagAssociation, +StoredWorkflow, HistoryTagAssociation, HistoryDatasetAssociationTagAssociation, ExtendedMetadata, ExtendedMetadataIndex, HistoryAnnotationAssociation, Job, JobParameter, -JobToInputLibraryDatasetAssociation, JobToInputDatasetAssociation, JobToOutputDatasetAssociation, ToolVersion ) +JobToInputLibraryDatasetAssociation, JobToInputDatasetAssociation, JobToOutputDatasetAssociation, +Page, PageRevision) +from galaxy.model.tool_shed_install import ToolVersion from galaxy.util.json import to_json_string from sqlalchemy import and_ @@ -93,7 +95,6 @@ class ViewQueryBaseClass(object): if field.id_decode: conditional.right = trans.security.decode_id( conditional.right ) - def filter(self, left, operator, right): if operator == 'and': self.filter(left.left, left.operator, left.right) @@ -104,13 +105,15 @@ class ViewQueryBaseClass(object): self.do_query = True field = self.FIELDS[left_base] if field.sqlalchemy_field is not None: + clazz, attribute = field.sqlalchemy_field + sqlalchemy_field_value = getattr(clazz, attribute) if operator == "=": #print field.sqlalchemy_field == right, field.sqlalchemy_field, right - self.query = self.query.filter( field.sqlalchemy_field == right ) + self.query = self.query.filter( sqlalchemy_field_value == right ) elif operator == "!=": - self.query = self.query.filter( field.sqlalchemy_field != right ) + self.query = self.query.filter( sqlalchemy_field_value != right ) elif operator == "like": - self.query = self.query.filter( field.sqlalchemy_field.like(right) ) + self.query = self.query.filter( sqlalchemy_field_value.like(right) ) else: raise GalaxyParseError("Invalid comparison operator: %s" % (operator)) elif field.handler is not None: @@ -131,7 +134,7 @@ class ViewQueryBaseClass(object): for row in self.query.distinct().all(): selected = True for f in self.post_filter: - if not f[0](row, f[1],f[2],f[3]): + if not f[0](row, f[1], f[2], f[3]): selected = False if selected: yield row @@ -141,6 +144,7 @@ class ViewQueryBaseClass(object): #Library Dataset Searching ################## + def library_extended_metadata_filter(view, left, operator, right): view.do_query = True if 'extended_metadata_joined' not in view.state: @@ -157,6 +161,7 @@ def library_extended_metadata_filter(view, left, operator, right): ) ) + def ldda_parent_library_filter(item, left, operator, right): if operator == '=': return right == item.library_dataset.folder.parent_library.id @@ -168,12 +173,12 @@ def ldda_parent_library_filter(item, left, operator, right): class LibraryDatasetDatasetView(ViewQueryBaseClass): VIEW_NAME = "library_dataset_dataset" FIELDS = { - 'extended_metadata' : ViewField('extended_metadata', handler=library_extended_metadata_filter), - 'name' : ViewField('name', sqlalchemy_field=LibraryDatasetDatasetAssociation.name ), - 'id' : ViewField('id', sqlalchemy_field=LibraryDatasetDatasetAssociation.id, id_decode=True), - 'deleted' : ViewField('deleted', sqlalchemy_field=LibraryDatasetDatasetAssociation.deleted), - 'parent_library_id' : ViewField('parent_library_id', id_decode=True, post_filter=ldda_parent_library_filter), - 'data_type' : ViewField('data_type', sqlalchemy_field=LibraryDatasetDatasetAssociation.extension) + 'extended_metadata': ViewField('extended_metadata', handler=library_extended_metadata_filter), + 'name': ViewField('name', sqlalchemy_field=(LibraryDatasetDatasetAssociation, "name")), + 'id': ViewField('id', sqlalchemy_field=LibraryDatasetDatasetAssociation.id, id_decode=True), + 'deleted': ViewField('deleted', sqlalchemy_field=(LibraryDatasetDatasetAssociation, "deleted")), + 'parent_library_id': ViewField('parent_library_id', id_decode=True, post_filter=ldda_parent_library_filter), + 'data_type': ViewField('data_type', sqlalchemy_field=(LibraryDatasetDatasetAssociation, "extension")), } def search(self, trans): @@ -187,20 +192,18 @@ class LibraryDatasetDatasetView(ViewQueryBaseClass): class LibraryView(ViewQueryBaseClass): VIEW_NAME = "library" FIELDS = { - 'name' : ViewField('name', sqlalchemy_field=Library.name ), - 'id' : ViewField('id', sqlalchemy_field=Library.id, id_decode=True), - 'deleted' : ViewField('deleted', sqlalchemy_field=Library.deleted) + 'name': ViewField('name', sqlalchemy_field=(Library, "name")), + 'id': ViewField('id', sqlalchemy_field=Library.id, id_decode=True), + 'deleted': ViewField('deleted', sqlalchemy_field=(Library, "deleted")), } def search(self, trans): self.query = trans.sa_session.query( Library ) - ################## #Library Folder Searching ################## - def library_folder_parent_library_id_filter(item, left, operator, right): if operator == '=': return item.parent_library.id == right @@ -208,6 +211,7 @@ def library_folder_parent_library_id_filter(item, left, operator, right): return item.parent_library.id != right raise GalaxyParseError("Invalid comparison operator: %s" % (operator)) + def library_path_filter(item, left, operator, right): lpath = "/" + "/".join(item.library_path) if operator == '=': @@ -220,22 +224,20 @@ def library_path_filter(item, left, operator, right): class LibraryFolderView(ViewQueryBaseClass): VIEW_NAME = "library_folder" FIELDS = { - 'name' : ViewField('name', sqlalchemy_field=LibraryFolder.name ), - 'id' : ViewField('id', sqlalchemy_field=LibraryFolder.id, id_decode=True), - 'parent_id' : ViewField('parent_id', sqlalchemy_field=LibraryFolder.parent_id, id_decode=True ), - 'parent_library_id' : ViewField('parent_library_id', post_filter=library_folder_parent_library_id_filter, id_decode=True), - 'library_path' : ViewField('library_path', post_filter=library_path_filter) + 'name': ViewField('name', sqlalchemy_field=(LibraryFolder, "name")), + 'id': ViewField('id', sqlalchemy_field=(LibraryFolder, "id"), id_decode=True), + 'parent_id': ViewField('parent_id', sqlalchemy_field=(LibraryFolder, "parent_id"), id_decode=True), + 'parent_library_id': ViewField('parent_library_id', post_filter=library_folder_parent_library_id_filter, id_decode=True), + 'library_path': ViewField('library_path', post_filter=library_path_filter) } def search(self, trans): self.query = trans.sa_session.query( LibraryFolder ) - ################## #Library Dataset Searching ################## - def library_dataset_name_filter(item, left, operator, right): if operator == '=': return item.name == right @@ -247,34 +249,32 @@ def library_dataset_name_filter(item, left, operator, right): class LibraryDatasetView(ViewQueryBaseClass): VIEW_NAME = "library_dataset" FIELDS = { - 'name' : ViewField('name', post_filter=library_dataset_name_filter), - 'id' : ViewField('id', sqlalchemy_field=LibraryDataset.id, id_decode=True), - 'folder_id' : ViewField('folder_id', sqlalchemy_field=LibraryDataset.folder_id, id_decode=True) + 'name': ViewField('name', post_filter=library_dataset_name_filter), + 'id': ViewField('id', sqlalchemy_field=(LibraryDataset, "id"), id_decode=True), + 'folder_id': ViewField('folder_id', sqlalchemy_field=(LibraryDataset, "folder_id"), id_decode=True) } def search(self, trans): self.query = trans.sa_session.query( LibraryDataset ) - ################## #Tool Searching ################## - class ToolView(ViewQueryBaseClass): VIEW_NAME = "tool" FIELDS = { - 'tool_id' : ViewField('name', sqlalchemy_field=ToolVersion.tool_id ), - 'id' : ViewField('id', sqlalchemy_field=ToolVersion.id) + 'tool_id': ViewField('name', sqlalchemy_field=(ToolVersion, "tool_id")), + 'id': ViewField('id', sqlalchemy_field=(ToolVersion, "id")), } def search(self, trans): - self.query = trans.sa_session.query( ToolVersion ) + self.query = trans.install_model.context.query( ToolVersion ) + ################## #History Dataset Searching ################## - def history_dataset_handle_tag(view, left, operator, right): if operator == "=": view.do_query = True @@ -292,16 +292,39 @@ def history_dataset_handle_tag(view, left, operator, right): raise GalaxyParseError("Invalid comparison operator: %s" % (operator)) +def history_dataset_extended_metadata_filter(view, left, operator, right): + view.do_query = True + if 'extended_metadata_joined' not in view.state: + view.query = view.query.join( ExtendedMetadata ) + view.state['extended_metadata_joined'] = True + alias = aliased( ExtendedMetadataIndex ) + field = "/%s" % ("/".join(left.split(".")[1:])) + #print "FIELD", field + view.query = view.query.filter( + and_( + ExtendedMetadata.id == alias.extended_metadata_id, + alias.path == field, + alias.value == str(right) + ) + ) + + + class HistoryDatasetView(ViewQueryBaseClass): DOMAIN = "history_dataset" FIELDS = { - 'name' : ViewField('name', sqlalchemy_field=HistoryDatasetAssociation.name), - 'id' : ViewField('id',sqlalchemy_field=HistoryDatasetAssociation.id, id_decode=True), - 'tag' : ViewField("tag", handler=history_dataset_handle_tag), - 'copied_from_ldda_id' : ViewField("copied_from_ldda_id", - sqlalchemy_field=HistoryDatasetAssociation.copied_from_library_dataset_dataset_association_id, + 'name': ViewField('name', sqlalchemy_field=(HistoryDatasetAssociation, "name")), + 'id': ViewField('id', sqlalchemy_field=(HistoryDatasetAssociation, "id"), id_decode=True), + 'history_id': ViewField('history_id', sqlalchemy_field=(HistoryDatasetAssociation, "history_id"), id_decode=True), + 'tag': ViewField("tag", handler=history_dataset_handle_tag), + 'copied_from_ldda_id': ViewField("copied_from_ldda_id", + sqlalchemy_field=(HistoryDatasetAssociation, "copied_from_library_dataset_dataset_association_id"), id_decode=True), - 'deleted' : ViewField('deleted', sqlalchemy_field=HistoryDatasetAssociation.deleted) + 'copied_from_hda_id': ViewField("copied_from_hda_id", + sqlalchemy_field=(HistoryDatasetAssociation, "copied_from_history_dataset_association_id"), + id_decode=True), + 'deleted': ViewField('deleted', sqlalchemy_field=(HistoryDatasetAssociation, "deleted")), + 'extended_metadata': ViewField('extended_metadata', handler=history_dataset_extended_metadata_filter) } def search(self, trans): @@ -350,10 +373,11 @@ def history_handle_annotation(view, left, operator, right): class HistoryView(ViewQueryBaseClass): DOMAIN = "history" FIELDS = { - 'name' : ViewField('name', sqlalchemy_field=History.name), - 'id' : ViewField('id', sqlalchemy_field=History.id, id_decode=True), - 'tag' : ViewField("tag", handler=history_handle_tag), - 'annotation' : ViewField("annotation", handler=history_handle_annotation) + 'name': ViewField('name', sqlalchemy_field=(History, "name")), + 'id': ViewField('id', sqlalchemy_field=(History, "id"), id_decode=True), + 'tag': ViewField("tag", handler=history_handle_tag), + 'annotation': ViewField("annotation", handler=history_handle_annotation), + 'deleted': ViewField('deleted', sqlalchemy_field=(History, "deleted")) } def search(self, trans): @@ -382,22 +406,21 @@ def workflow_tag_handler(view, left, operator, right): class WorkflowView(ViewQueryBaseClass): DOMAIN = "workflow" FIELDS = { - 'name' : ViewField('name', sqlalchemy_field=StoredWorkflow.name), - 'id' : ViewField('id', sqlalchemy_field=StoredWorkflow.id, id_decode=True), - 'tag' : ViewField('tag', handler=workflow_tag_handler) + 'name': ViewField('name', sqlalchemy_field=(StoredWorkflow, "name")), + 'id': ViewField('id', sqlalchemy_field=(StoredWorkflow, "id"), id_decode=True), + 'tag': ViewField('tag', handler=workflow_tag_handler), + 'deleted': ViewField('deleted', sqlalchemy_field=(StoredWorkflow, "deleted")), } def search(self, trans): self.query = trans.sa_session.query( StoredWorkflow ) - ################## #Job Searching ################## - def job_param_filter(view, left, operator, right): view.do_query = True alias = aliased( JobParameter ) @@ -410,6 +433,7 @@ def job_param_filter(view, left, operator, right): ) ) + def job_input_hda_filter(view, left, operator, right): view.do_query = True alias = aliased( JobToInputDatasetAssociation ) @@ -422,6 +446,7 @@ def job_input_hda_filter(view, left, operator, right): ) ) + def job_input_ldda_filter(view, left, operator, right): view.do_query = True alias = aliased( JobToInputLibraryDatasetAssociation ) @@ -435,7 +460,6 @@ def job_input_ldda_filter(view, left, operator, right): ) - def job_output_hda_filter(view, left, operator, right): view.do_query = True alias = aliased( JobToOutputDatasetAssociation ) @@ -452,36 +476,72 @@ def job_output_hda_filter(view, left, operator, right): class JobView(ViewQueryBaseClass): DOMAIN = "job" FIELDS = { - 'tool_name' : ViewField('tool_name', sqlalchemy_field=Job.tool_id), - 'state' : ViewField('state', sqlalchemy_field=Job.state), - 'param' : ViewField('param', handler=job_param_filter), - 'input_ldda' : ViewField('input_ldda', handler=job_input_ldda_filter, id_decode=True), - 'input_hda' : ViewField('input_hda', handler=job_input_hda_filter, id_decode=True), - 'output_hda' : ViewField('output_hda', handler=job_output_hda_filter, id_decode=True) + 'tool_name': ViewField('tool_name', sqlalchemy_field=(Job, "tool_id")), + 'state': ViewField('state', sqlalchemy_field=(Job, "state")), + 'param': ViewField('param', handler=job_param_filter), + 'input_ldda': ViewField('input_ldda', handler=job_input_ldda_filter, id_decode=True), + 'input_hda': ViewField('input_hda', handler=job_input_hda_filter, id_decode=True), + 'output_hda': ViewField('output_hda', handler=job_output_hda_filter, id_decode=True) } def search(self, trans): self.query = trans.sa_session.query( Job ) +################## +#Page Searching +################## + + +class PageView(ViewQueryBaseClass): + DOMAIN = "page" + FIELDS = { + 'id': ViewField('id', sqlalchemy_field=(Page, "id"), id_decode=True), + 'slug': ViewField('slug', sqlalchemy_field=(Page, "slug")), + 'title': ViewField('title', sqlalchemy_field=(Page, "title")), + 'deleted': ViewField('deleted', sqlalchemy_field=(Page, "deleted")) + } + + def search(self, trans): + self.query = trans.sa_session.query( Page ) + + +################## +#Page Revision Searching +################## + + +class PageRevisionView(ViewQueryBaseClass): + DOMAIN = "page_revision" + FIELDS = { + 'id': ViewField('id', sqlalchemy_field=(PageRevision, "id"), id_decode=True), + 'title': ViewField('title', sqlalchemy_field=(PageRevision, "title")), + 'page_id': ViewField('page_id', sqlalchemy_field=(PageRevision, "page_id"), id_decode=True), + } + + def search(self, trans): + self.query = trans.sa_session.query( PageRevision ) + """ The view mapping takes a user's name for a table and maps it to a View class that will handle queries """ view_mapping = { - 'library' : LibraryView, - 'library_folder' : LibraryFolderView, - 'library_dataset_dataset' : LibraryDatasetDatasetView, - 'library_dataset' : LibraryDatasetView, - 'lda' : LibraryDatasetView, - 'ldda' : LibraryDatasetDatasetView, - 'history_dataset' : HistoryDatasetView, - 'hda' : HistoryDatasetView, - 'history' : HistoryView, - 'workflow' : WorkflowView, - 'tool' : ToolView, - 'job' : JobView, + 'library': LibraryView, + 'library_folder': LibraryFolderView, + 'library_dataset_dataset': LibraryDatasetDatasetView, + 'library_dataset': LibraryDatasetView, + 'lda': LibraryDatasetView, + 'ldda': LibraryDatasetDatasetView, + 'history_dataset': HistoryDatasetView, + 'hda': HistoryDatasetView, + 'history': HistoryView, + 'workflow': WorkflowView, + 'tool': ToolView, + 'job': JobView, + 'page': PageView, + 'page_revision': PageRevisionView, } """ @@ -531,6 +591,7 @@ not_quote = anything:x ?(x != "'") -> x not_dquote = anything:x ?(x != '"') -> x """ + class GalaxyQuery: """ This class represents a data structure of a compiled GQL query @@ -540,6 +601,7 @@ class GalaxyQuery: self.table_name = table_name self.conditional = conditional + class GalaxyQueryComparison: """ This class represents the data structure of the comparison arguments of a @@ -550,6 +612,7 @@ class GalaxyQueryComparison: self.operator = operator self.right = right + class GalaxyQueryAnd: """ This class represents the data structure of the comparison arguments of a @@ -564,6 +627,7 @@ class GalaxyQueryAnd: class GalaxyParseError(Exception): pass + class SearchQuery: def __init__(self, view, query): self.view = view @@ -579,7 +643,8 @@ class SearchQuery: self.view.filter( self.query.conditional.left, self.query.conditional.operator, - self.query.conditional.right ) + self.query.conditional.right + ) return self.view.get_results(True) def item_to_api_value(self, item): @@ -593,17 +658,16 @@ class SearchQuery: return o - class GalaxySearchEngine: """ Primary class for searching. Parses GQL (Galaxy Query Language) queries and returns a 'SearchQuery' class """ def __init__(self): self.parser = parsley.makeGrammar(gqlGrammar, { - 're' : re, - 'GalaxyQuery' : GalaxyQuery, - 'GalaxyQueryComparison' : GalaxyQueryComparison, - 'GalaxyQueryAnd' : GalaxyQueryAnd + 're': re, + 'GalaxyQuery': GalaxyQuery, + 'GalaxyQueryComparison': GalaxyQueryComparison, + 'GalaxyQueryAnd': GalaxyQueryAnd }) def query(self, query_text): @@ -613,4 +677,3 @@ class GalaxySearchEngine: view = view_mapping[q.table_name]() return SearchQuery(view, q) raise GalaxyParseError("No such table %s" % (q.table_name)) - diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py new file mode 100644 index 00000000000..980cfa53c86 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -0,0 +1,650 @@ +import logging +import os +from galaxy.model.item_attrs import Dictifiable +from galaxy.util.bunch import Bunch +from galaxy.util import asbool +from tool_shed.util import common_util + +log = logging.getLogger( __name__ ) + + +class ToolShedRepository( object ): + dict_collection_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', + 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) + dict_element_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', + 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) + installation_status = Bunch( NEW='New', + CLONING='Cloning', + SETTING_TOOL_VERSIONS='Setting tool versions', + INSTALLING_REPOSITORY_DEPENDENCIES='Installing repository dependencies', + INSTALLING_TOOL_DEPENDENCIES='Installing tool dependencies', + LOADING_PROPRIETARY_DATATYPES='Loading proprietary datatypes', + INSTALLED='Installed', + DEACTIVATED='Deactivated', + ERROR='Error', + UNINSTALLED='Uninstalled' ) + states = Bunch( INSTALLING='running', + OK='ok', + WARNING='queued', + ERROR='error', + UNINSTALLED='deleted_new' ) + + def __init__( self, id=None, create_time=None, tool_shed=None, name=None, description=None, owner=None, installed_changeset_revision=None, + changeset_revision=None, ctx_rev=None, metadata=None, includes_datatypes=False, tool_shed_status=None, deleted=False, + uninstalled=False, dist_to_shed=False, status=None, error_message=None ): + self.id = id + self.create_time = create_time + self.tool_shed = tool_shed + self.name = name + self.description = description + self.owner = owner + self.installed_changeset_revision = installed_changeset_revision + self.changeset_revision = changeset_revision + self.ctx_rev = ctx_rev + self.metadata = metadata + self.includes_datatypes = includes_datatypes + self.tool_shed_status = tool_shed_status + self.deleted = deleted + self.uninstalled = uninstalled + self.dist_to_shed = dist_to_shed + self.status = status + self.error_message = error_message + + def as_dict( self, value_mapper=None ): + return self.to_dict( view='element', value_mapper=value_mapper ) + + @property + def can_install( self ): + return self.status == self.installation_status.NEW + + @property + def can_reset_metadata( self ): + return self.status == self.installation_status.INSTALLED + + @property + def can_uninstall( self ): + return self.status != self.installation_status.UNINSTALLED + + @property + def can_deactivate( self ): + return self.status not in [ self.installation_status.DEACTIVATED, self.installation_status.UNINSTALLED ] + + @property + def can_reinstall_or_activate( self ): + return self.deleted + + def get_shed_config_filename( self ): + shed_config_filename = None + if self.metadata: + shed_config_filename = self.metadata.get( 'shed_config_filename', shed_config_filename ) + return shed_config_filename + + def get_shed_config_dict( self, app, default=None ): + """ + Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry + in the shed_tool_conf_dict. + """ + if not self.shed_config_filename: + self.guess_shed_config( app, default=default ) + if self.shed_config_filename: + for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + if self.shed_config_filename == shed_tool_conf_dict[ 'config_filename' ]: + return shed_tool_conf_dict + return default + + def get_tool_relative_path( self, app ): + shed_conf_dict = self.get_shed_config_dict( app ) + tool_path = None + relative_path = None + if shed_conf_dict: + tool_path = shed_conf_dict[ 'tool_path' ] + relative_path = os.path.join( self.tool_shed_path_name, 'repos', self.owner, self.name, self.installed_changeset_revision ) + return tool_path, relative_path + + def guess_shed_config( self, app, default=None ): + tool_ids = [] + metadata = self.metadata or {} + for tool in metadata.get( 'tools', [] ): + tool_ids.append( tool.get( 'guid' ) ) + for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + name = shed_tool_conf_dict[ 'config_filename' ] + for elem in shed_tool_conf_dict[ 'config_elems' ]: + if elem.tag == 'tool': + for sub_elem in elem.findall( 'id' ): + tool_id = sub_elem.text.strip() + if tool_id in tool_ids: + self.shed_config_filename = name + return shed_tool_conf_dict + elif elem.tag == "section": + for tool_elem in elem.findall( 'tool' ): + for sub_elem in tool_elem.findall( 'id' ): + tool_id = sub_elem.text.strip() + if tool_id in tool_ids: + self.shed_config_filename = name + return shed_tool_conf_dict + if self.includes_datatypes: + #we need to search by filepaths here, which is less desirable + tool_shed_url = self.tool_shed + if tool_shed_url.find( ':' ) > 0: + # Eliminate the port, if any, since it will result in an invalid directory name. + tool_shed_url = tool_shed_url.split( ':' )[ 0 ] + tool_shed = tool_shed_url.rstrip( '/' ) + for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + tool_path = shed_tool_conf_dict[ 'tool_path' ] + relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) + if os.path.exists( relative_path ): + self.shed_config_filename = shed_tool_conf_dict[ 'config_filename' ] + return shed_tool_conf_dict + return default + + @property + def has_readme_files( self ): + if self.metadata: + return 'readme_files' in self.metadata + return False + + @property + def has_repository_dependencies( self ): + if self.metadata: + repository_dependencies_dict = self.metadata.get( 'repository_dependencies', {} ) + repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) + # [["http://localhost:9009", "package_libgtextutils_0_6", "test", "e2003cbf18cd", "True", "True"]] + for rd_tup in repository_dependencies: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( rd_tup ) + if not asbool( only_if_compiling_contained_td ): + return True + return False + + @property + def has_repository_dependencies_only_if_compiling_contained_td( self ): + if self.metadata: + repository_dependencies_dict = self.metadata.get( 'repository_dependencies', {} ) + repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) + # [["http://localhost:9009", "package_libgtextutils_0_6", "test", "e2003cbf18cd", "True", "True"]] + for rd_tup in repository_dependencies: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( rd_tup ) + if not asbool( only_if_compiling_contained_td ): + return False + return True + return False + + @property + def in_error_state( self ): + return self.status == self.installation_status.ERROR + + @property + def includes_data_managers( self ): + if self.metadata: + return bool( len( self.metadata.get( 'data_manager', {} ).get( 'data_managers', {} ) ) ) + return False + + @property + def includes_tools( self ): + if self.metadata: + return 'tools' in self.metadata + return False + + @property + def includes_tools_for_display_in_tool_panel( self ): + if self.includes_tools: + tool_dicts = self.metadata[ 'tools' ] + for tool_dict in tool_dicts: + if tool_dict.get( 'add_to_tool_panel', True ): + return True + return False + + @property + def includes_tool_dependencies( self ): + if self.metadata: + return 'tool_dependencies' in self.metadata + return False + + @property + def includes_workflows( self ): + if self.metadata: + return 'workflows' in self.metadata + return False + + @property + def installed_repository_dependencies( self ): + """Return the repository's repository dependencies that are currently installed.""" + installed_required_repositories = [] + for required_repository in self.repository_dependencies: + if required_repository.status == self.installation_status.INSTALLED: + installed_required_repositories.append( required_repository ) + return installed_required_repositories + + @property + def installed_tool_dependencies( self ): + """Return the repository's tool dependencies that are currently installed, but possibly in an error state.""" + installed_dependencies = [] + for tool_dependency in self.tool_dependencies: + if tool_dependency.status in [ ToolDependency.installation_status.INSTALLED ]: + installed_dependencies.append( tool_dependency ) + return installed_dependencies + + @property + def is_deprecated_in_tool_shed( self ): + if self.tool_shed_status: + return asbool( self.tool_shed_status.get( 'repository_deprecated', False ) ) + return False + + @property + def is_deactivated_or_installed( self ): + return self.status in [ self.installation_status.DEACTIVATED, + self.installation_status.INSTALLED ] + + @property + def is_installed( self ): + return self.status == self.installation_status.INSTALLED + + @property + def is_latest_installable_revision( self ): + if self.tool_shed_status: + return asbool( self.tool_shed_status.get( 'latest_installable_revision', False ) ) + return False + + @property + def missing_repository_dependencies( self ): + """Return the repository's repository dependencies that are not currently installed, and may not ever have been installed.""" + missing_required_repositories = [] + for required_repository in self.repository_dependencies: + if required_repository.status not in [ self.installation_status.INSTALLED ]: + missing_required_repositories.append( required_repository ) + return missing_required_repositories + + @property + def missing_tool_dependencies( self ): + """Return the repository's tool dependencies that are not currently installed, and may not ever have been installed.""" + missing_dependencies = [] + for tool_dependency in self.tool_dependencies: + if tool_dependency.status not in [ ToolDependency.installation_status.INSTALLED ]: + missing_dependencies.append( tool_dependency ) + return missing_dependencies + + def repo_files_directory( self, app ): + repo_path = self.repo_path( app ) + if repo_path: + return os.path.join( repo_path, self.name ) + return None + + def repo_path( self, app ): + tool_shed_url = self.tool_shed + if tool_shed_url.find( ':' ) > 0: + # Eliminate the port, if any, since it will result in an invalid directory name. + tool_shed_url = tool_shed_url.split( ':' )[ 0 ] + tool_shed = tool_shed_url.rstrip( '/' ) + for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ): + tool_path = shed_tool_conf_dict[ 'tool_path' ] + relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) + if os.path.exists( relative_path ): + return relative_path + return None + + @property + def repository_dependencies( self ): + """ + Return all of this repository's repository dependencies, ignoring their attributes like prior_installation_required and + only_if_compiling_contained_td. + """ + required_repositories = [] + for rrda in self.required_repositories: + repository_dependency = rrda.repository_dependency + required_repository = repository_dependency.repository + if required_repository: + required_repositories.append( required_repository ) + return required_repositories + + @property + def repository_dependencies_being_installed( self ): + """Return the repository's repository dependencies that are currently being installed.""" + required_repositories_being_installed = [] + for required_repository in self.repository_dependencies: + if required_repository.status in [ self.installation_status.CLONING, + self.installation_status.INSTALLING_REPOSITORY_DEPENDENCIES, + self.installation_status.INSTALLING_TOOL_DEPENDENCIES, + self.installation_status.LOADING_PROPRIETARY_DATATYPES, + self.installation_status.SETTING_TOOL_VERSIONS ]: + required_repositories_being_installed.append( required_repository ) + return required_repositories_being_installed + + @property + def repository_dependencies_missing_or_being_installed( self ): + """Return the repository's repository dependencies that are either missing or currently being installed.""" + required_repositories_missing_or_being_installed = [] + for required_repository in self.repository_dependencies: + if required_repository.status in [ self.installation_status.ERROR, + self.installation_status.INSTALLING, + self.installation_status.NEVER_INSTALLED, + self.installation_status.UNINSTALLED ]: + required_repositories_missing_or_being_installed.append( required_repository ) + return required_repositories_missing_or_being_installed + + @property + def repository_dependencies_with_installation_errors( self ): + """Return the repository's repository dependencies that have installation errors.""" + required_repositories_with_installation_errors = [] + for required_repository in self.repository_dependencies: + if required_repository.status == self.installation_status.ERROR: + required_repositories_with_installation_errors.append( required_repository ) + return required_repositories_with_installation_errors + + @property + def requires_prior_installation_of( self ): + """ + Return a list of repository dependency tuples like (tool_shed, name, owner, changeset_revision, prior_installation_required) for this + repository's repository dependencies where prior_installation_required is True. By definition, repository dependencies are required to + be installed in order for this repository to function correctly. However, those repository dependencies that are defined for this + repository with prior_installation_required set to True place them in a special category in that the required repositories must be + installed before this repository is installed. Among other things, this enables these "special" repository dependencies to include + information that enables the successful intallation of this repository. This method is not used during the initial installation of + this repository, but only after it has been installed (metadata must be set for this repository in order for this method to be useful). + """ + required_rd_tups_that_must_be_installed = [] + if self.has_repository_dependencies: + rd_tups = self.metadata[ 'repository_dependencies' ][ 'repository_dependencies' ] + for rd_tup in rd_tups: + if len( rd_tup ) == 5: + tool_shed, name, owner, changeset_revision, prior_installation_required = rd_tup + if asbool( prior_installation_required ): + required_rd_tups_that_must_be_installed.append( ( tool_shed, name, owner, changeset_revision, 'True', 'False' ) ) + elif len( rd_tup ) == 6: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = rd_tup + # The repository dependency will only be required to be previsously installed if it does not fall into the category of + # a repository that must be installed only so that it's contained tool dependency can be used for compiling the tool + # dependency of the dependent repository. + if not asbool( only_if_compiling_contained_td ): + if asbool( prior_installation_required ): + required_rd_tups_that_must_be_installed.append( ( tool_shed, name, owner, changeset_revision, 'True', 'False' ) ) + return required_rd_tups_that_must_be_installed + + @property + def revision_update_available( self ): + # This method should be named update_available, but since it is no longer possible to drop a table column using migration scripts + # with the sqlite database (see ~/galaxy/model/migrate/versions/0016_drop_update_available_col_add_tool_shed_status_col.py), we + # have to name it in such a way that it will not conflict with the eliminated tool_shed_repository.update_available column (which + # cannot be eliminated if using the sqlite database). + if self.tool_shed_status: + return asbool( self.tool_shed_status.get( 'revision_update', False ) ) + return False + + def set_shed_config_filename( self, value ): + self.metadata[ 'shed_config_filename' ] = value + + shed_config_filename = property( get_shed_config_filename, set_shed_config_filename ) + + def to_dict( self, view='collection', value_mapper=None ): + if value_mapper is None: + value_mapper = {} + rval = {} + try: + visible_keys = self.__getattribute__( 'dict_' + view + '_visible_keys' ) + except AttributeError: + raise Exception( 'Unknown API view: %s' % view ) + for key in visible_keys: + try: + rval[ key ] = self.__getattribute__( key ) + if key in value_mapper: + rval[ key ] = value_mapper.get( key, rval[ key ] ) + except AttributeError: + rval[ key ] = None + return rval + + @property + def tool_dependencies_being_installed( self ): + dependencies_being_installed = [] + for tool_dependency in self.tool_dependencies: + if tool_dependency.status == ToolDependency.installation_status.INSTALLING: + dependencies_being_installed.append( tool_dependency ) + return dependencies_being_installed + + @property + def tool_dependencies_installed_or_in_error( self ): + """Return the repository's tool dependencies that are currently installed, but possibly in an error state.""" + installed_dependencies = [] + for tool_dependency in self.tool_dependencies: + if tool_dependency.status in [ ToolDependency.installation_status.INSTALLED, + ToolDependency.installation_status.ERROR ]: + installed_dependencies.append( tool_dependency ) + return installed_dependencies + + @property + def tool_dependencies_missing_or_being_installed( self ): + dependencies_missing_or_being_installed = [] + for tool_dependency in self.tool_dependencies: + if tool_dependency.status in [ ToolDependency.installation_status.ERROR, + ToolDependency.installation_status.INSTALLING, + ToolDependency.installation_status.NEVER_INSTALLED, + ToolDependency.installation_status.UNINSTALLED ]: + dependencies_missing_or_being_installed.append( tool_dependency ) + return dependencies_missing_or_being_installed + + @property + def tool_dependencies_with_installation_errors( self ): + dependencies_with_installation_errors = [] + for tool_dependency in self.tool_dependencies: + if tool_dependency.status == ToolDependency.installation_status.ERROR: + dependencies_with_installation_errors.append( tool_dependency ) + return dependencies_with_installation_errors + + @property + def tool_shed_path_name( self ): + tool_shed_url = self.tool_shed + if tool_shed_url.find( ':' ) > 0: + # Eliminate the port, if any, since it will result in an invalid directory name. + tool_shed_url = tool_shed_url.split( ':' )[ 0 ] + return tool_shed_url.rstrip( '/' ) + + @property + def tuples_of_repository_dependencies_needed_for_compiling_td( self ): + """ + Return tuples defining this repository's repository dependencies that are necessary only for compiling this repository's tool + dependencies. + """ + rd_tups_of_repositories_needed_for_compiling_td = [] + if self.metadata: + repository_dependencies = self.metadata.get( 'repository_dependencies', None ) + rd_tups = repository_dependencies[ 'repository_dependencies' ] + for rd_tup in rd_tups: + if len( rd_tup ) == 6: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = rd_tup + if asbool( only_if_compiling_contained_td ): + rd_tups_of_repositories_needed_for_compiling_td.append( ( tool_shed, name, owner, changeset_revision, 'False', 'True' ) ) + return rd_tups_of_repositories_needed_for_compiling_td + + @property + def uninstalled_repository_dependencies( self ): + """Return the repository's repository dependencies that have been uninstalled.""" + uninstalled_required_repositories = [] + for required_repository in self.repository_dependencies: + if required_repository.status == self.installation_status.UNINSTALLED: + uninstalled_required_repositories.append( required_repository ) + return uninstalled_required_repositories + + @property + def uninstalled_tool_dependencies( self ): + """Return the repository's tool dependencies that have been uninstalled.""" + uninstalled_tool_dependencies = [] + for tool_dependency in self.tool_dependencies: + if tool_dependency.status == ToolDependency.installation_status.UNINSTALLED: + uninstalled_tool_dependencies.append( tool_dependency ) + return uninstalled_tool_dependencies + + @property + def upgrade_available( self ): + if self.tool_shed_status: + if self.is_deprecated_in_tool_shed: + # Only allow revision upgrades if the repository is not deprecated in the tool shed. + return False + return asbool( self.tool_shed_status.get( 'revision_upgrade', False ) ) + return False + + +class RepositoryRepositoryDependencyAssociation( object ): + def __init__( self, tool_shed_repository_id=None, repository_dependency_id=None ): + self.tool_shed_repository_id = tool_shed_repository_id + self.repository_dependency_id = repository_dependency_id + + +class RepositoryDependency( object ): + def __init__( self, tool_shed_repository_id=None ): + self.tool_shed_repository_id = tool_shed_repository_id + + +class ToolDependency( object ): + installation_status = Bunch( NEVER_INSTALLED='Never installed', + INSTALLING='Installing', + INSTALLED='Installed', + ERROR='Error', + UNINSTALLED='Uninstalled' ) + + states = Bunch( INSTALLING='running', + OK='ok', + WARNING='queued', + ERROR='error', + UNINSTALLED='deleted_new' ) + + def __init__( self, tool_shed_repository_id=None, name=None, version=None, type=None, status=None, error_message=None ): + self.tool_shed_repository_id = tool_shed_repository_id + self.name = name + self.version = version + self.type = type + self.status = status + self.error_message = error_message + + @property + def can_install( self ): + return self.status in [ self.installation_status.NEVER_INSTALLED, self.installation_status.UNINSTALLED ] + + @property + def can_uninstall( self ): + return self.status in [ self.installation_status.ERROR, self.installation_status.INSTALLED ] + + @property + def can_update( self ): + return self.status in [ self.installation_status.NEVER_INSTALLED, + self.installation_status.INSTALLED, + self.installation_status.ERROR, + self.installation_status.UNINSTALLED ] + + def get_env_shell_file_path( self, app ): + installation_directory = self.installation_directory( app ) + file_path = os.path.join( installation_directory, 'env.sh' ) + if os.path.exists( file_path ): + return file_path + return None + + @property + def in_error_state( self ): + return self.status == self.installation_status.ERROR + + def installation_directory( self, app ): + if self.type == 'package': + return os.path.join( app.config.tool_dependency_dir, + self.name, + self.version, + self.tool_shed_repository.owner, + self.tool_shed_repository.name, + self.tool_shed_repository.installed_changeset_revision ) + if self.type == 'set_environment': + return os.path.join( app.config.tool_dependency_dir, + 'environment_settings', + self.name, + self.tool_shed_repository.owner, + self.tool_shed_repository.name, + self.tool_shed_repository.installed_changeset_revision ) + + @property + def is_installed( self ): + return self.status == self.installation_status.INSTALLED + + +class ToolVersion( object, Dictifiable ): + dict_element_visible_keys = ( 'id', 'tool_shed_repository' ) + + def __init__( self, id=None, create_time=None, tool_id=None, tool_shed_repository=None ): + self.id = id + self.create_time = create_time + self.tool_id = tool_id + self.tool_shed_repository = tool_shed_repository + + def get_previous_version( self, app ): + context = app.install_model.context + tva = context.query( app.install_model.ToolVersionAssociation ) \ + .filter( app.install_model.ToolVersionAssociation.table.c.tool_id == self.id ) \ + .first() + if tva: + return context.query( app.install_model.ToolVersion ) \ + .filter( app.install_model.ToolVersion.table.c.id == tva.parent_id ) \ + .first() + return None + + def get_next_version( self, app ): + context = app.install_model.context + tva = context.query( app.install_model.ToolVersionAssociation ) \ + .filter( app.install_model.ToolVersionAssociation.table.c.parent_id == self.id ) \ + .first() + if tva: + return context.query( app.install_model.ToolVersion ) \ + .filter( app.install_model.ToolVersion.table.c.id == tva.tool_id ) \ + .first() + return None + + def get_versions( self, app ): + tool_versions = [] + # Prepend ancestors. + + def __ancestors( app, tool_version ): + # Should we handle multiple parents at each level? + previous_version = tool_version.get_previous_version( app ) + if previous_version: + if previous_version not in tool_versions: + tool_versions.insert( 0, previous_version ) + __ancestors( app, previous_version ) + + # Append descendants. + def __descendants( app, tool_version ): + # Should we handle multiple child siblings at each level? + next_version = tool_version.get_next_version( app ) + if next_version: + if next_version not in tool_versions: + tool_versions.append( next_version ) + __descendants( app, next_version ) + __ancestors( app, self ) + if self not in tool_versions: + tool_versions.append( self ) + __descendants( app, self ) + return tool_versions + + def get_version_ids( self, app, reverse=False ): + if reverse: + version_ids = [] + for tool_version in self.get_versions( app ): + version_ids.insert( 0, tool_version.tool_id ) + return version_ids + return [ tool_version.tool_id for tool_version in self.get_versions( app ) ] + + def to_dict( self, view='element' ): + rval = super( ToolVersion, self ).to_dict( view=view ) + rval['tool_name'] = self.tool_id + for a in self.parent_tool_association: + rval['parent_tool_id'] = a.parent_id + for a in self.child_tool_association: + rval['child_tool_id'] = a.tool_id + return rval + + +class ToolVersionAssociation( object ): + def __init__( self, id=None, tool_id=None, parent_id=None ): + self.id = id + self.tool_id = tool_id + self.parent_id = parent_id + + +class MigrateTools( object ): + def __init__( self, repository_id=None, repository_path=None, version=None ): + self.repository_id = repository_id + self.repository_path = repository_path + self.version = version diff --git a/lib/galaxy/model/tool_shed_install/mapping.py b/lib/galaxy/model/tool_shed_install/mapping.py new file mode 100644 index 00000000000..cd52f9d5b85 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/mapping.py @@ -0,0 +1,121 @@ +from galaxy.model import tool_shed_install as install_model +from sqlalchemy import MetaData +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, Table, TEXT +from sqlalchemy.orm import relation, mapper +from galaxy.model.custom_types import JSONType, TrimmedString +from galaxy.model.orm.now import now +from galaxy.model.base import ModelMapping +from galaxy.model.orm.engine_factory import build_engine + +metadata = MetaData() + +install_model.ToolShedRepository.table = Table( "tool_shed_repository", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed", TrimmedString( 255 ), index=True ), + Column( "name", TrimmedString( 255 ), index=True ), + Column( "description" , TEXT ), + Column( "owner", TrimmedString( 255 ), index=True ), + Column( "installed_changeset_revision", TrimmedString( 255 ) ), + Column( "changeset_revision", TrimmedString( 255 ), index=True ), + Column( "ctx_rev", TrimmedString( 10 ) ), + Column( "metadata", JSONType, nullable=True ), + Column( "includes_datatypes", Boolean, index=True, default=False ), + Column( "tool_shed_status", JSONType, nullable=True ), + Column( "deleted", Boolean, index=True, default=False ), + Column( "uninstalled", Boolean, default=False ), + Column( "dist_to_shed", Boolean, default=False ), + Column( "status", TrimmedString( 255 ) ), + Column( "error_message", TEXT ) ) + +install_model.RepositoryRepositoryDependencyAssociation.table = Table( 'repository_repository_dependency_association', metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True ), + Column( "repository_dependency_id", Integer, ForeignKey( "repository_dependency.id" ), index=True ) ) + +install_model.RepositoryDependency.table = Table( "repository_dependency", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=False ) ) + +install_model.ToolDependency.table = Table( "tool_dependency", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=False ), + Column( "name", TrimmedString( 255 ) ), + Column( "version", TEXT ), + Column( "type", TrimmedString( 40 ) ), + Column( "status", TrimmedString( 255 ), nullable=False ), + Column( "error_message", TEXT ) ) + +install_model.ToolVersion.table = Table( "tool_version", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_id", String( 255 ) ), + Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=True ) ) + +install_model.ToolVersionAssociation.table = Table( "tool_version_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "tool_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ), + Column( "parent_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ) ) + +install_model.MigrateTools.table = Table( "migrate_tools", metadata, + Column( "repository_id", TrimmedString( 255 ) ), + Column( "repository_path", TEXT ), + Column( "version", Integer ) ) + +mapper( install_model.ToolShedRepository, install_model.ToolShedRepository.table, + properties=dict( tool_versions=relation( install_model.ToolVersion, + primaryjoin=( install_model.ToolShedRepository.table.c.id == install_model.ToolVersion.table.c.tool_shed_repository_id ), + backref='tool_shed_repository' ), + tool_dependencies=relation( install_model.ToolDependency, + primaryjoin=( install_model.ToolShedRepository.table.c.id == install_model.ToolDependency.table.c.tool_shed_repository_id ), + order_by=install_model.ToolDependency.table.c.name, + backref='tool_shed_repository' ), + required_repositories=relation( install_model.RepositoryRepositoryDependencyAssociation, + primaryjoin=( install_model.ToolShedRepository.table.c.id == install_model.RepositoryRepositoryDependencyAssociation.table.c.tool_shed_repository_id ) ) ) ) + +mapper( install_model.RepositoryRepositoryDependencyAssociation, install_model.RepositoryRepositoryDependencyAssociation.table, + properties=dict( repository=relation( install_model.ToolShedRepository, + primaryjoin=( install_model.RepositoryRepositoryDependencyAssociation.table.c.tool_shed_repository_id == install_model.ToolShedRepository.table.c.id ) ), + repository_dependency=relation( install_model.RepositoryDependency, + primaryjoin=( install_model.RepositoryRepositoryDependencyAssociation.table.c.repository_dependency_id == install_model.RepositoryDependency.table.c.id ) ) ) ) + +mapper( install_model.RepositoryDependency, install_model.RepositoryDependency.table, + properties=dict( repository=relation( install_model.ToolShedRepository, + primaryjoin=( install_model.RepositoryDependency.table.c.tool_shed_repository_id == install_model.ToolShedRepository.table.c.id ) ) ) ) + +mapper( install_model.ToolDependency, install_model.ToolDependency.table ) + +mapper( install_model.ToolVersion, install_model.ToolVersion.table, + properties=dict( + parent_tool_association=relation( install_model.ToolVersionAssociation, + primaryjoin=( install_model.ToolVersion.table.c.id == install_model.ToolVersionAssociation.table.c.tool_id ) ), + child_tool_association=relation( install_model.ToolVersionAssociation, + primaryjoin=( install_model.ToolVersion.table.c.id == install_model.ToolVersionAssociation.table.c.parent_id ) ) + ) + ) + +mapper( install_model.ToolVersionAssociation, install_model.ToolVersionAssociation.table ) + + +def init( url, engine_options={}, create_tables=False ): + """Connect mappings to the database""" + # Load the appropriate db module + engine = build_engine( url, engine_options ) + # Connect the metadata to the database. + metadata.bind = engine + result = ModelMapping( [ install_model ], engine=engine ) + # Create tables if needed + if create_tables: + metadata.create_all() + # metadata.engine.commit() + result.create_tables = create_tables + #load local galaxy security policy + return result diff --git a/test-data/empty_file.dat b/lib/galaxy/model/tool_shed_install/migrate/__init__.py similarity index 100% rename from test-data/empty_file.dat rename to lib/galaxy/model/tool_shed_install/migrate/__init__.py diff --git a/lib/galaxy/model/tool_shed_install/migrate/check.py b/lib/galaxy/model/tool_shed_install/migrate/check.py new file mode 100644 index 00000000000..70e4d43218b --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/check.py @@ -0,0 +1,113 @@ +import sys +import os.path +import logging + +from galaxy import eggs +eggs.require( "SQLAlchemy" ) +eggs.require( "decorator" ) # Required by sqlalchemy-migrate +eggs.require( "Tempita " ) # Required by sqlalchemy-migrate +eggs.require( "sqlalchemy-migrate" ) + +from sqlalchemy import * +from sqlalchemy.exc import NoSuchTableError +from migrate.versioning import repository, schema + +from galaxy.model.orm import dialect_to_egg + +log = logging.getLogger( __name__ ) + +# path relative to galaxy +migrate_repository_directory = os.path.dirname( __file__ ).replace( os.getcwd() + os.path.sep, '', 1 ) +migrate_repository = repository.Repository( migrate_repository_directory ) + + +def create_or_verify_database( url, engine_options={}, app=None ): + """ + """ + dialect = ( url.split( ':', 1 ) )[0] + try: + egg = dialect_to_egg[dialect] + try: + eggs.require( egg ) + log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) + except: + # If the module is in the path elsewhere (i.e. non-egg), it'll still load. + log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) + except KeyError: + # Let this go, it could possibly work with db's we don't support + log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) + # Create engine and metadata + engine = create_engine( url, **engine_options ) + + def migrate(): + try: + # Declare the database to be under a repository's version control + db_schema = schema.ControlledSchema.create( engine, migrate_repository ) + except: + # The database is already under version control + db_schema = schema.ControlledSchema( engine, migrate_repository ) + # Apply all scripts to get to current version + migrate_to_current_version( engine, db_schema ) + + meta = MetaData( bind=engine ) + if app and getattr( app.config, 'database_auto_migrate', False ): + migrate() + return + + # Try to load tool_shed_repository table + try: + Table( "tool_shed_repository", meta, autoload=True ) + except NoSuchTableError: + # No table means a completely uninitialized database. If we + # have an app, we'll set it's new_installation setting to True + # so the tool migration process will be skipped. + migrate() + return + + try: + Table( "migrate_version", meta, autoload=True ) + except NoSuchTableError: + # The database exists but is not yet under migrate version control, so init with version 1 + log.info( "Adding version control to existing database" ) + try: + Table( "metadata_file", meta, autoload=True ) + schema.ControlledSchema.create( engine, migrate_repository, version=2 ) + except NoSuchTableError: + schema.ControlledSchema.create( engine, migrate_repository, version=1 ) + + # Verify that the code and the DB are in sync + db_schema = schema.ControlledSchema( engine, migrate_repository ) + if migrate_repository.versions.latest != db_schema.version: + exception_msg = "Your database has version '%d' but this code expects version '%d'. " % ( db_schema.version, migrate_repository.versions.latest ) + exception_msg += "Back up your database and then migrate the schema by running the following from your Galaxy installation directory:" + exception_msg += "\n\nsh manage_db.sh upgrade install\n" + + else: + log.info( "At database version %d" % db_schema.version ) + + +def migrate_to_current_version( engine, schema ): + # Changes to get to current version + changeset = schema.changeset( None ) + for ver, change in changeset: + nextver = ver + changeset.step + log.info( 'Migrating %s -> %s... ' % ( ver, nextver ) ) + old_stdout = sys.stdout + + class FakeStdout( object ): + def __init__( self ): + self.buffer = [] + + def write( self, s ): + self.buffer.append( s ) + + def flush( self ): + pass + + sys.stdout = FakeStdout() + try: + schema.runchange( ver, change, changeset.step ) + finally: + for message in "".join( sys.stdout.buffer ).split( "\n" ): + log.info( message ) + sys.stdout = old_stdout diff --git a/lib/galaxy/model/tool_shed_install/migrate/migrate.cfg b/lib/galaxy/model/tool_shed_install/migrate/migrate.cfg new file mode 100644 index 00000000000..acfc9879b87 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/migrate.cfg @@ -0,0 +1,20 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=ToolShedInstall + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0001_add_tool_shed_repository_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0001_add_tool_shed_repository_table.py new file mode 120000 index 00000000000..e0b183100f2 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0001_add_tool_shed_repository_table.py @@ -0,0 +1 @@ +../../../migrate/versions/0082_add_tool_shed_repository_table.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0002_add_tool_shed_repository_table_columns.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0002_add_tool_shed_repository_table_columns.py new file mode 120000 index 00000000000..7cacf92509e --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0002_add_tool_shed_repository_table_columns.py @@ -0,0 +1 @@ +../../../migrate/versions/0086_add_tool_shed_repository_table_columns.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0003_tool_id_guid_map_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0003_tool_id_guid_map_table.py new file mode 120000 index 00000000000..3e9dea9c20a --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0003_tool_id_guid_map_table.py @@ -0,0 +1 @@ +../../../migrate/versions/0087_tool_id_guid_map_table.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0004_add_installed_changeset_revison_column.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0004_add_installed_changeset_revison_column.py new file mode 120000 index 00000000000..4e57eb188b8 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0004_add_installed_changeset_revison_column.py @@ -0,0 +1 @@ +../../../migrate/versions/0088_add_installed_changeset_revison_column.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0005_add_tool_shed_repository_table_columns.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0005_add_tool_shed_repository_table_columns.py new file mode 120000 index 00000000000..6359c3b9bbb --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0005_add_tool_shed_repository_table_columns.py @@ -0,0 +1 @@ +../../../migrate/versions/0090_add_tool_shed_repository_table_columns.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0006_add_tool_version_tables.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0006_add_tool_version_tables.py new file mode 120000 index 00000000000..9a4526d9b96 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0006_add_tool_version_tables.py @@ -0,0 +1 @@ +../../../migrate/versions/0091_add_tool_version_tables.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0007_add_migrate_tools_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0007_add_migrate_tools_table.py new file mode 120000 index 00000000000..9fbb816a5cf --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0007_add_migrate_tools_table.py @@ -0,0 +1 @@ +../../../migrate/versions/0092_add_migrate_tools_table.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0008_add_ctx_rev_column.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0008_add_ctx_rev_column.py new file mode 120000 index 00000000000..a1268f1ff03 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0008_add_ctx_rev_column.py @@ -0,0 +1 @@ +../../../migrate/versions/0097_add_ctx_rev_column.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0009_add_tool_dependency_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0009_add_tool_dependency_table.py new file mode 120000 index 00000000000..a119ddd378f --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0009_add_tool_dependency_table.py @@ -0,0 +1 @@ +../../../migrate/versions/0099_add_tool_dependency_table.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0010_alter_tool_dependency_table_version_column.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0010_alter_tool_dependency_table_version_column.py new file mode 120000 index 00000000000..1a39c5872a2 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0010_alter_tool_dependency_table_version_column.py @@ -0,0 +1 @@ +../../../migrate/versions/0100_alter_tool_dependency_table_version_column.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0011_drop_installed_changeset_revision_column.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0011_drop_installed_changeset_revision_column.py new file mode 120000 index 00000000000..eed3a389c27 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0011_drop_installed_changeset_revision_column.py @@ -0,0 +1 @@ +../../../migrate/versions/0101_drop_installed_changeset_revision_column.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0012_add_tool_dependency_status_columns.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0012_add_tool_dependency_status_columns.py new file mode 120000 index 00000000000..c5759e0e88d --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0012_add_tool_dependency_status_columns.py @@ -0,0 +1 @@ +../../../migrate/versions/0102_add_tool_dependency_status_columns.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0013_add_tool_shed_repository_status_columns.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0013_add_tool_shed_repository_status_columns.py new file mode 120000 index 00000000000..c17fdea9421 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0013_add_tool_shed_repository_status_columns.py @@ -0,0 +1 @@ +../../../migrate/versions/0103_add_tool_shed_repository_status_columns.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0014_add_repository_dependency_tables.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0014_add_repository_dependency_tables.py new file mode 120000 index 00000000000..5f07fdf3a63 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0014_add_repository_dependency_tables.py @@ -0,0 +1 @@ +../../../migrate/versions/0109_add_repository_dependency_tables.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0015_update_migrate_tools_table.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0015_update_migrate_tools_table.py new file mode 120000 index 00000000000..55c1ced6cd5 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0015_update_migrate_tools_table.py @@ -0,0 +1 @@ +../../../migrate/versions/0113_update_migrate_tools_table.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0016_update_migrate_tools_table_again.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0016_update_migrate_tools_table_again.py new file mode 120000 index 00000000000..7af8a2a1890 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0016_update_migrate_tools_table_again.py @@ -0,0 +1 @@ +../../../migrate/versions/0114_update_migrate_tools_table_again.py \ No newline at end of file diff --git a/lib/galaxy/model/tool_shed_install/migrate/versions/0017_drop_update_available_col_add_tool_shed_status_col.py b/lib/galaxy/model/tool_shed_install/migrate/versions/0017_drop_update_available_col_add_tool_shed_status_col.py new file mode 120000 index 00000000000..d874f486173 --- /dev/null +++ b/lib/galaxy/model/tool_shed_install/migrate/versions/0017_drop_update_available_col_add_tool_shed_status_col.py @@ -0,0 +1 @@ +../../../migrate/versions/0116_drop_update_available_col_add_tool_shed_status_col.py \ No newline at end of file diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 29f32282894..41698401d11 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -9,14 +9,19 @@ import random import shutil import logging import threading +from xml.etree import ElementTree -from galaxy import util -from galaxy.jobs import Sleeper -from galaxy.model import directory_hash_id -from galaxy.exceptions import ObjectNotFound, ObjectInvalid +from galaxy.util import umask_fix_perms, force_symlink +from galaxy.exceptions import ObjectInvalid, ObjectNotFound +from galaxy.util.sleeper import Sleeper +from galaxy.util.directory_hash import directory_hash_id from galaxy.util.odict import odict +try: + from sqlalchemy.orm import object_session +except ImportError: + object_session = None -from sqlalchemy.orm import object_session +NO_SESSION_ERROR_MESSAGE = "Attempted to 'create' object store entity in configuration with no database session present." log = logging.getLogger( __name__ ) @@ -202,7 +207,7 @@ class DiskObjectStore(ObjectStore): self.extra_dirs['job_work'] = config.job_working_directory self.extra_dirs['temp'] = config.new_file_path #The new config_xml overrides universe settings. - if config_xml: + if config_xml is not None: for e in config_xml: if e.tag == 'files_dir': self.file_path = e.get('path') @@ -290,8 +295,8 @@ class DiskObjectStore(ObjectStore): os.makedirs(dir) # Create the file if it does not exist if not dir_only: - open(path, 'w').close() - util.umask_fix_perms(path, self.config.umask, 0666) + open(path, 'w').close() # Should be rb? + umask_fix_perms(path, self.config.umask, 0666) def empty(self, obj, **kwargs): return os.path.getsize(self.get_filename(obj, **kwargs)) == 0 @@ -320,7 +325,7 @@ class DiskObjectStore(ObjectStore): return False def get_data(self, obj, start=0, count=-1, **kwargs): - data_file = open(self.get_filename(obj, **kwargs), 'r') + data_file = open(self.get_filename(obj, **kwargs), 'r') # Should be rb? data_file.seek(start) content = data_file.read(count) data_file.close() @@ -345,7 +350,7 @@ class DiskObjectStore(ObjectStore): if file_name and self.exists(obj, **kwargs): try: if preserve_symlinks and os.path.islink( file_name ): - util.force_symlink( os.readlink( file_name ), self.get_filename( obj, **kwargs ) ) + force_symlink( os.readlink( file_name ), self.get_filename( obj, **kwargs ) ) else: shutil.copy( file_name, self.get_filename( obj, **kwargs ) ) except IOError, ex: @@ -358,7 +363,7 @@ class DiskObjectStore(ObjectStore): def get_store_usage_percent(self): st = os.statvfs(self.file_path) - return (float(st.f_blocks - st.f_bavail)/st.f_blocks) * 100 + return ( float( st.f_blocks - st.f_bavail ) / st.f_blocks ) * 100 class CachingObjectStore(ObjectStore): @@ -428,7 +433,7 @@ class NestedObjectStore(ObjectStore): return store.__getattribute__(method)(obj, **kwargs) if default_is_exception: raise default( 'objectstore, __call_method failed: %s on %s, kwargs: %s' - %( method, str( obj ), str( kwargs ) ) ) + % ( method, str( obj ), str( kwargs ) ) ) else: return default @@ -442,7 +447,7 @@ class DistributedObjectStore(NestedObjectStore): def __init__(self, config, config_xml=None, fsmon=False): super(DistributedObjectStore, self).__init__(config, config_xml=config_xml) - if not config_xml: + if config_xml is None: self.distributed_config = config.distributed_object_store_config_file assert self.distributed_config is not None, "distributed object store ('object_store = distributed') " \ "requires a config file, please set one in " \ @@ -462,10 +467,9 @@ class DistributedObjectStore(NestedObjectStore): self.filesystem_monitor_thread.start() log.info("Filesystem space monitor started") - def __parse_distributed_config(self, config, config_xml = None): - if not config_xml: - tree = util.parse_xml(self.distributed_config) - root = tree.getroot() + def __parse_distributed_config(self, config, config_xml=None): + if config_xml is None: + root = ElementTree.parse(self.distributed_config).getroot() log.debug('Loading backends for distributed object store from %s' % self.distributed_config) else: root = config_xml.find('backends') @@ -512,7 +516,7 @@ class DistributedObjectStore(NestedObjectStore): if pct > maxpct: new_weighted_backend_ids = filter(lambda x: x != id, new_weighted_backend_ids) self.weighted_backend_ids = new_weighted_backend_ids - self.sleeper.sleep(120) # Test free space every 2 minutes + self.sleeper.sleep(120) # Test free space every 2 minutes def create(self, obj, **kwargs): """ @@ -524,9 +528,8 @@ class DistributedObjectStore(NestedObjectStore): obj.object_store_id = random.choice(self.weighted_backend_ids) except IndexError: raise ObjectInvalid( 'objectstore.create, could not generate obj.object_store_id: %s, kwargs: %s' - %( str( obj ), str( kwargs ) ) ) - object_session( obj ).add( obj ) - object_session( obj ).flush() + % ( str( obj ), str( kwargs ) ) ) + create_object_in_session( obj ) log.debug("Selected backend '%s' for creation of %s %s" % (obj.object_store_id, obj.__class__.__name__, obj.id)) else: log.debug("Using preferred backend '%s' for creation of %s %s" % (obj.object_store_id, obj.__class__.__name__, obj.id)) @@ -538,7 +541,7 @@ class DistributedObjectStore(NestedObjectStore): return self.backends[object_store_id].__getattribute__(method)(obj, **kwargs) if default_is_exception: raise default( 'objectstore, __call_method failed: %s on %s, kwargs: %s' - %( method, str( obj ), str( kwargs ) ) ) + % ( method, str( obj ), str( kwargs ) ) ) else: return default @@ -554,8 +557,7 @@ class DistributedObjectStore(NestedObjectStore): if store.exists(obj, **kwargs): log.warning('%s object with ID %s found in backend object store with ID %s' % (obj.__class__.__name__, obj.id, id)) obj.object_store_id = id - object_session( obj ).add( obj ) - object_session( obj ).flush() + create_object_in_session( obj ) return id return None @@ -594,34 +596,40 @@ def build_object_store_from_config(config, fsmon=False, config_xml=None): Depending on the configuration setting, invoke the appropriate object store """ - if not config_xml and config.object_store_config_file: + if config_xml is None and config.object_store_config_file: # This is a top level invocation of build_object_store_from_config, and # we have an object_store_conf.xml -- read the .xml and build # accordingly - tree = util.parse_xml(config.object_store_config_file) - root = tree.getroot() + root = ElementTree.parse(config.object_store_config_file).getroot() store = root.get('type') config_xml = root - elif config_xml: + elif config_xml is not None: store = config_xml.get('type') else: store = config.object_store if store == 'disk': return DiskObjectStore(config=config, config_xml=config_xml) - elif store == 's3' or store == 'swift': - from galaxy.objectstore.s3 import S3ObjectStore + elif store == 's3': + from .s3 import S3ObjectStore return S3ObjectStore(config=config, config_xml=config_xml) + elif store == 'swift': + from .s3 import SwiftObjectStore + return SwiftObjectStore(config=config, config_xml=config_xml) elif store == 'distributed': return DistributedObjectStore(config=config, fsmon=fsmon, config_xml=config_xml) elif store == 'hierarchical': return HierarchicalObjectStore(config=config, config_xml=config_xml) elif store == 'irods': - from galaxy.objectstore.rods import IRODSObjectStore + from .rods import IRODSObjectStore return IRODSObjectStore(config=config, config_xml=config_xml) + elif store == 'lwr': + from .lwr import LwrObjectStore + return LwrObjectStore(config=config, config_xml=config_xml) else: log.error("Unrecognized object store definition: {0}".format(store)) + def local_extra_dirs( func ): """ A decorator for non-local plugins to utilize local directories for their extra_dirs (job_working_directory and temp). """ @@ -635,6 +643,7 @@ def local_extra_dirs( func ): raise Exception( "Could not call DiskObjectStore's %s method, does your Object Store plugin inherit from DiskObjectStore?" % func.__name__ ) return wraps + def convert_bytes(bytes): """ A helper function used for pretty printing disk usage """ if bytes is None: @@ -656,3 +665,12 @@ def convert_bytes(bytes): else: size = '%.2fb' % bytes return size + + +def create_object_in_session( obj ): + session = object_session( obj ) if object_session is not None else None + if session is not None: + object_session( obj ).add( obj ) + object_session( obj ).flush() + else: + raise Exception( NO_SESSION_ERROR_MESSAGE ) diff --git a/lib/galaxy/objectstore/lwr.py b/lib/galaxy/objectstore/lwr.py new file mode 100644 index 00000000000..d051deeb52d --- /dev/null +++ b/lib/galaxy/objectstore/lwr.py @@ -0,0 +1,76 @@ +from __future__ import absolute_import # Need to import lwr_client absolutely. +from ..objectstore import ObjectStore +try: + from galaxy.jobs.runners.lwr_client.manager import ObjectStoreClientManager +except ImportError: + from lwr.lwr_client.manager import ObjectStoreClientManager + + +class LwrObjectStore(ObjectStore): + """ + Object store implementation that delegates to a remote LWR server. + + This may be more aspirational than practical for now, it would be good to + Galaxy to a point that a handler thread could be setup that doesn't attempt + to access the disk files returned by a (this) object store - just passing + them along to the LWR unmodified. That modification - along with this + implementation and LWR job destinations would then allow Galaxy to fully + manage jobs on remote servers with completely different mount points. + + This implementation should be considered beta and may be dropped from + Galaxy at some future point or significantly modified. + """ + + def __init__(self, config, config_xml): + self.lwr_client = self.__build_lwr_client(config_xml) + + def exists(self, obj, **kwds): + return self.lwr_client.exists(**self.__build_kwds(obj, **kwds)) + + def file_ready(self, obj, **kwds): + return self.lwr_client.file_ready(**self.__build_kwds(obj, **kwds)) + + def create(self, obj, **kwds): + return self.lwr_client.create(**self.__build_kwds(obj, **kwds)) + + def empty(self, obj, **kwds): + return self.lwr_client.empty(**self.__build_kwds(obj, **kwds)) + + def size(self, obj, **kwds): + return self.lwr_client.size(**self.__build_kwds(obj, **kwds)) + + def delete(self, obj, **kwds): + return self.lwr_client.delete(**self.__build_kwds(obj, **kwds)) + + # TODO: Optimize get_data. + def get_data(self, obj, **kwds): + return self.lwr_client.get_data(**self.__build_kwds(obj, **kwds)) + + def get_filename(self, obj, **kwds): + return self.lwr_client.get_filename(**self.__build_kwds(obj, **kwds)) + + def update_from_file(self, obj, **kwds): + return self.lwr_client.update_from_file(**self.__build_kwds(obj, **kwds)) + + def get_store_usage_percent(self): + return self.lwr_client.get_store_usage_percent() + + def get_object_url(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None): + return None + + def __build_kwds(self, obj, **kwds): + kwds['object_id'] = obj.id + return kwds + pass + + def __build_lwr_client(self, config_xml): + url = config_xml.get("url") + private_token = config_xml.get("private_token", None) + transport = config_xml.get("transport", None) + manager_options = dict(transport=transport) + client_options = dict(url=url, private_token=private_token) + lwr_client = ObjectStoreClientManager(**manager_options).get_client(client_options) + return lwr_client + + def shutdown(self): + pass diff --git a/lib/galaxy/objectstore/rods.py b/lib/galaxy/objectstore/rods.py index 852aca5e5be..4b814d55d0e 100644 --- a/lib/galaxy/objectstore/rods.py +++ b/lib/galaxy/objectstore/rods.py @@ -12,12 +12,20 @@ from posixpath import join as path_join from posixpath import basename as path_basename from posixpath import dirname as path_dirname -from galaxy.objectstore import DiskObjectStore, ObjectStore, local_extra_dirs from galaxy.exceptions import ObjectNotFound +from ..objectstore import DiskObjectStore, ObjectStore, local_extra_dirs -import galaxy.eggs -galaxy.eggs.require( 'PyRods' ) -import irods +try: + import galaxy.eggs + galaxy.eggs.require( 'PyRods' ) +except Exception: + pass +try: + import irods +except ImportError: + irods = None + +NO_PYRODS_ERROR_MESSAGE = "IRODS object store configured, but no PyRods dependency available. Please install and properly configure PyRods or modify object store configuration." log = logging.getLogger( __name__ ) @@ -27,6 +35,8 @@ class IRODSObjectStore( DiskObjectStore, ObjectStore ): Galaxy object store based on iRODS """ def __init__( self, config, file_path=None, extra_dirs=None ): + if irods is None: + raise Exception(NO_PYRODS_ERROR_MESSAGE) super( IRODSObjectStore, self ).__init__( config, file_path=file_path, extra_dirs=extra_dirs ) self.cache_path = config.object_store_cache_path self.default_resource = config.irods_default_resource or None @@ -146,7 +156,7 @@ class IRODSObjectStore( DiskObjectStore, ObjectStore ): doi = irods.dataObjInp_t() doi.objPath = rods_path doi.createMode = 0640 - doi.dataSize = 0 # 0 actually means "unknown", although literally 0 would be preferable + doi.dataSize = 0 # 0 actually means "unknown", although literally 0 would be preferable irods.addKeyVal( doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource ) status = irods.rcDataObjCreate( self.rods_conn, doi ) assert status >= 0, 'create(): rcDataObjCreate() failed: %s: %s: %s' % ( rods_path, status, irods.strerror( status ) ) @@ -237,7 +247,7 @@ class IRODSObjectStore( DiskObjectStore, ObjectStore ): incoming_path = os.path.join( os.path.dirname( cached_path ), "__incoming_%s" % os.path.basename( cached_path ) ) doi = irods.dataObjInp_t() doi.objPath = self.__get_rods_path( obj, **kwargs ) - doi.dataSize = 0 # TODO: does this affect performance? should we get size? + doi.dataSize = 0 # TODO: does this affect performance? should we get size? doi.numThreads = 0 # TODO: might want to VERIFY_CHKSUM_KW log.debug( 'get_filename(): caching %s to %s', doi.objPath, incoming_path ) @@ -296,6 +306,7 @@ class IRODSObjectStore( DiskObjectStore, ObjectStore ): def get_store_usage_percent(self): return 0.0 + # monkeypatch an strerror method into the irods module def _rods_strerror( errno ): """ @@ -309,7 +320,9 @@ def _rods_strerror( errno ): irods.__rods_strerror_map[ v ] = name return irods.__rods_strerror_map.get( errno, 'GALAXY_NO_ERRNO_MAPPING_FOUND' ) -irods.strerror = _rods_strerror +if irods is not None: + irods.strerror = _rods_strerror + def rods_connect(): """ diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index 7d45c8746d0..2062781b883 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -10,21 +10,26 @@ import threading import subprocess from datetime import datetime -from galaxy import util -from galaxy.jobs import Sleeper -from galaxy.model import directory_hash_id -from galaxy.objectstore import ObjectStore, convert_bytes +from galaxy.util import umask_fix_perms +from galaxy.util.sleeper import Sleeper +from galaxy.util.directory_hash import directory_hash_id +from ..objectstore import ObjectStore, convert_bytes from galaxy.exceptions import ObjectNotFound import multiprocessing -from galaxy.objectstore.s3_multipart_upload import multipart_upload -import boto -from boto.s3.key import Key -from boto.s3.connection import S3Connection -from boto.exception import S3ResponseError +from .s3_multipart_upload import multipart_upload +try: + import boto + from boto.s3.key import Key + from boto.s3.connection import S3Connection + from boto.exception import S3ResponseError +except ImportError: + boto = None + +NO_BOTO_ERROR_MESSAGE = "S3/Swift object store configured, but no boto dependency available. Please install and properly configure boto or modify object store configuration." log = logging.getLogger( __name__ ) -logging.getLogger('boto').setLevel(logging.INFO) # Otherwise boto is quite noisy +logging.getLogger('boto').setLevel(logging.INFO) # Otherwise boto is quite noisy class S3ObjectStore(ObjectStore): @@ -33,15 +38,15 @@ class S3ObjectStore(ObjectStore): cache exists that is used as an intermediate location for files between Galaxy and S3. """ - def __init__(self, config, config_xml=None): - super(S3ObjectStore, self).__init__() + def __init__(self, config, config_xml): + if boto is None: + raise Exception(NO_BOTO_ERROR_MESSAGE) + super(S3ObjectStore, self).__init__(config, config_xml) self.config = config self.staging_path = self.config.file_path - self.s3_conn = get_OS_connection(self.config) - self.bucket = self._get_bucket(self.config.os_bucket_name) - self.use_rr = self.config.os_use_reduced_redundancy - self.cache_size = self.config.object_store_cache_size self.transfer_progress = 0 + self._parse_config_xml(config_xml) + self._configure_connection() # Clean cache only if value is set in universe_wsgi.ini if self.cache_size != -1: # Convert GBs to bytes for comparison @@ -58,8 +63,33 @@ class S3ObjectStore(ObjectStore): except OSError: self.use_axel = False + def _configure_connection(self): + log.debug("Configuring S3 Connection") + self.conn = S3Connection(self.access_key, self.secret_key) + + def _parse_config_xml(self, config_xml): + try: + a_xml = config_xml.findall('auth')[0] + self.access_key = a_xml.get('access_key') + self.secret_key = a_xml.get('secret_key') + b_xml = config_xml.findall('bucket')[0] + self.bucket = b_xml.get('name') + self.use_rr = b_xml.get('use_reduced_redundancy', False) + cn_xml = config_xml.findall('connection')[0] + self.host = cn_xml.get('host', None) + self.port = int(cn_xml.get('port', 6000)) + self.is_secure = cn_xml.get('is_secure', True) + self.conn_path = cn_xml.get('conn_path', '/') + c_xml = config_xml.findall('cache')[0] + self.cache_size = float(c_xml.get('size', -1)) + self.cache_path = c_xml.get('path') + except Exception: + # Toss it back up after logging, we can't continue loading at this point. + log.exception("Malformed ObjectStore Configuration XML -- unable to continue") + raise + def __cache_monitor(self): - time.sleep(2) # Wait for things to load before starting the monitor + time.sleep(2) # Wait for things to load before starting the monitor while self.running: total_size = 0 # Is this going to be too expensive of an operation to be done frequently? @@ -87,7 +117,7 @@ class S3ObjectStore(ObjectStore): # For now, delete enough to leave at least 10% of the total cache free delete_this_much = total_size - cache_limit self.__clean_cache(file_list, delete_this_much) - self.sleeper.sleep(30) # Test cache size every 30 seconds? + self.sleeper.sleep(30) # Test cache size every 30 seconds? def __clean_cache(self, file_list, delete_this_much): """ Keep deleting files from the file_list until the size of the deleted @@ -127,11 +157,11 @@ class S3ObjectStore(ObjectStore): it a few times. Raise error is connection is not established. """ for i in range(5): try: - bucket = self.s3_conn.get_bucket(bucket_name) + bucket = self.conn.get_bucket(bucket_name) log.debug("Using cloud object store with bucket '%s'" % bucket.name) return bucket except S3ResponseError: - log.debug("Could not get bucket '%s', attempt %s/5" % (bucket_name, i+1)) + log.debug("Could not get bucket '%s', attempt %s/5" % (bucket_name, i + 1)) time.sleep(2) # All the attempts have been exhausted and connection was not established, # raise error @@ -140,13 +170,13 @@ class S3ObjectStore(ObjectStore): def _fix_permissions(self, rel_path): """ Set permissions on rel_path""" for basedir, dirs, files in os.walk(rel_path): - util.umask_fix_perms(basedir, self.config.umask, 0777, self.config.gid) + umask_fix_perms(basedir, self.config.umask, 0777, self.config.gid) for f in files: path = os.path.join(basedir, f) # Ignore symlinks if os.path.islink(path): continue - util.umask_fix_perms( path, self.config.umask, 0666, self.config.gid ) + umask_fix_perms( path, self.config.umask, 0666, self.config.gid ) def _construct_path(self, obj, dir_only=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, **kwargs): rel_path = os.path.join(*directory_hash_id(obj.id)) @@ -260,7 +290,7 @@ class S3ObjectStore(ObjectStore): return True else: log.debug("Pulled key '%s' into cache to %s" % (rel_path, self._get_cache_path(rel_path))) - self.transfer_progress = 0 # Reset transfer progress counter + self.transfer_progress = 0 # Reset transfer progress counter key.get_contents_to_filename(self._get_cache_path(rel_path), cb=self._transfer_cb, num_cb=10) return True except S3ResponseError, ex: @@ -290,15 +320,16 @@ class S3ObjectStore(ObjectStore): # print "Pushing cache file '%s' of size %s bytes to key '%s'" % (source_file, os.path.getsize(source_file), rel_path) # print "+ Push started at '%s'" % start_time mb_size = os.path.getsize(source_file) / 1e6 - if mb_size < 60 or self.config.object_store == 'swift': - self.transfer_progress = 0 # Reset transfer progress counter + #DBTODO Hack, refactor this logic. + if mb_size < 60 or type(self) == SwiftObjectStore: + self.transfer_progress = 0 # Reset transfer progress counter key.set_contents_from_filename(source_file, reduced_redundancy=self.use_rr, cb=self._transfer_cb, num_cb=10) else: multipart_upload(self.bucket, key.name, source_file, mb_size, use_rr=self.use_rr) end_time = datetime.now() # print "+ Push ended at '%s'; %s bytes transfered in %ssec" % (end_time, os.path.getsize(source_file), end_time-start_time) - log.debug("Pushed cache file '%s' to key '%s' (%s bytes transfered in %s sec)" % (source_file, rel_path, os.path.getsize(source_file), end_time-start_time)) + log.debug("Pushed cache file '%s' to key '%s' (%s bytes transfered in %s sec)" % (source_file, rel_path, os.path.getsize(source_file), end_time - start_time)) return True else: log.error("Tried updating key '%s' from source file '%s', but source file does not exist." @@ -384,7 +415,7 @@ class S3ObjectStore(ObjectStore): return bool(self.size(obj, **kwargs) > 0) else: raise ObjectNotFound( 'objectstore.empty, object does not exist: %s, kwargs: %s' - %( str( obj ), str( kwargs ) ) ) + % ( str( obj ), str( kwargs ) ) ) def size(self, obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) @@ -460,7 +491,7 @@ class S3ObjectStore(ObjectStore): return cache_path # Check if the file exists in persistent storage and, if it does, pull it into cache elif self.exists(obj, **kwargs): - if dir_only: # Directories do not get pulled into cache + if dir_only: # Directories do not get pulled into cache return cache_path else: if self._pull_into_cache(rel_path): @@ -470,7 +501,7 @@ class S3ObjectStore(ObjectStore): # if dir_only: # return cache_path raise ObjectNotFound( 'objectstore.get_filename, no cache_path: %s, kwargs: %s' - %( str( obj ), str( kwargs ) ) ) + % ( str( obj ), str( kwargs ) ) ) # return cache_path # Until the upload tool does not explicitly create the dataset, return expected path def update_from_file(self, obj, file_name=None, create=False, **kwargs): @@ -496,14 +527,14 @@ class S3ObjectStore(ObjectStore): self._push_to_os(rel_path, source_file) else: raise ObjectNotFound( 'objectstore.update_from_file, object does not exist: %s, kwargs: %s' - %( str( obj ), str( kwargs ) ) ) + % ( str( obj ), str( kwargs ) ) ) def get_object_url(self, obj, **kwargs): if self.exists(obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) try: key = Key(self.bucket, rel_path) - return key.generate_url(expires_in = 86400) # 24hrs + return key.generate_url(expires_in=86400) # 24hrs except S3ResponseError, ex: log.warning("Trouble generating URL for dataset '%s': %s" % (rel_path, ex)) return None @@ -512,24 +543,19 @@ class S3ObjectStore(ObjectStore): return 0.0 -def get_OS_connection(config): +class SwiftObjectStore(S3ObjectStore): """ - Get a connection object for a cloud Object Store specified in the config. - Currently, this is a ``boto`` connection object. + Object store that stores objects as items in a Swift bucket. A local + cache exists that is used as an intermediate location for files between + Galaxy and Swift. """ - log.debug("Getting a connection object for '{0}' object store".format(config.object_store)) - a_key = config.os_access_key - s_key = config.os_secret_key - if config.object_store == 's3': - return S3Connection(a_key, s_key) - else: - # Establish the connection now - calling_format = boto.s3.connection.OrdinaryCallingFormat() - s3_conn = boto.connect_s3(aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=config.os_is_secure, - host=config.os_host, - port=int(config.os_port), - calling_format=calling_format, - path=config.os_conn_path) - return s3_conn + + def _configure_connection(self): + log.debug("Configuring Swift Connection") + self.conn = boto.connect_s3(aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + is_secure=self.is_secure, + host=self.host, + port=self.port, + calling_format=boto.s3.connection.OrdinaryCallingFormat(), + path=self.conn_path) diff --git a/lib/galaxy/objectstore/s3_multipart_upload.py b/lib/galaxy/objectstore/s3_multipart_upload.py index a9f14acac96..e95729e15a1 100644 --- a/lib/galaxy/objectstore/s3_multipart_upload.py +++ b/lib/galaxy/objectstore/s3_multipart_upload.py @@ -4,10 +4,8 @@ Split large file into multiple pieces for upload to S3. This parallelizes the task over available cores using multiprocessing. Code mostly taken form CloudBioLinux. """ -from __future__ import with_statement import os -import sys import glob import subprocess import contextlib @@ -16,10 +14,17 @@ import functools import multiprocessing from multiprocessing.pool import IMapIterator -from galaxy import eggs -eggs.require('boto') +try: + from galaxy import eggs + eggs.require('boto') +except ImportError: + pass + +try: + import boto +except ImportError: + boto = None -import boto def map_wrap(f): @functools.wraps(f) diff --git a/lib/galaxy/quota/__init__.py b/lib/galaxy/quota/__init__.py index ad6890564d5..8c298d52daa 100644 --- a/lib/galaxy/quota/__init__.py +++ b/lib/galaxy/quota/__init__.py @@ -138,7 +138,11 @@ class QuotaAgent( NoQuotaAgent ): # get the usage, if it wasn't passed if usage is False: usage = self.get_usage( trans, user, history ) - return min( ( int( float( usage ) / quota * 100 ), 100 ) ) + try: + return min( ( int( float( usage ) / quota * 100 ), 100 ) ) + except ZeroDivisionError: + return 100 + def set_entity_quota_associations( self, quotas=[], users=[], groups=[], delete_existing_assocs=True ): for quota in quotas: diff --git a/lib/galaxy/security/__init__.py b/lib/galaxy/security/__init__.py index be02177f856..3a30a870b44 100644 --- a/lib/galaxy/security/__init__.py +++ b/lib/galaxy/security/__init__.py @@ -2,7 +2,9 @@ Galaxy Security """ -import logging, socket, operator +import logging +import socket +import operator from datetime import datetime, timedelta from galaxy.util.bunch import Bunch from galaxy.util import listify @@ -10,96 +12,131 @@ from galaxy.model.orm import * log = logging.getLogger(__name__) + class Action( object ): def __init__( self, action, description, model ): self.action = action self.description = description self.model = model + class RBACAgent: """Class that handles galaxy security""" permitted_actions = Bunch( - DATASET_MANAGE_PERMISSIONS = Action( "manage permissions", "Users having associated role can manage the roles associated with permissions on this dataset", "grant" ), - DATASET_ACCESS = Action( "access", "Users having associated role can import this dataset into their history for analysis", "restrict" ), - LIBRARY_ACCESS = Action( "access library", "Restrict access to this library to only users having associated role", "restrict" ), - LIBRARY_ADD = Action( "add library item", "Users having associated role can add library items to this library item", "grant" ), - LIBRARY_MODIFY = Action( "modify library item", "Users having associated role can modify this library item", "grant" ), - LIBRARY_MANAGE = Action( "manage library permissions", "Users having associated role can manage roles associated with permissions on this library item", "grant" ), + DATASET_MANAGE_PERMISSIONS=Action( "manage permissions", "Users having associated role can manage the roles associated with permissions on this dataset", "grant" ), + DATASET_ACCESS=Action( "access", "Users having associated role can import this dataset into their history for analysis", "restrict" ), + LIBRARY_ACCESS=Action( "access library", "Restrict access to this library to only users having associated role", "restrict" ), + LIBRARY_ADD=Action( "add library item", "Users having associated role can add library items to this library item", "grant" ), + LIBRARY_MODIFY=Action( "modify library item", "Users having associated role can modify this library item", "grant" ), + LIBRARY_MANAGE=Action( "manage library permissions", "Users having associated role can manage roles associated with permissions on this library item", "grant" ), # Request type permissions - REQUEST_TYPE_ACCESS = Action( "access request_type", "Restrict access to this request type to only users having associated role", "restrict" ) - + REQUEST_TYPE_ACCESS=Action( "access request_type", "Restrict access to this request type to only users having associated role", "restrict" ) ) + def get_action( self, name, default=None ): """Get a permitted action by its dict key or action name""" for k, v in self.permitted_actions.items(): if k == name or v.action == name: return v return default + def get_actions( self ): """Get all permitted actions as a list of Action objects""" return self.permitted_actions.__dict__.values() + def get_item_actions( self, action, item ): raise 'No valid method of retrieving action (%s) for item %s.' % ( action, item ) - def guess_derived_permissions_for_datasets( self, datasets = [] ): + + def guess_derived_permissions_for_datasets( self, datasets=[] ): raise "Unimplemented Method" + def can_access_dataset( self, roles, dataset ): raise "Unimplemented Method" + def can_manage_dataset( self, roles, dataset ): raise "Unimplemented Method" + def can_access_library( self, roles, library ): raise "Unimplemented Method" + def can_add_library_item( self, roles, item ): raise "Unimplemented Method" + def can_modify_library_item( self, roles, item ): raise "Unimplemented Method" + def can_manage_library_item( self, roles, item ): raise "Unimplemented Method" + def associate_components( self, **kwd ): raise 'No valid method of associating provided components: %s' % kwd + def create_private_user_role( self, user ): raise "Unimplemented Method" + def get_private_user_role( self, user ): raise "Unimplemented Method" + def get_accessible_request_types( self, trans, user ): raise "Unimplemented Method" + def user_set_default_permissions( self, user, permissions={}, history=False, dataset=False ): raise "Unimplemented Method" + def history_set_default_permissions( self, history, permissions=None, dataset=False, bypass_manage_permission=False ): raise "Unimplemented Method" + def set_all_dataset_permissions( self, dataset, permissions ): raise "Unimplemented Method" + def set_dataset_permission( self, dataset, permission ): raise "Unimplemented Method" + def set_all_library_permissions( self, trans, dataset, permissions ): raise "Unimplemented Method" + def library_is_public( self, library ): raise "Unimplemented Method" + def make_library_public( self, library ): raise "Unimplemented Method" + def get_accessible_libraries( self, trans, user ): raise "Unimplemented Method" + def get_permitted_libraries( self, trans, user, actions ): raise "Unimplemented Method" + def folder_is_public( self, library ): raise "Unimplemented Method" + def make_folder_public( self, folder, count=0 ): raise "Unimplemented Method" + def dataset_is_public( self, dataset ): raise "Unimplemented Method" + def make_dataset_public( self, dataset ): raise "Unimplemented Method" + def get_permissions( self, library_dataset ): raise "Unimplemented Method" + def get_all_roles( self, trans, cntrller ): raise "Unimplemented Method" + def get_legitimate_roles( self, trans, item, cntrller ): raise "Unimplemented Method" + def derive_roles_from_access( self, trans, item_id, cntrller, library=False, **kwd ): raise "Unimplemented Method" + def get_component_associations( self, **kwd ): raise "Unimplemented Method" + def components_are_associated( self, **kwd ): return bool( self.get_component_associations( **kwd ) ) + def convert_permitted_action_strings( self, permitted_action_strings ): """ When getting permitted actions from an untrusted source like a @@ -107,6 +144,7 @@ class RBACAgent: """ return filter( lambda x: x is not None, [ self.permitted_actions.get( action_string ) for action_string in permitted_action_strings ] ) + class GalaxyRBACAgent( RBACAgent ): def __init__( self, model, permitted_actions=None ): self.model = model @@ -118,10 +156,12 @@ class GalaxyRBACAgent( RBACAgent ): ( self.model.LibraryFolder, self.model.LibraryFolderPermissions ), ( self.model.LibraryDataset, self.model.LibraryDatasetPermissions ), ( self.model.LibraryDatasetDatasetAssociation, self.model.LibraryDatasetDatasetAssociationPermissions ) ) + @property def sa_session( self ): """Returns a SQLAlchemy session""" return self.model.context + def sort_by_attr( self, seq, attr ): """ Sort the sequence of objects by object's attribute @@ -137,19 +177,20 @@ class GalaxyRBACAgent( RBACAgent ): intermed = map( None, map( getattr, seq, ( attr, ) * len( seq ) ), xrange( len( seq ) ), seq ) intermed.sort() return map( operator.getitem, intermed, ( -1, ) * len( intermed ) ) + def get_all_roles( self, trans, cntrller ): admin_controller = cntrller in [ 'library_admin' ] roles = set() if not trans.user: return trans.sa_session.query( trans.app.model.Role ) \ - .filter( and_( self.model.Role.table.c.deleted==False, + .filter( and_( self.model.Role.table.c.deleted == False, self.model.Role.table.c.type != self.model.Role.types.PRIVATE, self.model.Role.table.c.type != self.model.Role.types.SHARING ) ) \ .order_by( self.model.Role.table.c.name ) if admin_controller: # The library is public and the user is an admin, so all roles are legitimate for role in trans.sa_session.query( trans.app.model.Role ) \ - .filter( self.model.Role.table.c.deleted==False ) \ + .filter( self.model.Role.table.c.deleted == False ) \ .order_by( self.model.Role.table.c.name ): roles.add( role ) else: @@ -160,12 +201,13 @@ class GalaxyRBACAgent( RBACAgent ): roles.add( role ) # Add all remaining non-private, non-sharing roles for role in trans.sa_session.query( trans.app.model.Role ) \ - .filter( and_( self.model.Role.table.c.deleted==False, + .filter( and_( self.model.Role.table.c.deleted == False, self.model.Role.table.c.type != self.model.Role.types.PRIVATE, self.model.Role.table.c.type != self.model.Role.types.SHARING ) ) \ .order_by( self.model.Role.table.c.name ): roles.add( role ) return self.sort_by_attr( [ role for role in roles ], 'name' ) + def get_legitimate_roles( self, trans, item, cntrller ): """ Return a sorted list of legitimate roles that can be associated with a permission on @@ -215,6 +257,7 @@ class GalaxyRBACAgent( RBACAgent ): if admin_controller or self.ok_to_display( trans.user, ura.role ): roles.add( ura.role ) return self.sort_by_attr( [ role for role in roles ], 'name' ) + def ok_to_display( self, user, role ): """ Method for checking if: @@ -257,7 +300,6 @@ class GalaxyRBACAgent( RBACAgent ): break return ret_val - def get_actions_for_items( self, trans, action, permission_items ): # TODO: Rename this; it's a replacement for get_item_actions, but it # doesn't represent what it's really doing, which is confusing. @@ -345,7 +387,6 @@ class GalaxyRBACAgent( RBACAgent ): return ret_permissions - def allow_action_on_libitems( self, trans, user_roles, action, items ): """ This should be the equivalent of allow_action defined on multiple items. @@ -359,7 +400,7 @@ class GalaxyRBACAgent( RBACAgent ): # Change item to lib_dataset or vice-versa. for item in items: - if all_items_actions.has_key( item.id ): + if item.id in all_items_actions: item_actions = all_items_actions[ item.id ] if self.permitted_actions.DATASET_ACCESS == action: @@ -399,7 +440,6 @@ class GalaxyRBACAgent( RBACAgent ): log.debug( "allow_action_for_items: test end" ) return ret_allow_action - # DELETEME: SM: DO NOT TOUCH! This actually works. def dataset_access_mapping( self, trans, user_roles, datasets ): ''' @@ -432,7 +472,8 @@ class GalaxyRBACAgent( RBACAgent ): # so this will have to be rewritten to support other items. libitems_public_map = self.datasets_are_public( trans, libitems ) libitems_allow_action_map = self.allow_action_on_libitems( - trans, user_roles, self.permitted_actions.DATASET_ACCESS, libitems ) + trans, user_roles, self.permitted_actions.DATASET_ACCESS, libitems + ) can_access = {} for libitem in libitems: can_access[ libitem.id ] = libitems_public_map[ libitem.id ] or libitems_allow_action_map[ libitem.id ] @@ -440,15 +481,18 @@ class GalaxyRBACAgent( RBACAgent ): def item_permission_map_for_modify( self, trans, user_roles, libitems ): return self.allow_action_on_libitems( - trans, user_roles, self.permitted_actions.LIBRARY_MODIFY, libitems ) + trans, user_roles, self.permitted_actions.LIBRARY_MODIFY, libitems + ) def item_permission_map_for_manage( self, trans, user_roles, libitems ): return self.allow_action_on_libitems( - trans, user_roles, self.permitted_actions.LIBRARY_MANAGE, libitems ) + trans, user_roles, self.permitted_actions.LIBRARY_MANAGE, libitems + ) def item_permission_map_for_add( self, trans, user_roles, libitems ): return self.allow_action_on_libitems( - trans, user_roles, self.permitted_actions.LIBRARY_ADD, libitems ) + trans, user_roles, self.permitted_actions.LIBRARY_ADD, libitems + ) def can_access_dataset( self, user_roles, dataset ): # SM: dataset_is_public will access dataset.actions, which is a @@ -458,8 +502,10 @@ class GalaxyRBACAgent( RBACAgent ): def can_manage_dataset( self, roles, dataset ): return self.allow_action( roles, self.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset ) + def can_access_library( self, roles, library ): return self.library_is_public( library ) or self.allow_action( roles, self.permitted_actions.LIBRARY_ACCESS, library ) + def get_accessible_libraries( self, trans, user ): """Return all data libraries that the received user can access""" accessible_libraries = [] @@ -480,6 +526,7 @@ class GalaxyRBACAgent( RBACAgent ): .order_by( trans.app.model.Library.name ): accessible_libraries.append( library ) return accessible_libraries + def has_accessible_folders( self, trans, folder, user, roles, search_downward=True ): if self.has_accessible_library_datasets( trans, folder, user, roles, search_downward=search_downward ) or \ self.can_add_library_item( roles, folder ) or \ @@ -490,20 +537,23 @@ class GalaxyRBACAgent( RBACAgent ): for folder in folder.active_folders: return self.has_accessible_folders( trans, folder, user, roles, search_downward=search_downward ) return False + def has_accessible_library_datasets( self, trans, folder, user, roles, search_downward=True ): for library_dataset in trans.sa_session.query( trans.model.LibraryDataset ) \ .filter( and_( trans.model.LibraryDataset.table.c.deleted == False, - trans.app.model.LibraryDataset.table.c.folder_id==folder.id ) ): + trans.app.model.LibraryDataset.table.c.folder_id == folder.id ) ): if self.can_access_library_item( roles, library_dataset, user ): return True if search_downward: return self.__active_folders_have_accessible_library_datasets( trans, folder, user, roles ) return False + def __active_folders_have_accessible_library_datasets( self, trans, folder, user, roles ): for active_folder in folder.active_folders: if self.has_accessible_library_datasets( trans, active_folder, user, roles ): return True return False + def can_access_library_item( self, roles, item, user ): if type( item ) == self.model.Library: return self.can_access_library( roles, item ) @@ -514,12 +564,15 @@ class GalaxyRBACAgent( RBACAgent ): elif type( item ) == self.model.LibraryDatasetDatasetAssociation: return self.can_access_library( roles, item.library_dataset.folder.parent_library ) and self.can_access_dataset( roles, item.dataset ) else: - log.warning( 'Unknown library item type: %s' % type ( item ) ) + log.warning( 'Unknown library item type: %s' % type( item ) ) return False + def can_add_library_item( self, roles, item ): return self.allow_action( roles, self.permitted_actions.LIBRARY_ADD, item ) + def can_modify_library_item( self, roles, item ): return self.allow_action( roles, self.permitted_actions.LIBRARY_MODIFY, item ) + def can_manage_library_item( self, roles, item ): return self.allow_action( roles, self.permitted_actions.LIBRARY_MANAGE, item ) @@ -554,6 +607,7 @@ class GalaxyRBACAgent( RBACAgent ): # join existing roles with new roles perms[ action ].extend( filter( lambda x: x not in perms[ action ], roles ) ) return perms + def associate_components( self, **kwd ): if 'user' in kwd: if 'group' in kwd: @@ -567,26 +621,31 @@ class GalaxyRBACAgent( RBACAgent ): if 'dataset' in kwd and 'role' in kwd: return self.associate_action_dataset_role( kwd['action'], kwd['dataset'], kwd['role'] ) raise 'No valid method of associating provided components: %s' % kwd + def associate_user_group( self, user, group ): assoc = self.model.UserGroupAssociation( user, group ) self.sa_session.add( assoc ) self.sa_session.flush() return assoc + def associate_user_role( self, user, role ): assoc = self.model.UserRoleAssociation( user, role ) self.sa_session.add( assoc ) self.sa_session.flush() return assoc + def associate_group_role( self, group, role ): assoc = self.model.GroupRoleAssociation( group, role ) self.sa_session.add( assoc ) self.sa_session.flush() return assoc + def associate_action_dataset_role( self, action, dataset, role ): assoc = self.model.DatasetPermissions( action, dataset, role ) self.sa_session.add( assoc ) self.sa_session.flush() return assoc + def create_private_user_role( self, user ): # Create private role role = self.model.Role( name=user.email, description='Private Role for ' + user.email, type=self.model.Role.types.PRIVATE ) @@ -595,6 +654,7 @@ class GalaxyRBACAgent( RBACAgent ): # Add user to role self.associate_components( role=role, user=user ) return role + def get_private_user_role( self, user, auto_create=False ): role = self.sa_session.query( self.model.Role ) \ .filter( and_( self.model.Role.table.c.name == user.email, @@ -606,11 +666,13 @@ class GalaxyRBACAgent( RBACAgent ): else: return None return role + def get_sharing_roles( self, user ): return self.sa_session.query( self.model.Role ) \ .filter( and_( ( self.model.Role.table.c.name ).like( "Sharing role for: %" + user.email + "%" ), self.model.Role.table.c.type == self.model.Role.types.SHARING ) ) - def user_set_default_permissions( self, user, permissions={}, history=False, dataset=False, bypass_manage_permission=False, default_access_private = False ): + + def user_set_default_permissions( self, user, permissions={}, history=False, dataset=False, bypass_manage_permission=False, default_access_private=False ): # bypass_manage_permission is used to change permissions of datasets in a userless history when logging in flush_needed = False if user is None: @@ -637,6 +699,7 @@ class GalaxyRBACAgent( RBACAgent ): if history: for history in user.active_histories: self.history_set_default_permissions( history, permissions=permissions, dataset=dataset, bypass_manage_permission=bypass_manage_permission ) + def user_get_default_permissions( self, user ): permissions = {} for dup in user.default_permissions: @@ -646,6 +709,7 @@ class GalaxyRBACAgent( RBACAgent ): else: permissions[ action ] = [ dup.role ] return permissions + def history_set_default_permissions( self, history, permissions={}, dataset=False, bypass_manage_permission=False ): # bypass_manage_permission is used to change permissions of datasets in a user-less history when logging in flush_needed = False @@ -680,6 +744,7 @@ class GalaxyRBACAgent( RBACAgent ): continue if bypass_manage_permission or self.can_manage_dataset( user.all_roles(), dataset ): self.set_all_dataset_permissions( dataset, permissions ) + def history_get_default_permissions( self, history ): permissions = {} for dhp in history.default_permissions: @@ -689,6 +754,7 @@ class GalaxyRBACAgent( RBACAgent ): else: permissions[ action ] = [ dhp.role ] return permissions + def set_all_dataset_permissions( self, dataset, permissions={} ): """ Set new permissions on a dataset, eliminating all current permissions @@ -721,6 +787,7 @@ class GalaxyRBACAgent( RBACAgent ): if flush_needed: self.sa_session.flush() return "" + def set_dataset_permission( self, dataset, permission={} ): """ Set a specific permission on a dataset, leaving all other current permissions on the dataset alone @@ -741,6 +808,7 @@ class GalaxyRBACAgent( RBACAgent ): flush_needed = True if flush_needed: self.sa_session.flush() + def get_permissions( self, item ): """ Return a dictionary containing the actions and associated roles on item @@ -755,6 +823,7 @@ class GalaxyRBACAgent( RBACAgent ): else: permissions[ action ] = [ item_permission.role ] return permissions + def get_accessible_request_types( self, trans, user ): """Return all RequestTypes that the received user has permission to access.""" accessible_request_types = [] @@ -775,13 +844,15 @@ class GalaxyRBACAgent( RBACAgent ): .order_by( trans.app.model.RequestType.name ): accessible_request_types.append( request_type ) return accessible_request_types + def copy_dataset_permissions( self, src, dst ): if not isinstance( src, self.model.Dataset ): src = src.dataset if not isinstance( dst, self.model.Dataset ): dst = dst.dataset self.set_all_dataset_permissions( dst, self.get_permissions( src ) ) - def privately_share_dataset( self, dataset, users = [] ): + + def privately_share_dataset( self, dataset, users=[] ): intersect = None for user in users: roles = [ ura.role for ura in user.roles if ura.role.type == self.model.Role.types.SHARING ] @@ -801,13 +872,14 @@ class GalaxyRBACAgent( RBACAgent ): sharing_role = role break if sharing_role is None: - sharing_role = self.model.Role( name = "Sharing role for: " + ", ".join( [ u.email for u in users ] ), - type = self.model.Role.types.SHARING ) + sharing_role = self.model.Role( name="Sharing role for: " + ", ".join( [ u.email for u in users ] ), + type=self.model.Role.types.SHARING ) self.sa_session.add( sharing_role ) self.sa_session.flush() for user in users: self.associate_components( user=user, role=sharing_role ) self.set_dataset_permission( dataset, { self.permitted_actions.DATASET_ACCESS : [ sharing_role ] } ) + def set_all_library_permissions( self, trans, library_item, permissions={} ): # Set new permissions on library_item, eliminating all current permissions flush_needed = False @@ -828,6 +900,9 @@ class GalaxyRBACAgent( RBACAgent ): # so it is possible that some Datasets have no roles associated with the DATASET_MANAGE_PERMISSIONS # permission. In this case, we'll reset this permission to the library_item user's private role. if not library_item.dataset.has_manage_permissions_roles( trans ): + # Well this looks like a bug, this should be looked at. + # Default permissions above is single hash that keeps getting reeditted here + # because permission is being defined instead of permissions. -John permission = {} permissions[ self.permitted_actions.DATASET_MANAGE_PERMISSIONS ] = [ trans.app.security_agent.get_private_user_role( library_item.user ) ] self.set_dataset_permission( library_item.dataset, permissions ) @@ -840,6 +915,7 @@ class GalaxyRBACAgent( RBACAgent ): self.set_dataset_permission( library_item.dataset, permissions ) if flush_needed: self.sa_session.flush() + def library_is_public( self, library, contents=False ): if contents: # Check all contained folders and datasets to find any that are not public @@ -847,6 +923,7 @@ class GalaxyRBACAgent( RBACAgent ): return False # A library is considered public if there are no "access" actions associated with it. return self.permitted_actions.LIBRARY_ACCESS.action not in [ a.action for a in library.actions ] + def make_library_public( self, library, contents=False ): flush_needed = False if contents: @@ -859,6 +936,7 @@ class GalaxyRBACAgent( RBACAgent ): flush_needed = True if flush_needed: self.sa_session.flush() + def folder_is_public( self, folder ): for sub_folder in folder.folders: if not self.folder_is_public( sub_folder ): @@ -868,6 +946,7 @@ class GalaxyRBACAgent( RBACAgent ): if ldda and ldda.dataset and not self.dataset_is_public( ldda.dataset ): return False return True + def make_folder_public( self, folder ): # Make all of the contents (include deleted contents, but not purged contents) of folder public for sub_folder in folder.folders: @@ -909,7 +988,6 @@ class GalaxyRBACAgent( RBACAgent ): datasets_public[ permission.dataset_id ] = False return datasets_public - def make_dataset_public( self, dataset ): # A dataset is considered public if there are no "access" actions associated with it. Any # other actions ( 'manage permissions', 'edit metadata' ) are irrelevant. @@ -920,6 +998,7 @@ class GalaxyRBACAgent( RBACAgent ): flush_needed = True if flush_needed: self.sa_session.flush() + def derive_roles_from_access( self, trans, item_id, cntrller, library=False, **kwd ): # Check the access permission on a dataset. If library is true, item_id refers to a library. If library # is False, item_id refers to a dataset ( item_id must currently be decoded before being sent ). The @@ -1017,6 +1096,7 @@ class GalaxyRBACAgent( RBACAgent ): else: permissions[ self.get_action( v.action ) ] = in_roles return permissions, in_roles, error, msg + def copy_library_permissions( self, trans, source_library_item, target_library_item, user=None ): # Copy all relevant permissions from source. permissions = {} @@ -1037,13 +1117,14 @@ class GalaxyRBACAgent( RBACAgent ): # Make sure user's private role is included private_role = self.model.security_agent.get_private_user_role( user ) for name, action in self.permitted_actions.items(): - if not permission_class.filter_by( role_id = private_role.id, action = action.action ).first(): + if not permission_class.filter_by( role_id=private_role.id, action=action.action ).first(): lp = permission_class( action.action, target_library_item, private_role ) self.sa_session.add( lp ) self.sa_session.flush() else: raise 'Invalid class (%s) specified for target_library_item (%s)' % \ ( target_library_item.__class__, target_library_item.__class__.__name__ ) + def get_permitted_libraries( self, trans, user, actions ): """ This method is historical (it is not currently used), but may be useful again at some @@ -1073,6 +1154,7 @@ class GalaxyRBACAgent( RBACAgent ): if can_show: libraries[ library ] = hidden_folder_ids return libraries + def show_library_item( self, user, roles, library_item, actions_to_check, hidden_folder_ids='' ): """ This method must be sent an instance of Library() or LibraryFolder(). Recursive execution produces a @@ -1097,6 +1179,7 @@ class GalaxyRBACAgent( RBACAgent ): else: hidden_folder_ids = '%d' % folder.id return False, hidden_folder_ids + def get_showable_folders( self, user, roles, library_item, actions_to_check, hidden_folder_ids=[], showable_folders=[] ): """ This method must be sent an instance of Library(), all the folders of which are scanned to determine if @@ -1115,6 +1198,7 @@ class GalaxyRBACAgent( RBACAgent ): for folder in library_item.active_folders: self.get_showable_folders( user, roles, folder, actions_to_check, showable_folders=showable_folders ) return showable_folders + def set_entity_user_associations( self, users=[], roles=[], groups=[], delete_existing_assocs=True ): for user in users: if delete_existing_assocs: @@ -1131,6 +1215,7 @@ class GalaxyRBACAgent( RBACAgent ): self.associate_components( user=user, role=role ) for group in groups: self.associate_components( user=user, group=group ) + def set_entity_group_associations( self, groups=[], users=[], roles=[], delete_existing_assocs=True ): for group in groups: if delete_existing_assocs: @@ -1144,6 +1229,7 @@ class GalaxyRBACAgent( RBACAgent ): self.associate_components( group=group, role=role ) for user in users: self.associate_components( group=group, user=user ) + def set_entity_role_associations( self, roles=[], users=[], groups=[], delete_existing_assocs=True ): for role in roles: if delete_existing_assocs: @@ -1157,20 +1243,22 @@ class GalaxyRBACAgent( RBACAgent ): self.associate_components( user=user, role=role ) for group in groups: self.associate_components( group=group, role=role ) + def get_component_associations( self, **kwd ): assert len( kwd ) == 2, 'You must specify exactly 2 Galaxy security components to check for associations.' if 'dataset' in kwd: if 'action' in kwd: - return self.sa_session.query( self.model.DatasetPermissions ).filter_by( action = kwd['action'].action, dataset_id = kwd['dataset'].id ).first() + return self.sa_session.query( self.model.DatasetPermissions ).filter_by( action=kwd['action'].action, dataset_id=kwd['dataset'].id ).first() elif 'user' in kwd: if 'group' in kwd: - return self.sa_session.query( self.model.UserGroupAssociation ).filter_by( group_id = kwd['group'].id, user_id = kwd['user'].id ).first() + return self.sa_session.query( self.model.UserGroupAssociation ).filter_by( group_id=kwd['group'].id, user_id=kwd['user'].id ).first() elif 'role' in kwd: - return self.sa_session.query( self.model.UserRoleAssociation ).filter_by( role_id = kwd['role'].id, user_id = kwd['user'].id ).first() + return self.sa_session.query( self.model.UserRoleAssociation ).filter_by( role_id=kwd['role'].id, user_id=kwd['user'].id ).first() elif 'group' in kwd: if 'role' in kwd: - return self.sa_session.query( self.model.GroupRoleAssociation ).filter_by( role_id = kwd['role'].id, group_id = kwd['group'].id ).first() + return self.sa_session.query( self.model.GroupRoleAssociation ).filter_by( role_id=kwd['role'].id, group_id=kwd['group'].id ).first() raise 'No valid method of associating provided components: %s' % kwd + def check_folder_contents( self, user, roles, folder, hidden_folder_ids='' ): """ This method must always be sent an instance of LibraryFolder(). Recursive execution produces a @@ -1210,6 +1298,7 @@ class GalaxyRBACAgent( RBACAgent ): else: hidden_folder_ids = '%d' % sub_folder.id return False, hidden_folder_ids + def can_access_request_type( self, roles, request_type ): action = self.permitted_actions.REQUEST_TYPE_ACCESS request_type_actions = [] @@ -1224,12 +1313,12 @@ class GalaxyRBACAgent( RBACAgent ): ret_val = True break return ret_val + def set_request_type_permissions( self, request_type, permissions={} ): # Set new permissions on request_type, eliminating all current permissions for role_assoc in request_type.actions: self.sa_session.delete( role_assoc ) # Add the new permissions on request_type - item_class = self.model.RequestType permission_class = self.model.RequestTypePermissions flush_needed = False for action, roles in permissions.items(): @@ -1241,6 +1330,7 @@ class GalaxyRBACAgent( RBACAgent ): if flush_needed: self.sa_session.flush() + class HostAgent( RBACAgent ): """ A simple security agent which allows access to datasets based on host. @@ -1249,30 +1339,33 @@ class HostAgent( RBACAgent ): """ # TODO: Make sites user configurable sites = Bunch( - ucsc_main = ( 'hgw1.cse.ucsc.edu', 'hgw2.cse.ucsc.edu', 'hgw3.cse.ucsc.edu', 'hgw4.cse.ucsc.edu', + ucsc_main=( 'hgw1.cse.ucsc.edu', 'hgw2.cse.ucsc.edu', 'hgw3.cse.ucsc.edu', 'hgw4.cse.ucsc.edu', 'hgw5.cse.ucsc.edu', 'hgw6.cse.ucsc.edu', 'hgw7.cse.ucsc.edu', 'hgw8.cse.ucsc.edu' ), - ucsc_test = ( 'hgwdev.cse.ucsc.edu', ), - ucsc_archaea = ( 'lowepub.cse.ucsc.edu', ) + ucsc_test=( 'hgwdev.cse.ucsc.edu', ), + ucsc_archaea=( 'lowepub.cse.ucsc.edu', ) ) + def __init__( self, model, permitted_actions=None ): self.model = model if permitted_actions: self.permitted_actions = permitted_actions + @property def sa_session( self ): """Returns a SQLAlchemy session""" return self.model.context + def allow_action( self, addr, action, **kwd ): if 'dataset' in kwd and action == self.permitted_actions.DATASET_ACCESS: hda = kwd['dataset'] if action == self.permitted_actions.DATASET_ACCESS and action.action not in [ dp.action for dp in hda.dataset.actions ]: log.debug( 'Allowing access to public dataset with hda: %i.' % hda.id ) - return True # dataset has no roles associated with the access permission, thus is already public + return True # dataset has no roles associated with the access permission, thus is already public hdadaa = self.sa_session.query( self.model.HistoryDatasetAssociationDisplayAtAuthorization ) \ - .filter_by( history_dataset_association_id = hda.id ).first() + .filter_by( history_dataset_association_id=hda.id ).first() if not hdadaa: log.debug( 'Denying access to private dataset with hda: %i. No hdadaa record for this dataset.' % hda.id ) - return False # no auth + return False # no auth # We could just look up the reverse of addr, but then we'd also # have to verify it with the forward address and special case any # IPs (instead of hosts) in the server list. @@ -1284,22 +1377,23 @@ class HostAgent( RBACAgent ): # balancing their connections (as UCSC does), this is okay. try: if socket.gethostbyname( server ) == addr: - break # remote host is in the server list + break # remote host is in the server list except ( socket.error, socket.gaierror ): - pass # can't resolve, try next + pass # can't resolve, try next else: log.debug( 'Denying access to private dataset with hda: %i. Remote addr is not a valid server for site: %s.' % ( hda.id, hdadaa.site ) ) - return False # remote addr is not in the server list + return False # remote addr is not in the server list if ( datetime.utcnow() - hdadaa.update_time ) > timedelta( seconds=60 ): log.debug( 'Denying access to private dataset with hda: %i. Authorization was granted, but has expired.' % hda.id ) - return False # not authz'd in the last 60 seconds + return False # not authz'd in the last 60 seconds log.debug( 'Allowing access to private dataset with hda: %i. Remote server is: %s.' % ( hda.id, server ) ) return True else: raise 'The dataset access permission is the only valid permission in the host security agent.' + def set_dataset_permissions( self, hda, user, site ): hdadaa = self.sa_session.query( self.model.HistoryDatasetAssociationDisplayAtAuthorization ) \ - .filter_by( history_dataset_association_id = hda.id ).first() + .filter_by( history_dataset_association_id=hda.id ).first() if hdadaa: hdadaa.update_time = datetime.utcnow() else: @@ -1307,6 +1401,7 @@ class HostAgent( RBACAgent ): self.sa_session.add( hdadaa ) self.sa_session.flush() + def get_permitted_actions( filter=None ): '''Utility method to return a subset of RBACAgent's permitted actions''' if filter is None: diff --git a/lib/galaxy/security/passwords.py b/lib/galaxy/security/passwords.py index bdcc2514738..8c720cd0418 100644 --- a/lib/galaxy/security/passwords.py +++ b/lib/galaxy/security/passwords.py @@ -2,9 +2,10 @@ import hmac import hashlib from struct import Struct from operator import xor -from itertools import izip, starmap +from itertools import starmap from os import urandom from base64 import b64encode +from galaxy.util import safe_str_cmp SALT_LENGTH = 12 KEY_LENGTH = 24 @@ -73,11 +74,3 @@ def pbkdf2_bin( data, salt, iterations=1000, keylen=24, hashfunc=None ): rv = starmap( xor, zip( rv, u ) ) #Python < 2.6.8: starmap requires function inputs to be tuples, so we need to use zip instead of izip buf.extend(rv) return ''.join(map(chr, buf))[:keylen] - -def safe_str_cmp(a, b): - if len(a) != len(b): - return False - rv = 0 - for x, y in izip(a, b): - rv |= ord(x) ^ ord(y) - return rv == 0 \ No newline at end of file diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index d72a648bda9..e81930b87e1 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -4,6 +4,7 @@ Classes encapsulating galaxy tools and tool configuration. import binascii import glob +import json import logging import os import pipes @@ -19,14 +20,12 @@ import urllib from math import isinf from galaxy import eggs -eggs.require( "simplejson" ) -eggs.require( "MarkupSafe" ) #MarkupSafe must load before mako +eggs.require( "MarkupSafe" ) # MarkupSafe must load before mako eggs.require( "Mako" ) eggs.require( "elementtree" ) eggs.require( "Paste" ) eggs.require( "SQLAlchemy >= 0.4" ) -import simplejson from cgi import FieldStorage from elementtree import ElementTree from mako.template import Template @@ -40,7 +39,7 @@ from galaxy.jobs import ParallelismInfo from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions.data_source import DataSourceToolAction from galaxy.tools.actions.data_manager import DataManagerToolAction -from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY +from galaxy.tools.deps import build_dependency_manager from galaxy.tools.deps.requirements import parse_requirements_from_xml from galaxy.tools.parameters import check_param, params_from_strings, params_to_strings from galaxy.tools.parameters.basic import (BaseURLToolParameter, @@ -52,7 +51,7 @@ from galaxy.tools.parameters.input_translation import ToolInputTranslator from galaxy.tools.parameters.output import ToolOutputActionGroup from galaxy.tools.parameters.validation import LateValidationError from galaxy.tools.filters import FilterFactory -from galaxy.tools.test import ToolTestBuilder +from galaxy.tools.test import parse_tests_elem from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object, xml_text, xml_to_string from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext @@ -63,17 +62,20 @@ from galaxy.util.template import fill_template from galaxy.web import url_for from galaxy.web.form_builder import SelectField from galaxy.model.item_attrs import Dictifiable +from galaxy.model import Workflow from tool_shed.util import shed_util_common from .loader import load_tool, template_macro_params log = logging.getLogger( __name__ ) -WORKFLOW_PARAMETER_REGULAR_EXPRESSION = re.compile( '''\$\{.+?\}''' ) +WORKFLOW_PARAMETER_REGULAR_EXPRESSION = re.compile( '''\$\{.+?\}''' ) + class ToolNotFoundException( Exception ): pass + def to_dict_helper( obj, kwargs ): """ Helper function that provides the appropriate kwargs to to_dict an object. """ @@ -83,6 +85,7 @@ def to_dict_helper( obj, kwargs ): return obj.to_dict( **kwargs ) + class ToolBox( object, Dictifiable ): """Container for a collection of tools""" @@ -143,13 +146,13 @@ class ToolBox( object, Dictifiable ): # 1) adjusting init_tools() and called methods to get this right # 2) redesigning the code and/or data structure used to read/write integrated_tool_panel.xml for key, value in self.integrated_tool_panel.iteritems(): - if key.startswith( 'section_' ): + if isinstance( value, ToolSection ): for section_key, section_value in value.elems.iteritems(): if section_value is None: - if section_key.startswith( 'tool_' ): + if isinstance( section_value, Tool ): tool_id = section_key[5:] value.elems[section_key] = self.tools_by_id.get( tool_id ) - elif section_key.startswith( 'workflow_' ): + elif isinstance( section_value, Workflow ): workflow_id = section_key[9:] value.elems[section_key] = self.workflows_by_id.get( workflow_id ) @@ -253,19 +256,19 @@ class ToolBox( object, Dictifiable ): def load_tool_panel( self ): for key, val in self.integrated_tool_panel.items(): - if key.startswith( 'tool_' ): + if isinstance( val, Tool ): tool_id = key.replace( 'tool_', '', 1 ) if tool_id in self.tools_by_id: self.__add_tool_to_tool_panel( tool_id, self.tool_panel, section=False ) - elif key.startswith( 'workflow_' ): + elif isinstance( val, Workflow ): workflow_id = key.replace( 'workflow_', '', 1 ) if workflow_id in self.workflows_by_id: workflow = self.workflows_by_id[ workflow_id ] self.tool_panel[ key ] = workflow log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) - elif key.startswith( 'label_' ): + elif isinstance( val, ToolSectionLabel ): self.tool_panel[ key ] = val - elif key.startswith( 'section_' ): + elif isinstance( val, ToolSection ): elem = ElementTree.Element( 'section' ) elem.attrib[ 'id' ] = val.id or '' elem.attrib[ 'name' ] = val.name or '' @@ -277,13 +280,13 @@ class ToolBox( object, Dictifiable ): tool_id = section_key.replace( 'tool_', '', 1 ) if tool_id in self.tools_by_id: self.__add_tool_to_tool_panel( tool_id, section, section=True ) - elif section_key.startswith( 'workflow_' ): + elif isinstance( section_val, Workflow ): workflow_id = section_key.replace( 'workflow_', '', 1 ) if workflow_id in self.workflows_by_id: workflow = self.workflows_by_id[ workflow_id ] section.elems[ section_key ] = workflow log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) - elif section_key.startswith( 'label_' ): + elif isinstance( val, ToolSectionLabel ): if section_val: section.elems[ section_key ] = section_val log.debug( "Loaded label: %s" % ( section_val.text ) ) @@ -316,7 +319,7 @@ class ToolBox( object, Dictifiable ): elif section_elem.tag == 'label': key = 'label_%s' % section_elem.get( 'id' ) section.elems[ key ] = None - key = 'section_%s' % elem.get( 'id' ) + key = elem.get( 'id' ) self.integrated_tool_panel[ key ] = section elif elem.tag == 'label': key = 'label_%s' % elem.get( 'id' ) @@ -332,28 +335,28 @@ class ToolBox( object, Dictifiable ): os.write( fd, '\n' ) for key, item in self.integrated_tool_panel.items(): if item: - if key.startswith( 'tool_' ): + if isinstance( item, Tool ): os.write( fd, ' \n' % item.id ) - elif key.startswith( 'workflow_' ): + elif isinstance( item, Workflow ): os.write( fd, ' \n' % item.id ) - elif key.startswith( 'label_' ): + elif isinstance( item, ToolSectionLabel ): label_id = item.id or '' label_text = item.text or '' label_version = item.version or '' os.write( fd, '