opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/dev' into ldap

Browse Source
This commit is contained in:
Anthony Bretaudeau
2016-07-22 09:22:08 +02:00
parent 8b9d77ee3b 8dbf987e10
commit b08dfa2e72
379 changed files with 8794 additions and 4454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/py3_sources.txt
+32 -15
View File
@@ -3,29 +3,40 @@ cron/
lib/galaxy/actions/
lib/galaxy/auth/
lib/galaxy/config.py
lib/galaxy/dataset_collections/
lib/galaxy/datatypes/binary.py
lib/galaxy/datatypes/converters/
lib/galaxy/datatypes/dataproviders/
lib/galaxy/datatypes/sequence.py
lib/galaxy/datatypes/sniff.py
lib/galaxy/datatypes/tabular.py
lib/galaxy/dependencies/
lib/galaxy/eggs/
lib/galaxy/exceptions/
lib/galaxy/external_services/
lib/galaxy/forms/
lib/galaxy/jobs/
lib/galaxy/managers/
lib/galaxy/model/__init__.py
lib/galaxy/model/migrate/
lib/galaxy/objectstore/
lib/galaxy/openid/
lib/galaxy/quota/
lib/galaxy/sample_tracking/
lib/galaxy/security/
lib/galaxy/tags/
lib/galaxy/tools/cwl/
lib/galaxy/tools/parser/
lib/galaxy/tools/deps/
lib/galaxy/tools/lint.py
lib/galaxy/tools/lint_util.py
lib/galaxy/tools/linters/
lib/galaxy/tools/loader.py
lib/galaxy/tools/loader_directory.py
lib/galaxy/tools/linters/
lib/galaxy/tools/deps/
lib/galaxy/tools/toolbox/
lib/galaxy/tools/parser/
lib/galaxy/tools/toolbox/
lib/galaxy/tours/
lib/galaxy/util/
lib/galaxy/visualization/
lib/galaxy/work/
lib/galaxy_ext/
lib/galaxy_utils/
@@ -40,22 +51,28 @@ lib/tool_shed/repository_types/
lib/tool_shed/tools/
lib/tool_shed/util/
lib/tool_shed/utility_containers/
scripts/api/common.py
scripts/api/display.py
scripts/api/workflow_execute_parameters.py
scripts/api/
scripts/auth/
scripts/bootstrap_history.py
scripts/build_toolbox.py
scripts/check_eggs.py
scripts/check_galaxy.py
scripts/check_python.py
scripts/cleanup_datasets/admin_cleanup_datasets.py
scripts/cleanup_datasets/cleanup_datasets.py
test/api/test_workflows_from_yaml.py
test/base/
test/casperjs/
test/functional/
test/integration/
test/manual/
test/unit/tools/test_actions.py
test/unit/workflows/test_run_parameters.py
scripts/cleanup_datasets/pgcleanup.py
scripts/cleanup_datasets/populate_uuid.py
scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py
scripts/cleanup_datasets/rename_purged_datasets.py
scripts/cleanup_datasets/update_dataset_size.py
scripts/cleanup_datasets/update_metadata.py
scripts/data_libraries/build_whoosh_index.py
scripts/db_shell.py
scripts/drmaa_external_runner.py
test/
tool_list.py
tools/data_source/
tools/evolution/
tools/sr_mapping/
tools/stats/aggregate_scores_in_intervals.py
tools/visualization/
Makefile
+6 -2
View File
@@ -12,6 +12,7 @@ VENV?=.venv
IN_VENV=if [ -f $(VENV)/bin/activate ]; then . $(VENV)/bin/activate; fi;
PROJECT_URL?=https://github.com/galaxyproject/galaxy
GRUNT_DOCKER_NAME:=galaxy/client-builder:16.01
GRUNT_EXEC?=node_modules/grunt-cli/bin/grunt
all: help
@echo "This makefile is primarily used for building Galaxy's JS client. A sensible all target is not yet implemented."
@@ -59,10 +60,13 @@ npm-deps: ## Install NodeJS dependencies.
cd client && npm install
grunt: npm-deps ## Calls out to Grunt to build client
cd client && node_modules/grunt-cli/bin/grunt
cd client && $(GRUNT_EXEC)
style: npm-deps ## Calls the style task of Grunt
cd client && node_modules/grunt-cli/bin/grunt style
cd client && $(GRUNT_EXEC) style
client-install-libs: npm-deps ## Fetch updated client dependencies using bower.
cd client && $(GRUNT_EXEC) install-libs
client: grunt style ## Rebuild all client-side artifacts
client/galaxy/scripts/mvc/collection/collection-li.js
+2
View File
@@ -125,6 +125,8 @@ DCListItemView.prototype.templates = (function(){
_l( 'a pair of datasets' ),
'<% } else if( collection.collection_type === "list:paired" ){ %>',
_l( 'a list of <%- countText %>dataset pairs' ),
'<% } else if( collection.collection_type === "list:list" ){ %>',
_l( 'a list of <%- countText %>dataset lists' ),
'<% } %>',
'</div>'
], 'collection' );
client/galaxy/scripts/mvc/collection/collection-model.js
+51 -3
View File
@@ -504,10 +504,58 @@ var ListPairedDatasetCollection = DatasetCollection.extend(
});
//==============================================================================
/** @class Backbone model for a list dataset collection within a list:list dataset collection. */
var NestedListDCDCE = ListDatasetCollection.extend( BASE_MVC.mixin( DatasetCollectionElementMixin,
/** @lends NestedListDCDCE.prototype */{
/** This is both a collection and a collection element - call the constructor */
constructor : function( attributes, options ){
this.debug( '\t NestedListDCDCE.constructor:', attributes, options );
DatasetCollectionElementMixin.constructor.call( this, attributes, options );
},
/** String representation. */
toString : function(){
var objStr = ( this.object )?( '' + this.object ):( this.get( 'element_identifier' ) );
return ([ 'NestedListDCDCE(', objStr, ')' ].join( '' ));
}
}));
//==============================================================================
/** @class Backbone collection containing list dataset collections. */
var NestedListDCDCECollection = NestedDCDCECollection.extend({
/** We know this collection is composed of only nested pair collections */
model: NestedListDCDCE,
/** String representation. */
toString : function(){
return ([ 'NestedListDCDCECollection(', this.length, ')' ].join( '' ));
}
});
//==============================================================================
/** @class Backbone Model for a DatasetCollection (list) that contains other lists. */
var ListOfListsDatasetCollection = DatasetCollection.extend({
/** list:paired is the only collection that itself contains collections */
collectionClass : NestedListDCDCECollection,
/** String representation. */
toString : function(){
return ([ 'ListOfListsDatasetCollection(', this.get( 'name' ), ')' ].join( '' ));
}
});
//==============================================================================
return {
ListDatasetCollection : ListDatasetCollection,
PairDatasetCollection : PairDatasetCollection,
ListPairedDatasetCollection : ListPairedDatasetCollection
ListDatasetCollection : ListDatasetCollection,
PairDatasetCollection : PairDatasetCollection,
ListPairedDatasetCollection : ListPairedDatasetCollection,
ListOfListsDatasetCollection: ListOfListsDatasetCollection
};
});
client/galaxy/scripts/mvc/collection/collection-view-edit.js
+22 -2
View File
@@ -128,7 +128,7 @@ var NestedPairCollectionViewEdit = PairCollectionViewEdit.extend(
// =============================================================================
/** @class non-editable, read-only View/Controller for a dataset collection. */
/** @class editable, View/Controller for a list of pairs dataset collection. */
var ListOfPairsCollectionViewEdit = CollectionViewEdit.extend(
/** @lends ListOfPairsCollectionView.prototype */{
@@ -146,11 +146,31 @@ var ListOfPairsCollectionViewEdit = CollectionViewEdit.extend(
});
// =============================================================================
/** @class View/Controller for a list of lists dataset collection. */
var ListOfListsCollectionViewEdit = CollectionViewEdit.extend(
/** @lends ListOfListsCollectionView.prototype */{
//TODO: not strictly needed - due to switch in CollectionView._getContentClass
/** sub view class used for nested collections */
NestedDCDCEViewClass : DC_EDIT.NestedDCDCEListItemEdit.extend({
foldoutPanelClass : NestedPairCollectionViewEdit
}),
// ........................................................................ misc
/** string rep */
toString : function(){
return 'ListOfListsCollectionViewEdit(' + (( this.model )?( this.model.get( 'name' )):( '' )) + ')';
}
});
//==============================================================================
return {
CollectionViewEdit : CollectionViewEdit,
ListCollectionViewEdit : ListCollectionViewEdit,
PairCollectionViewEdit : PairCollectionViewEdit,
ListOfPairsCollectionViewEdit : ListOfPairsCollectionViewEdit
ListOfPairsCollectionViewEdit : ListOfPairsCollectionViewEdit,
ListOfListsCollectionViewEdit : ListOfListsCollectionViewEdit
};
});
client/galaxy/scripts/mvc/collection/collection-view.js
+20 -1
View File
@@ -153,6 +153,8 @@ CollectionView.prototype.templates = (function(){
_l( 'a pair of datasets' ),
'<% } else if( collection.collection_type === "list:paired" ){ %>',
_l( 'a list of paired datasets' ),
'<% } else if( collection.collection_type === "list:list" ){ %>',
_l( 'a list of dataset lists' ),
'<% } %>',
'</div>',
'</div>',
@@ -215,11 +217,28 @@ var ListOfPairsCollectionView = CollectionView.extend(
});
// =============================================================================
/** @class non-editable, read-only View/Controller for a list of lists dataset collection. */
var ListOfListsCollectionView = CollectionView.extend({
/** sub view class used for nested collections */
NestedDCDCEViewClass : DC_LI.NestedDCDCEListItemView.extend({
foldoutPanelClass : PairCollectionView
}),
/** string rep */
toString : function(){
return 'ListOfListsCollectionView(' + (( this.model )?( this.model.get( 'name' )):( '' )) + ')';
}
});
//==============================================================================
return {
CollectionView : CollectionView,
ListCollectionView : ListCollectionView,
PairCollectionView : PairCollectionView,
ListOfPairsCollectionView : ListOfPairsCollectionView
ListOfPairsCollectionView : ListOfPairsCollectionView,
ListOfListsCollectionView : ListOfListsCollectionView
};
});
client/galaxy/scripts/mvc/dataset/data.js
+6 -3
View File
@@ -64,7 +64,7 @@ var TabularDataset = Dataset.extend({
defaults: _.extend({}, Dataset.prototype.defaults, {
chunk_url: null,
first_data_chunk: null,
chunk_index: -1,
offset: 0,
at_eof: false
}),
@@ -72,7 +72,9 @@ var TabularDataset = Dataset.extend({
Dataset.prototype.initialize.call(this);
// If first data chunk is available, next chunk is 1.
this.attributes.chunk_index = (this.attributes.first_data_chunk ? 1 : 0);
if (this.attributes.first_data_chunk){
this.attributes.offset = this.attributes.first_data_chunk.offset;
}
this.attributes.chunk_url = Galaxy.root + 'dataset/display?dataset_id=' + this.id;
this.attributes.url_viz = Galaxy.root + 'visualization';
},
@@ -90,12 +92,13 @@ var TabularDataset = Dataset.extend({
var self = this,
next_chunk = $.Deferred();
$.getJSON(this.attributes.chunk_url, {
chunk: self.attributes.chunk_index++
offset: self.attributes.offset
}).success(function(chunk) {
var rval;
if (chunk.ck_data !== '') {
// Found chunk.
rval = chunk;
self.attributes.offset = chunk.offset;
}
else {
// At EOF.
client/galaxy/scripts/mvc/form/form-data.js
+10 -4
View File
@@ -7,8 +7,7 @@ define([ 'utils/utils' ], function( Utils ) {
this.app = app;
},
/** Creates a checksum.
*/
/** Creates a checksum. */
checksum: function() {
var sum = '';
var self = this;
@@ -22,8 +21,15 @@ define([ 'utils/utils' ], function( Utils ) {
return sum;
},
/** Convert dom into a dictionary of flat id/value pairs used e.g. on job submission.
*/
/** Set parameter values from model */
set: function( model ) {
for ( var attr in model.attributes ) {
var index = this.match( attr );
index && this.app.field_list[ index ].value( model.get( attr ) );
}
},
/** Convert dom into a dictionary of flat id/value pairs used e.g. on job submission. */
create: function() {
var self = this;
client/galaxy/scripts/mvc/form/form-parameters.js
+33 -68
View File
@@ -31,17 +31,11 @@ define(['utils/utils',
'ftpfile' : '_fieldFtp'
},
initialize: function( app, options ) {
this.app = app;
},
/** Returns an input field for a given field type
*/
/** Returns an input field for a given field type */
create: function( input_def ) {
var fieldClass = this.types[ input_def.type ];
var field = typeof( this[ fieldClass ] ) === 'function' ? this[ fieldClass ].call( this, input_def ) : null;
if ( !field ) {
this.app.incompatible = true;
field = input_def.options ? this._fieldSelect( input_def ) : this._fieldText( input_def );
Galaxy.emit.debug('form-parameters::_addRow()', 'Auto matched field type (' + input_def.type + ').');
}
@@ -50,10 +44,8 @@ define(['utils/utils',
return field;
},
/** Data input field
*/
/** Data input field */
_fieldData: function( input_def ) {
var self = this;
return new SelectContent.View({
id : 'field-' + input_def.id,
extensions : input_def.extensions,
@@ -62,14 +54,11 @@ define(['utils/utils',
type : input_def.type,
flavor : input_def.flavor,
data : input_def.options,
onchange : function() {
self.app.trigger( 'change' );
}
onchange : input_def.onchange
});
},
/** Select/Checkbox/Radio options field
*/
/** Select/Checkbox/Radio options field */
_fieldSelect: function ( input_def ) {
// show text field e.g. in workflow editor
if( input_def.is_workflow ) {
@@ -81,11 +70,14 @@ define(['utils/utils',
input_def.error_text = 'Missing columns in referenced dataset.'
}
// configure options fields
var options = [];
_.each( input_def.options, function( option ) {
options.push( { label: option[ 0 ], value: option[ 1 ] } );
});
// identify available options
var data = input_def.data;
if( !data ) {
data = [];
_.each( input_def.options, function( option ) {
data.push( { label: option[ 0 ], value: option[ 1 ] } );
});
}
// identify display type
var SelectClass = Ui.Select;
@@ -96,24 +88,23 @@ define(['utils/utils',
case 'radio':
SelectClass = Ui.Radio;
break;
case 'radiobutton':
SelectClass = Ui.RadioButton;
break;
}
// create select field
var self = this;
return new SelectClass.View({
id : 'field-' + input_def.id,
data : options,
data : data,
error_text : input_def.error_text || 'No options available',
multiple : input_def.multiple,
optional : input_def.optional,
onchange : function() {
self.app.trigger( 'change' );
}
onchange : input_def.onchange
});
},
/** Drill down options field
*/
/** Drill down options field */
_fieldDrilldown: function ( input_def ) {
// show text field e.g. in workflow editor
if( input_def.is_workflow ) {
@@ -121,23 +112,19 @@ define(['utils/utils',
}
// create drill down field
var self = this;
return new Ui.Drilldown.View({
id : 'field-' + input_def.id,
data : input_def.options,
display : input_def.display,
optional : input_def.optional,
onchange : function() {
self.app.trigger( 'change' );
}
onchange : input_def.onchange
});
},
/** Text input field
*/
/** Text input field */
_fieldText: function( input_def ) {
// field replaces e.g. a select field
if ( input_def.options ) {
if ( input_def.options && input_def.data ) {
input_def.area = input_def.multiple;
if ( Utils.isEmpty( input_def.value ) ) {
input_def.value = null;
@@ -156,34 +143,27 @@ define(['utils/utils',
}
}
// create input element
var self = this;
return new Ui.Input({
id : 'field-' + input_def.id,
area : input_def.area,
onchange : function( new_value ) {
input_def.onchange ? input_def.onchange( new_value ) : self.app.trigger( 'change' );
}
placeholder : input_def.placeholder,
onchange : input_def.onchange
});
},
/** Slider field
*/
/** Slider field */
_fieldSlider: function( input_def ) {
var self = this;
return new Ui.Slider.View({
id : 'field-' + input_def.id,
precise : input_def.type == 'float',
is_workflow : input_def.is_workflow,
min : input_def.min,
max : input_def.max,
onchange : function() {
self.app.trigger( 'change' );
}
onchange : input_def.onchange
});
},
/** Hidden field
*/
/** Hidden field */
_fieldHidden: function( input_def ) {
return new Ui.Hidden({
id : 'field-' + input_def.id,
@@ -191,57 +171,42 @@ define(['utils/utils',
});
},
/** Boolean field
*/
/** Boolean field */
_fieldBoolean: function( input_def ) {
var self = this;
return new Ui.RadioButton.View({
id : 'field-' + input_def.id,
data : [ { label : 'Yes', value : 'true' },
{ label : 'No', value : 'false' }],
onchange : function() {
self.app.trigger( 'change' );
}
onchange : input_def.onchange
});
},
/** Color picker field
*/
/** Color picker field */
_fieldColor: function( input_def ) {
var self = this;
return new ColorPicker({
id : 'field-' + input_def.id,
onchange : function() {
self.app.trigger( 'change' );
}
onchange : input_def.onchange
});
},
/** Library dataset field
*/
/** Library dataset field */
_fieldLibrary: function( input_def ) {
var self = this;
return new SelectLibrary.View({
id : 'field-' + input_def.id,
optional : input_def.optional,
multiple : input_def.multiple,
onchange : function() {
self.app.trigger( 'change' );
}
onchange : input_def.onchange
});
},
/** FTP file field
*/
_fieldFtp: function( input_def ) {
var self = this;
return new SelectFtp.View({
id : 'field-' + input_def.id,
optional : input_def.optional,
multiple : input_def.multiple,
onchange : function() {
self.app.trigger( 'change' );
}
onchange : input_def.onchange
});
}
});
client/galaxy/scripts/mvc/form/form-repeat.js
+14 -15
View File
@@ -1,9 +1,6 @@
// dependencies
/** This class creates a ui component which enables the dynamic creation of portlets */
define(['utils/utils', 'mvc/ui/ui-table', 'mvc/ui/ui-portlet', 'mvc/ui/ui-misc'],
function(Utils, Table, Portlet, Ui) {
/** This class creates a ui component which enables the dynamic creation of portlets
*/
var View = Backbone.View.extend({
initialize : function(options) {
var self = this;
@@ -41,14 +38,12 @@ var View = Backbone.View.extend({
this.n = 0;
},
/** Number of repeat blocks
*/
/** Number of repeat blocks */
size: function() {
return this.n;
},
/** Add new repeat block
*/
/** Add new repeat block */
add: function(options) {
if (!options.id || this.list[options.id]) {
Galaxy.emit.debug('form-repeat::add()', 'Duplicate repeat block id.');
@@ -68,7 +63,7 @@ var View = Backbone.View.extend({
var portlet = new Portlet.View({
id : options.id,
title : 'placeholder',
cls : 'ui-portlet-repeat',
cls : options.cls || 'ui-portlet-repeat',
operations : {
button_delete : button_delete
}
@@ -84,8 +79,7 @@ var View = Backbone.View.extend({
this._refresh();
},
/** Delete repeat block
*/
/** Delete repeat block */
del: function(id) {
if (!this.list[id]) {
Galaxy.emit.debug('form-repeat::del()', 'Invalid repeat block id.');
@@ -99,8 +93,14 @@ var View = Backbone.View.extend({
this._refresh();
},
/** Hides add/del options
*/
/** Remove all */
delAll: function() {
for( var id in this.list ) {
this.del( id );
}
},
/** Hides add/del options */
hideOptions: function() {
this.button_new.$el.hide();
_.each( this.list, function( portlet ) {
@@ -111,8 +111,7 @@ var View = Backbone.View.extend({
}
},
/** Refresh view
*/
/** Refresh view */
_refresh: function() {
var index = 0;
for (var id in this.list) {
client/galaxy/scripts/mvc/form/form-section.js
+3 -1
View File
@@ -23,7 +23,7 @@ define(['utils/utils',
// create/render views
this.table = new Table.View(options);
this.parameters = new Parameters(app, options);
this.parameters = new Parameters();
this.setElement(this.table.$el);
this.render();
},
@@ -187,7 +187,9 @@ define(['utils/utils',
/** Add a single input field element
*/
_addRow: function(input_def) {
var self = this;
var id = input_def.id;
input_def.onchange = function() { self.app.trigger( 'change' ) };
var field = this.parameters.create(input_def);
this.app.field_list[id] = field;
var input_element = new InputElement(this.app, {
client/galaxy/scripts/mvc/form/form-view.js
+8 -6
View File
@@ -86,8 +86,7 @@ define(['utils/utils', 'mvc/ui/ui-portlet', 'mvc/ui/ui-misc', 'mvc/form/form-sec
}
},
/** Highlights errors
*/
/** Highlights errors */
errors: function(options) {
this.trigger('reset');
if (options && options.errors) {
@@ -101,8 +100,12 @@ define(['utils/utils', 'mvc/ui/ui-portlet', 'mvc/ui/ui-misc', 'mvc/form/form-sec
}
},
/** Render tool form
*/
/** Modify onchange event handler */
setOnChange: function( callback ) {
this.options.onchange = callback;
},
/** Render tool form */
render: function() {
// link this
var self = this;
@@ -153,8 +156,7 @@ define(['utils/utils', 'mvc/ui/ui-portlet', 'mvc/ui/ui-misc', 'mvc/form/form-sec
return this;
},
/** Renders the UI elements required for the form
*/
/** Renders the UI elements required for the form */
_renderForm: function() {
// create message view
this.message = new Ui.Message();
client/galaxy/scripts/mvc/history/hdca-li-edit.js
+2
View File
@@ -26,6 +26,8 @@ var HDCAListItemEdit = _super.extend(
return DC_VIEW_EDIT.PairCollectionViewEdit;
case 'list:paired':
return DC_VIEW_EDIT.ListOfPairsCollectionViewEdit;
case 'list:list':
return DC_VIEW_EDIT.ListOfListsCollectionViewEdit;
}
throw new TypeError( 'Uknown collection_type: ' + this.model.get( 'collection_type' ) );
},
client/galaxy/scripts/mvc/history/hdca-li.js
+2
View File
@@ -38,6 +38,8 @@ var HDCAListItemView = _super.extend(
return DC_VIEW.PairCollectionView;
case 'list:paired':
return DC_VIEW.ListOfPairsCollectionView;
case 'list:list':
return DC_VIEW.ListOfListsCollectionView;
}
throw new TypeError( 'Uknown collection_type: ' + this.model.get( 'collection_type' ) );
},
client/galaxy/scripts/mvc/history/hdca-model.js
+31 -7
View File
@@ -18,7 +18,8 @@ TODO:
var hcontentMixin = HISTORY_CONTENT.HistoryContentMixin,
ListDC = DC_MODEL.ListDatasetCollection,
PairDC = DC_MODEL.PairDatasetCollection,
ListPairedDC = DC_MODEL.ListPairedDatasetCollection;
ListPairedDC = DC_MODEL.ListPairedDatasetCollection,
ListOfListsDC = DC_MODEL.ListOfListsDatasetCollection;
//==============================================================================
/** Override to post to contents route w/o id. */
@@ -91,11 +92,8 @@ var HistoryPairDatasetCollection = PairDC.extend( hcontentMixin ).extend(
//==============================================================================
/** @class Backbone model for List of Pairs Dataset Collection within a History.
* @constructs
*/
var HistoryListPairedDatasetCollection = ListPairedDC.extend( hcontentMixin ).extend(
/** @lends HistoryListPairedDatasetCollection.prototype */{
/** @class Backbone model for List of Pairs Dataset Collection within a History. */
var HistoryListPairedDatasetCollection = ListPairedDC.extend( hcontentMixin ).extend({
defaults : _.extend( _.clone( ListPairedDC.prototype.defaults ), {
history_content_type: 'dataset_collection',
@@ -118,10 +116,36 @@ var HistoryListPairedDatasetCollection = ListPairedDC.extend( hcontentMixin ).ex
});
//==============================================================================
/** @class Backbone model for List of Lists Dataset Collection within a History. */
var HistoryListOfListsDatasetCollection = ListOfListsDC.extend( hcontentMixin ).extend({
defaults : _.extend( _.clone( ListOfListsDC.prototype.defaults ), {
history_content_type: 'dataset_collection',
collection_type : 'list:list',
model_class : 'HistoryDatasetCollectionAssociation'
}),
initialize : function( model, options ){
ListOfListsDC.prototype.initialize.call( this, model, options );
hcontentMixin.initialize.call( this, model, options );
},
/** Override to post to contents route w/o id. */
save : buildHDCASave( ListOfListsDC.prototype.save ),
/** String representation. */
toString : function(){
return ([ 'HistoryListOfListsDatasetCollection(', this.get( 'name' ), ')' ].join( '' ));
}
});
//==============================================================================
return {
HistoryListDatasetCollection : HistoryListDatasetCollection,
HistoryPairDatasetCollection : HistoryPairDatasetCollection,
HistoryListPairedDatasetCollection : HistoryListPairedDatasetCollection
HistoryListPairedDatasetCollection : HistoryListPairedDatasetCollection,
HistoryListOfListsDatasetCollection : HistoryListOfListsDatasetCollection
};
});
client/galaxy/scripts/mvc/history/history-contents.js
+13 -5
View File
@@ -44,12 +44,16 @@ var HistoryContents = Backbone.Collection
return new HDCA_MODEL.HistoryPairDatasetCollection( attrs, options );
case 'list:paired':
return new HDCA_MODEL.HistoryListPairedDatasetCollection( attrs, options );
case 'list:list':
return new HDCA_MODEL.HistoryListOfListsDatasetCollection( attrs, options );
}
// This is a hack inside a hack:
// Raise a plain object with validationError to fake a model.validationError
// (since we don't have a model to use validate with)
// (the outer hack being the mixed content/model function in this collection)
return { validationError : 'Unknown collection_type: ' + attrs.history_content_type };
var msg = 'Unknown collection_type: ' + attrs.collection_type;
console.warn( msg, attrs );
return { validationError : msg };
}
return { validationError : 'Unknown history_content_type: ' + attrs.history_content_type };
},
@@ -281,10 +285,14 @@ var HistoryContents = Backbone.Collection
type = ( contentType === 'hdca'? 'dataset_collection' : 'dataset' );
}
var collection = this,
xhr = jQuery.post( this.url(), {
content : id,
source : contentType,
type : type
xhr = jQuery.ajax( this.url(), {
method: 'POST',
contentType: 'application/json',
data: JSON.stringify({
content : id,
source : contentType,
type : type
})
})
.done( function( response ){
collection.add([ response ]);
client/galaxy/scripts/mvc/tool/tool-form-composite.js
+8 -8
View File
@@ -1,8 +1,9 @@
/** This is the run workflow tool form view. */
define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view', 'mvc/form/form-data', 'mvc/tool/tool-form-base' ],
function( Utils, Deferred, Ui, Form, FormData, ToolFormBase ) {
define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view', 'mvc/form/form-data', 'mvc/tool/tool-form-base', 'mvc/ui/ui-modal' ],
function( Utils, Deferred, Ui, Form, FormData, ToolFormBase, Modal ) {
var View = Backbone.View.extend({
initialize: function( options ) {
this.modal = parent.Galaxy.modal || new Modal.View();
this.model = options && options.model || new Backbone.Model( options );
this.deferred = new Deferred();
this.setElement( $( '<div/>' ).addClass( 'ui-form-composite' )
@@ -330,7 +331,7 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
var self = this;
var job_def = {
new_history_name : this.history_form.data.create()[ 'new_history|name' ],
wf_parm : this.wp_form ? this.wp_form.data.create() : {},
replacement_params : this.wp_form ? this.wp_form.data.create() : {},
inputs : {}
};
var validated = true;
@@ -371,7 +372,7 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
Galaxy.emit.debug( 'tool-form-composite::submit()', 'Validation complete.', job_def );
Utils.request({
type : 'POST',
url : Galaxy.root + 'api/workflows/' + this.workflow_id + '/invocations',
url : Galaxy.root + 'api_internal/workflows/' + this.model.id + '/run',
data : job_def,
success : function( response ) {
Galaxy.emit.debug( 'tool-form-composite::submit', 'Submission successful.', response );
@@ -393,13 +394,12 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
}
}
} else {
var modal = parent.Galaxy.modal;
modal && modal.show({
self.modal.show({
title : 'Job submission failed',
body : self._templateError( response && response.err_msg || job_def ),
buttons : {
'Close' : function() {
modal.hide();
self.modal.hide();
}
}
});
@@ -477,4 +477,4 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
return {
View: View
};
});
});
client/galaxy/scripts/mvc/ui/ui-misc.js
+1 -1
View File
@@ -67,7 +67,7 @@ define(['utils/utils',
if ( !this.model.get( 'persistent' ) ) {
var self = this;
this.timeout = window.setTimeout( function() {
self.$el.fadeOut();
self.model.set( 'message', '' );
}, 3000 );
}
} else {
client/galaxy/scripts/mvc/ui/ui-select-default.js
+1 -1
View File
@@ -18,7 +18,7 @@ var View = Backbone.View.extend({
disabled : false,
onchange : function(){}
}).set( options );
this.on( 'change', function() { self.model.get( 'onchange' )( self.value() ) } );
this.on( 'change', function() { self.model.get( 'onchange' ) && self.model.get( 'onchange' )( self.value() ) } );
this.listenTo( this.model, 'change:data', this._changeData, this );
this.listenTo( this.model, 'change:disabled', this._changeDisabled, this );
this.listenTo( this.model, 'change:wait', this._changeWait, this );
client/galaxy/scripts/mvc/ui/ui-tabs.js
+157 -247
View File
@@ -1,288 +1,198 @@
// dependencies
define(['utils/utils'], function(Utils) {
// return
/**
* Renders tabs e.g. used in the Charts editor, behaves similar to repeat and section rendering
*/
define( [ 'utils/utils' ], function( Utils ) {
var View = Backbone.View.extend({
// defaults options
optionsDefault: {
title_new : '',
operations : null,
onnew : null,
max : null,
onchange : null
},
// initialize
initialize : function(options) {
// configure
initialize : function( options ) {
var self = this;
this.visible = false;
this.$nav = null;
this.$content = null;
this.first_tab = null;
this.current_id = null;
// configure options
this.options = Utils.merge(options, this.optionsDefault);
// create tabs
var $tabs = $(this._template(this.options));
// link elements
this.$nav = $tabs.find('.tab-navigation');
this.$content = $tabs.find('.tab-content');
// create new element
this.setElement($tabs);
// clear list
this.list = {};
// link this
var self = this;
// append operations
if (this.options.operations) {
$.each(this.options.operations, function(name, item) {
item.$el.prop('id', name);
self.$nav.find('.operations').append(item.$el);
});
}
// add built-in add-new-tab tab
if (this.options.onnew) {
// create tab object
var $tab_new = $(this._template_tab_new(this.options));
// append to navbar
this.$nav.append($tab_new);
// add tooltip
$tab_new.tooltip({title: 'Add a new tab', placement: 'bottom', container: self.$el});
// link click event
$tab_new.on('click', function(e) {
$tab_new.tooltip('hide');
self.options.onnew();
});
}
},
// size
size: function() {
return _.size(this.list);
},
// front
current: function() {
return this.$el.find('.tab-pane.active').attr('id');
},
// append
add: function(options) {
// self
var self = this;
// get tab id
var id = options.id;
this.list = {};
this.options = Utils.merge( options, {
title_new : '',
operations : null,
onnew : null,
max : null,
onchange : null
});
this.setElement( $( this._template( this.options ) ) );
this.$nav = this.$( '.tab-navigation' );
this.$content = this.$( '.tab-content' );
this.$operations = this.$nav.find( '.tab-operations' );
// create tab object
var $tab_title = $(this._template_tab(options));
var $tab_content = $(this._template_tab_content(options));
// add to list
this.list[id] = options.ondel ? true : false;
// add a new tab either before the add-new-tab tab or behind the last tab
if (this.options.onnew) {
this.$nav.find('#new-tab').before($tab_title);
} else {
this.$nav.append($tab_title);
// Renders tab operations
if ( this.options.operations ) {
$.each( this.options.operations, function( name, item ) {
item.$el.prop( 'id', name );
self.$operations.append( item.$el );
});
}
// add content
$tab_content.append(options.$el);
this.$content.append($tab_content);
// activate this tab if this is the first tab
if (this.size() == 1) {
$tab_title.addClass('active');
$tab_content.addClass('active');
// Allows user to add new tabs
this.options.onnew && this.$nav.append( $( this._template_tab_new( this.options ) )
.tooltip( { title: 'Add a new tab', placement: 'bottom', container: self.$el } )
.on( 'click', function( e ) { self.options.onnew() } )
);
this.$tabnew = this.$nav.find( '.tab-new' );
// Remove all tooltips on click
this.$el.on( 'click', function() { $( '.tooltip' ).hide() } );
},
/** Returns current number of tabs */
size: function() {
return _.size( this.list );
},
/** Returns tab id for currently shown tab */
current: function() {
return this.$el.find( '.tab-pane.active' ).attr( 'id' );
},
/** Adds a new tab */
add: function( options ) {
var self = this;
var id = options.id;
var $tab_title = $( this._template_tab( options ) );
var $tab_content = $( '<div/>' ).attr( 'id', options.id ).addClass( 'tab-pane' );
// hide new tab if maximum number of tabs has been reached
this.list[ id ] = true;
if ( this.options.max && this.size() >= this.options.max ) {
this.$tabnew.hide();
}
// insert tab before new tab or as last tab
if ( this.options.onnew ) {
this.$tabnew.before( $tab_title );
} else {
this.$nav.append( $tab_title );
}
// assing delete callback if provided
if ( options.ondel ) {
$tab_title.find( '.tab-delete' ).tooltip( { title: 'Delete this tab', placement: 'bottom', container: self.$el } )
.on( 'click', function() { options.ondel() } );
} else {
$tab_title.tooltip( { title: options.tooltip, placement: 'bottom', container: self.$el } );
}
$tab_title.on( 'click', function( e ) {
e.preventDefault();
options.onclick ? options.onclick() : self.show( id );
});
this.$content.append( $tab_content.append( options.$el ) );
// assign current/first tab
if ( this.size() == 1 ) {
$tab_title.addClass( 'active' );
$tab_content.addClass( 'active' );
this.first_tab = id;
}
// hide add tab
if (this.options.max && this.size() >= this.options.max) {
this.$el.find('#new-tab').hide();
}
// add click event to remove tab
if (options.ondel) {
var $del_icon = $tab_title.find('#delete');
$del_icon.tooltip({title: 'Delete this tab', placement: 'bottom', container: self.$el});
$del_icon.on('click', function() {
$del_icon.tooltip('destroy');
self.$el.find('.tooltip').remove();
options.ondel();
return false;
});
}
// add custom click event handler
$tab_title.on('click', function(e) {
// prevent default
e.preventDefault();
// click
if (options.onclick) {
options.onclick();
} else {
self.show(id);
}
});
// initialize current id
if (!this.current_id) {
if ( !this.current_id ) {
this.current_id = id;
}
},
// delete tab
del: function(id) {
// delete tab from dom
this.$el.find('#tab-' + id).remove();
this.$el.find('#' + id).remove();
// check if first tab has been deleted
if (this.first_tab == id) {
this.first_tab = null;
}
// show first tab
if (this.first_tab != null) {
this.show(this.first_tab);
}
// delete from list
if (this.list[id]) {
delete this.list[id];
}
// show add tab
if (this.size() < this.options.max) {
this.$el.find('#new-tab').show();
/** Delete tab */
del: function( id ) {
this.$( '#tab-' + id ).remove();
this.$( '#' + id ).remove();
this.first_tab = this.first_tab == id ? null : this.first_tab;
this.first_tab != null && this.show( this.first_tab );
this.list[ id ] && delete this.list[ id ];
if ( this.size() < this.options.max ) {
this.$el.find( '.ui-tabs-new' ).show();
}
},
// delete tab
/** Delete all tabs */
delRemovable: function() {
for (var id in this.list) {
this.del(id);
for ( var id in this.list ) {
this.del( id );
}
},
// show
show: function(id){
// show tab view
this.$el.fadeIn('fast');
/** Show tab view and highlight a tab by id */
show: function( id ){
this.$el.fadeIn( 'fast' );
this.visible = true;
// show selected tab
if (id) {
// reassign active class
this.$el.find('#tab-' + this.current_id).removeClass('active');
this.$el.find('#' + this.current_id).removeClass('active');
this.$el.find('#tab-' + id).addClass('active');
this.$el.find('#' + id).addClass('active');
// update current id
if ( id ) {
this.$( '#tab-' + this.current_id ).removeClass('active' );
this.$( '#' + this.current_id ).removeClass('active' );
this.$( '#tab-' + id ).addClass( 'active' );
this.$( '#' + id ).addClass( 'active' );
this.current_id = id;
}
// change
if (this.options.onchange) {
this.options.onchange(id);
}
this.options.onchange && this.options.onchange( id );
},
// hide
/** Hide tab view */
hide: function(){
this.$el.fadeOut('fast');
this.$el.fadeOut( 'fast' );
this.visible = false;
},
// hide operation
hideOperation: function(id) {
this.$nav.find('#' + id).hide();
/** Hide operation by id */
hideOperation: function( id ) {
this.$nav.find( '#' + id ).hide();
},
// show operation
showOperation: function(id) {
this.$nav.find('#' + id).show();
/** Show operation by id */
showOperation: function( id ) {
this.$nav.find( '#' + id ).show();
},
// set operation
setOperation: function(id, callback) {
var $el = this.$nav.find('#' + id);
$el.off('click');
$el.on('click', callback);
/** Reassign an operation to a new callback */
setOperation: function( id, callback ) {
this.$nav.find( '#' + id ).off('click').on( 'click', callback );
},
// title
title: function(id, new_title) {
var $el = this.$el.find('#tab-title-text-' + id);
if (new_title) {
$el.html(new_title);
}
/** Set/Get title */
title: function( id, new_title ) {
var $el = this.$( '#tab-title-text-' + id );
new_title && $el.html( new_title );
return $el.html();
},
// retitle
retitle: function(new_title) {
/** Enumerate titles */
retitle: function( new_title ) {
var index = 0;
for (var id in this.list) {
this.title(id, ++index + ': ' + new_title);
for ( var id in this.list ) {
this.title( id, ++index + ': ' + new_title );
}
},
// fill template
_template: function(options) {
return '<div class="ui-tabs tabbable tabs-left">' +
'<ul id="tab-navigation" class="tab-navigation nav nav-tabs">' +
'<div class="operations" style="float: right; margin-bottom: 4px;"></div>' +
'</ul>'+
'<div id="tab-content" class="tab-content"/>' +
'</div>';
/** Main template */
_template: function( options ) {
return $( '<div/>' ).addClass( 'ui-tabs tabbable tabs-left' )
.append( $( '<ul/>' ).addClass( 'tab-navigation nav nav-tabs' )
.append( $( '<div/>' ).addClass( 'tab-operations' ) ) )
.append( $( '<div/>' ).addClass( 'tab-content' ) );
},
// fill template tab
_template_tab_new: function(options) {
return '<li id="new-tab">' +
'<a href="javascript:void(0);">' +
'<i class="ui-tabs-add fa fa-plus-circle"/>' +
options.title_new +
'</a>' +
'</li>';
/** Default tab templates */
_template_tab: function( options ) {
var $tmpl = $( '<li/>' ).addClass( 'tab-element' )
.attr( 'id', 'tab-' + options.id )
.append( $( '<a/>' ).attr( 'id', 'tab-title-link-' + options.id ) );
var $href = $tmpl.find( 'a' );
options.icon && $href.append( $( '<i/>' ).addClass( 'tab-icon fa' ).addClass( options.icon ) );
$href.append( $( '<span/>' ).attr( 'id', 'tab-title-text-' + options.id )
.addClass( 'tab-title-text' )
.append( options.title ) );
options.ondel && $href.append( $( '<i/>' ).addClass( 'tab-delete fa fa-minus-circle' ) );
return $tmpl;
},
// fill template tab
_template_tab: function(options) {
var tmpl = '<li id="tab-' + options.id + '" class="tab-element">' +
'<a id="tab-title-link-' + options.id + '" title="" href="#' + options.id + '" data-original-title="">' +
'<span id="tab-title-text-' + options.id + '" class="tab-title-text">' + options.title + '</span>';
if (options.ondel) {
tmpl += '<i id="delete" class="ui-tabs-delete fa fa-minus-circle"/>';
}
tmpl += '</a>' +
'</li>';
return tmpl;
},
// fill template tab content
_template_tab_content: function(options) {
return '<div id="' + options.id + '" class="tab-pane"/>';
/** Template for 'new' tab */
_template_tab_new: function( options ) {
return $( '<li/>' ).addClass( 'tab-new' )
.append( $( '<a/>' ).attr( 'href', 'javascript:void(0);' )
.append( $( '<i/>' ).addClass( 'tab-icon fa fa-plus-circle' ) )
.append( options.title_new ) );
}
});
client/galaxy/style/less/ui.less
+30 -9
View File
@@ -164,15 +164,24 @@
// tabs
.ui-tabs {
.ui-tabs-add {
font-size : 0.8em;
margin-right : @ui-margin-horizontal;
.tab-operations {
float: right;
margin-bottom: 4px;
.ui-button-icon {
margin-left: 4px;
}
}
.ui-tabs-delete {
font-size : 0.8em;
margin-left : @ui-margin-horizontal;
cursor : pointer;
.tab-element {
cursor: pointer;
}
.tab-icon {
font-size: 1.1em;
margin-right: @ui-margin-horizontal;
}
.tab-delete {
font-size: 0.8em;
margin-left: @ui-margin-horizontal;
cursor: pointer;
}
}
@@ -433,7 +442,7 @@
}
}
.ui-portlet-section {
.ui-portlet-panel {
&:extend(.ui-portlet-repeat all);
margin-top: @ui-margin-vertical;
border-left: solid 3px @side-panel-bg;
@@ -441,6 +450,10 @@
background: @side-panel-bg;
border-bottom: solid darken(@side-panel-bg, 10%) 1px;
}
}
.ui-portlet-section {
&:extend(.ui-portlet-panel all);
.portlet-highlight {
text-decoration: underline;
}
@@ -477,6 +490,14 @@
max-width: 900px;
}
.ui-portlet-plain {
&:extend(.ui-portlet all);
border: none;
.portlet-content {
padding: 0px;
}
}
// popovers
.ui-popover {
max-width: 700px;
config/datatypes_conf.xml.sample
+1
View File
@@ -147,6 +147,7 @@
<datatype extension="pgm" type="galaxy.datatypes.images:Pgm" mimetype="image/pgm"/>
<datatype extension="rna_eps" type="galaxy.datatypes.sequence:RNADotPlotMatrix" mimetype="image/eps" display_in_upload="True"/>
<datatype extension="zip" type="galaxy.datatypes.binary:CompressedZipArchive" display_in_upload="True"/>
<datatype extension="tar" type="galaxy.datatypes.binary:CompressedArchive" subclass="True" display_in_upload="True"/>
<!-- Proteomics Datatypes -->
<datatype extension="pepxml" type="galaxy.datatypes.proteomics:PepXml" mimetype="application/xml" display_in_upload="True"/>
<datatype extension="raw_pepxml" type="galaxy.datatypes.proteomics:PepXml" mimetype="application/xml" subclass="True"/>
config/galaxy.ini.sample
+6 -2
View File
@@ -418,7 +418,11 @@ paste.app_factory = galaxy.web.buildapp:app_factory
# In use only if activation_grace_period is set.
#inactivity_box_content = Your account has not been activated yet. Feel free to browse around and see what's available, but you won't be able to upload data or run jobs until you have verified your email address.
# Password expiration period (in days). Users are required to change their
# password every x days. Users will be redirected to the change password
# screen when they log in after their password expires. Enter 0 to disable
# password expiration.
#password_expiration_period = 0
# Galaxy Session Timeout
# This provides a timeout (in minutes) after which a user will have to log back in.
@@ -922,7 +926,7 @@ use_interactive = True
# Allow users to remove their datasets from disk immediately (otherwise,
# datasets will be removed after a time period specified by an administrator in
# the cleanup scripts run via cron)
#allow_user_dataset_purge = False
#allow_user_dataset_purge = True
# By default, users' data will be public, but setting this to True will cause
# it to be private. Does not affect existing users and data, only ones created
config/job_conf.xml.sample_advanced
+19 -3
View File
@@ -174,9 +174,14 @@
-->
<param id="k8s_config_path">/path/to/kubeconfig</param>
<!--- This is the path to the kube config file, which is normally on ~/.kube/config, but that will depend on
<!-- This is the path to the kube config file, which is normally on ~/.kube/config, but that will depend on
your installation. This is the file that tells the plugin where the k8s cluster is, access credentials,
etc. -->
etc. This parameter is not necessary and ignored if k8s_use_service_account is set to True -->
<param id="k8s_use_service_account">false</param>
<!-- For use when Kubernetes should be accessed from inside a Pod running Galaxy (that is,
galaxy is running inside Kubernetes). If this variable is True, then the previous k8s_config_path is
not required and actually ignored. It is not necessary to set this value if not setting it to true -->
<param id="k8s_persistent_volume_claim_name">galaxy_pvc</param>
<!-- The name of the Persisten Volume Claim (PVC) to be used, details above, needs to match the PVC's
@@ -379,9 +384,20 @@
<!-- Make sure to setup 3 real user parameters in galaxy.ini. -->
</destination>
<destination id="dynamic" runner="dynamic">
<!-- A destination that represents a method in the dynamic runner. -->
<!-- A destination that represents a method in the dynamic runner.
foo should be a Python function defined in any file in
lib/galaxy/jobs/rules.
-->
<param id="function">foo</param>
</destination>
<destination id="dtd_destination" runner="dynamic">
<!-- DTD is a special dynamic job destination type that builds up
rules given a YAML-based DSL (see config/tool_destinations.yml.sample
for the syntax).
-->
<param id="type">dtd</param>
</destination>
<destination id="load_balance" runner="dynamic">
<param id="type">choose_one</param>
<!-- Randomly assign jobs to various static destination ids -->
config/plugins/interactive_environments/jupyter/config/jupyter.xml
+1
View File
@@ -6,6 +6,7 @@
<model_class>HistoryDatasetAssociation</model_class>
<test type="isinstance" test_attr="datatype" result_type="datatype">tabular.Tabular</test>
<test type="isinstance" test_attr="datatype" result_type="datatype">data.Text</test>
<test type="isinstance" test_attr="datatype" result_type="datatype">binary.Binary</test>
<to_param param_attr="id">dataset_id</to_param>
</data_source>
</data_sources>
config/plugins/visualizations/charts/static/app.css
+1 -1
View File
@@ -58,7 +58,7 @@
.charts-viewport .info {
position: absolute;
margin: 20px 20px 50px 10px;
margin: 40px 20px 50px 10px;
}
.charts-viewport .text {
config/plugins/visualizations/charts/static/app.js
+66 -100
View File
@@ -1,107 +1,73 @@
// dependencies
define(['mvc/ui/ui-modal', 'mvc/ui/ui-portlet', 'mvc/ui/ui-misc', 'utils/utils',
'plugin/library/jobs', 'plugin/library/datasets', 'plugin/library/storage', 'utils/deferred',
'plugin/views/viewer', 'plugin/views/editor',
'plugin/models/config', 'plugin/models/chart',
'plugin/charts/types'],
function( Modal, Portlet, Ui, Utils, Jobs, Datasets, Storage, Deferred,
ViewerView, EditorView,
Config, Chart, Types
) {
/**
* Main application class.
*/
return Backbone.View.extend({
// initialize
initialize: function(options){
// deactivate all debugs outputs
//window.console.debug = function() {};
// link options
this.options = options;
define( [ 'mvc/ui/ui-modal', 'mvc/ui/ui-portlet', 'mvc/ui/ui-misc', 'utils/utils',
'plugin/library/jobs', 'plugin/library/datasets', 'plugin/library/storage', 'utils/deferred',
'plugin/views/viewer', 'plugin/views/editor',
'plugin/models/config', 'plugin/models/chart', 'plugin/charts/types' ],
function( Modal, Portlet, Ui, Utils, Jobs, Datasets, Storage, Deferred,
ViewerView, EditorView, Config, Chart, Types ) {
return Backbone.View.extend({
initialize: function(options){
this.options = options;
this.modal = parent.Galaxy && parent.Galaxy.modal || new Modal.View();
// link galaxy modal or create one
if (Galaxy && Galaxy.modal) {
this.modal = Galaxy.modal;
} else {
this.modal = new Modal.View();
}
//
// models
//
this.config = new Config();
this.types = new Types();
this.chart = new Chart();
//
// libraries
//
this.jobs = new Jobs(this);
this.datasets = new Datasets(this);
this.storage = new Storage(this);
this.deferred = new Deferred();
//
// views
//
this.viewer_view = new ViewerView(this);
this.editor_view = new EditorView(this);
// append views
this.$el.append(this.viewer_view.$el);
this.$el.append(this.editor_view.$el);
// pick start screen
if (!this.storage.load()) {
// show editor
this.go('editor');
} else {
// show viewport
this.go('viewer');
// draw chart
// models
this.config = new Config();
this.types = new Types();
this.chart = new Chart();
// libraries
this.jobs = new Jobs(this);
this.datasets = new Datasets( this );
this.storage = new Storage( this );
this.deferred = new Deferred();
// views
this.viewer_view = new ViewerView( this );
this.editor_view = new EditorView( this );
// append views
this.$el.append( this.viewer_view.$el );
this.$el.append( this.editor_view.$el );
// pick start screen
if ( !this.storage.load() ) {
this.go( 'editor' );
} else {
this.go( 'viewer' );
this.chart.trigger( 'redraw' );
}
},
/** Loads a view and makes sure that all others are hidden */
go: function( view_id ) {
$( '.tooltip' ).hide();
switch ( view_id ) {
case 'editor' :
this.editor_view.show();
this.viewer_view.hide();
break;
case 'viewer' :
this.editor_view.hide();
this.viewer_view.show();
break;
}
},
/** Returns root path */
chartPath: function( chart_type ) {
var path = chart_type.split( /_(.+)/ );
if ( path.length >= 2 ) {
return path[ 0 ] + '/' + path[ 1 ];
}
console.debug( 'FAILED app:chartPath() - Invalid format: ' + chart_type );
},
/** Message */
showModal: function( title, body ) {
var self = this;
this.deferred.execute(function() {
self.chart.trigger('redraw');
});
this.modal.show( { title: title, body: body, buttons: { 'Close': function() { self.modal.hide() } } } );
}
},
// loads a view and makes sure that all others are hidden
go: function(view_id) {
// hide all tooltips
$('.tooltip').hide();
// pick view
switch (view_id) {
case 'editor' :
this.editor_view.show();
this.viewer_view.hide();
break;
case 'viewer' :
this.editor_view.hide();
this.viewer_view.show();
break;
}
},
// get root path
chartPath: function(chart_type) {
// create path from id
var path = chart_type.split(/_(.+)/);
// check path
if (path.length >= 2) {
// return path
return path[0] + '/' + path[1];
} else {
// log status
console.debug('FAILED App:chartPath() - Invalid format: ' + chart_type);
}
return undefined;
}
});
});
});
config/plugins/visualizations/charts/static/charts/forms/default.js
+11 -3
View File
@@ -216,7 +216,8 @@ return {
show_legend : {
title : 'Show legend',
info : 'Would you like to add a legend?',
type : 'radiobutton',
type : 'select',
display : 'radiobutton',
init : 'true',
data : [
{
@@ -232,7 +233,8 @@ return {
use_panels : {
title : 'Use multi-panels',
info : 'Would you like to separate your data into individual panels?',
type : 'radiobutton',
type : 'select',
display : 'radiobutton',
init : 'false',
data : [
{
@@ -245,7 +247,13 @@ return {
}
]
}
}
},
series : [{
name : 'key',
label : 'Provide a label',
type : 'text',
placeholder : 'Data label'
}]
};
});
config/plugins/visualizations/charts/static/charts/jqplot/common/config.js
+4 -2
View File
@@ -15,7 +15,8 @@ return $.extend(true, {}, config_default, {
x_axis_grid : {
title : 'Axis grid',
info : 'Would you like to show grid lines for the X axis?',
type : 'radiobutton',
type : 'select',
display : 'radiobutton',
init : 'false',
data : [
{
@@ -31,7 +32,8 @@ return $.extend(true, {}, config_default, {
y_axis_grid : {
title : 'Axis grid',
info : 'Would you like to show grid lines for the Y axis?',
type : 'radiobutton',
type : 'select',
display : 'radiobutton',
init : 'true',
data : [
{
config/plugins/visualizations/charts/static/charts/nvd3/common/config.js
+26 -17
View File
@@ -1,18 +1,27 @@
define(['plugin/charts/forms/default', 'plugin/plugins/nvd3/nv.d3'], function(config_default) {
return $.extend(true, {}, config_default, {
title : '',
category : '',
library : 'NVD3',
tag : 'svg',
keywords : 'small',
columns : {
tooltip : {
title : 'Data point labels',
is_text : true,
is_numeric : true,
is_auto : true
}
}
});
define( [ 'plugin/charts/forms/default', 'plugin/plugins/nvd3/nv.d3' ], function( config_default ) {
return $.extend( true, {}, config_default, {
title : '',
category : '',
library : 'NVD3',
tag : 'svg',
keywords : 'small',
columns : {
tooltip : {
title : 'Data point labels',
is_text : true,
is_numeric : true,
is_auto : true
}
},
series : [{
name : 'key',
label : 'Provide a label',
type : 'text',
placeholder : 'Data label'
},{
name : 'color',
label : 'Pick a series color',
type : 'color'
}]
});
});
config/plugins/visualizations/charts/static/charts/nvd3/common/wrapper.js
+67 -130
View File
@@ -1,140 +1,77 @@
// dependencies
define(['plugin/charts/tools'], function(Tools) {
/**
* This is the common wrapper for nvd3 based visualizations.
*/
return Backbone.View.extend({
// initialize
initialize: function(app, options) {
// get parameters
this.options = options;
// define render function
var self = this;
options.render = function(canvas_id, groups) {
return self.render(canvas_id, groups)
};
define( [ 'plugin/charts/tools' ], function( Tools ) {
return Backbone.View.extend({
initialize: function( app, options ) {
var self = this;
this.options = options;
options.render = function( canvas_id, groups ) {
return self.render( canvas_id, groups )
};
Tools.panelHelper( app, options );
},
// call panel helper
Tools.panelHelper(app, options);
},
// render
render : function(canvas_id, groups) {
var chart = this.options.chart;
var type = this.options.type;
var makeConfig = this.options.makeConfig;
// create nvd3 model
var d3chart = nv.models[type]()
// request data
var self = this;
nv.addGraph(function() {
try {
// x axis label
d3chart.xAxis.axisLabel(chart.settings.get('x_axis_label'));
// y axis label
d3chart.yAxis.axisLabel(chart.settings.get('y_axis_label'))
.axisLabelDistance(30);
// controls
d3chart.options({showControls: false});
// legend
if (d3chart.showLegend) {
d3chart.showLegend(chart.settings.get('show_legend') == 'true');
}
// make categories
self._makeAxes(d3chart, groups, chart.settings);
// custom callback
if (makeConfig) {
makeConfig(d3chart);
}
// hide controls if in multi-viewer mode
if (chart.settings.get('use_panels') === 'true') {
d3chart.options({showControls: false});
}
// hide min/max values
d3chart.xAxis.showMaxMin(false);
d3chart.yAxis.showMaxMin(chart.definition.showmaxmin);
// configure tooltips
d3chart.tooltipContent(function(key, x, y, graph) {
return '<h3>' + (graph.point.tooltip || key) + '</h3>';
});
// draw chart
if ($('#' + canvas_id).length == 0) {
return;
}
var canvas = d3.select('#' + canvas_id);
canvas.datum(groups)
.call(d3chart);
// add zoom/pan handler
if (chart.definition.zoomable && chart.definition.zoomable != 'native') {
// clip edges
if (d3chart.clipEdge) {
d3chart.clipEdge(true);
}
// add zoom
Tools.addZoom({
xAxis : d3chart.xAxis,
yAxis : d3chart.yAxis,
yDomain: d3chart.yDomain,
xDomain: d3chart.xDomain,
redraw : function() { d3chart.update() },
svg : canvas
render: function( canvas_id, groups ) {
var self = this;
var chart = this.options.chart;
var type = this.options.type;
var makeConfig = this.options.makeConfig;
var d3chart = nv.models[ type ]();
nv.addGraph( function() {
try {
d3chart.xAxis.axisLabel( chart.settings.get( 'x_axis_label' ) );
d3chart.yAxis.axisLabel( chart.settings.get( 'y_axis_label' ) ).axisLabelDistance( 30 );
d3chart.options( { showControls: false } );
d3chart.showLegend && d3chart.showLegend(chart.settings.get('show_legend') == 'true');
self._makeAxes( d3chart, groups, chart.settings );
makeConfig && makeConfig( d3chart );
chart.settings.get( 'use_panels' ) === 'true' && d3chart.options( { showControls: false } );
d3chart.xAxis.showMaxMin( false );
d3chart.yAxis.showMaxMin( chart.definition.showmaxmin );
d3chart.tooltipContent( function( key, x, y, graph ) {
return '<h3>' + ( graph.point.tooltip || key ) + '</h3>';
});
}
// refresh on window resize
nv.utils.windowResize(d3chart.update);
} catch (err) {
self._handleError(chart, err);
}
});
// return
return true;
},
// create axes formatting
_makeAxes: function(d3chart, groups, settings) {
// result
var categories = Tools.makeCategories(groups);
// make axes
function makeTickFormat (id) {
Tools.makeTickFormat({
categories : categories.array[id],
type : settings.get(id + '_axis_type'),
precision : settings.get(id + '_axis_precision'),
formatter : function(formatter) {
if (formatter) {
d3chart[id + 'Axis'].tickFormat(function(value) {
return formatter(value);
});
if ( $( '#' + canvas_id ).length > 0 ) {
var canvas = d3.select( '#' + canvas_id );
canvas.datum( groups ).call( d3chart );
if ( chart.definition.zoomable && chart.definition.zoomable != 'native' ) {
d3chart.clipEdge && d3chart.clipEdge( true );
Tools.addZoom({
xAxis : d3chart.xAxis,
yAxis : d3chart.yAxis,
yDomain: d3chart.yDomain,
xDomain: d3chart.xDomain,
redraw : function() { d3chart.update() },
svg : canvas
});
}
nv.utils.windowResize( d3chart.update );
}
} catch ( err ) {
chart.state( 'failed', err );
}
});
};
makeTickFormat('x');
makeTickFormat('y');
},
// handle error
_handleError: function(chart, err) {
chart.state('failed', err);
}
});
return true;
},
/** Format axes ticks */
_makeAxes: function( d3chart, groups, settings ) {
var categories = Tools.makeCategories( groups );
function makeTickFormat( id ) {
Tools.makeTickFormat({
categories : categories.array[ id ],
type : settings.get( id + '_axis_type' ),
precision : settings.get( id + '_axis_precision' ),
formatter : function( formatter ) {
formatter && d3chart[ id + 'Axis' ].tickFormat( function( value ) {
return formatter( value );
});
}
});
};
makeTickFormat( 'x' );
makeTickFormat( 'y' );
}
});
});
config/plugins/visualizations/charts/static/charts/nvd3/pie/config.js
+4 -2
View File
@@ -52,7 +52,8 @@ return $.extend(true, {}, {
show_legend : {
title : 'Show legend',
info : 'Would you like to add a legend?',
type : 'radiobutton',
type : 'select',
display : 'radiobutton',
init : 'false',
data : [
{
@@ -103,7 +104,8 @@ return $.extend(true, {}, {
label_outside : {
title : 'Show outside',
info : 'Would you like to show labels outside the donut?',
type : 'radiobutton',
type : 'select',
display : 'radiobutton',
init : 'false',
data : [
{
config/plugins/visualizations/charts/static/library/datasets.js
+2 -8
View File
@@ -1,11 +1,8 @@
// dependencies
define(['utils/utils'], function(Utils) {
/**
* This class handles, formats and caches datasets.
*/
return Backbone.Collection.extend(
{
define( [ 'utils/utils' ], function( Utils ) {
return Backbone.Collection.extend({
// list of datasets
list: {},
@@ -14,10 +11,7 @@ return Backbone.Collection.extend(
// initialize
initialize: function(app, options){
// link app
this.app = app;
// configure options
this.options = Utils.merge(options, this.optionsDefault);
},
config/plugins/visualizations/charts/static/library/jobs.js
+139 -176
View File
@@ -1,190 +1,153 @@
// dependencies
define(['utils/utils'], function(Utils) {
/**
* This class handles job submissions to the charts tool.
*/
return Backbone.Model.extend({
// initialize
initialize: function(app, options) {
// link app
this.app = app;
define( [ 'utils/utils' ], function( Utils ) {
return Backbone.Model.extend({
initialize: function( app, options ) {
this.app = app;
this.options = Utils.merge( options, this.optionsDefault );
},
// link options
this.options = Utils.merge(options, this.optionsDefault);
},
// request job results
request: function(chart, settings_string, columns_string, success, error) {
// link this
var self = this;
// set chart state
chart.state('wait', 'Requesting job results...');
// get dataset job id if already set
var dataset_id_job = chart.get('dataset_id_job');
if (dataset_id_job != '') {
// wait for job completion
self._wait(chart, success, error);
} else {
// post job
self._submit(chart, settings_string, columns_string, success, error);
}
},
// clean
cleanup: function(chart) {
// link this
var self = this;
// cleanup previous dataset file
var previous = chart.get('dataset_id_job');
if (previous != '') {
Utils.request({
type : 'PUT',
url : config.root + 'api/histories/none/contents/' + previous,
data : { deleted: true },
success : function() {
// update galaxy history
self._refreshHdas();
}
});
// reset id
chart.set('dataset_id_job', '');
}
},
// create job
_submit: function(chart, settings_string, columns_string, success, error) {
// link this
var self = this;
// backup chart details
var chart_id = chart.id;
var chart_type = chart.get('type');
// get chart settings
var chart_definition = chart.definition;
// configure tool
data = {
'tool_id' : 'charts',
'inputs' : {
'input' : {
'id' : chart.get('dataset_id'),
'src' : 'hda'
},
'module' : chart_definition.execute,
'columns' : columns_string,
'settings' : settings_string
/* Request job results */
request: function( chart, settings_string, columns_string, success, error ) {
var self = this;
chart.state( 'wait', 'Requesting job results...' );
var dataset_id_job = chart.get( 'dataset_id_job' );
if (dataset_id_job != '') {
self._wait( chart, success, error );
} else {
self._submit( chart, settings_string, columns_string, success, error );
}
}
},
// set chart state
chart.state('wait', 'Sending job request...');
/* Remove previous data when re-running jobs */
cleanup: function( chart ) {
var self = this;
var previous = chart.get( 'dataset_id_job' );
if (previous != '') {
Utils.request({
type : 'PUT',
url : config.root + 'api/histories/none/contents/' + previous,
data : { deleted: true },
success : function() {
self._refreshHdas();
}
});
chart.set( 'dataset_id_job', '' );
}
},
// post job
Utils.request({
type : 'POST',
url : config.root + 'api/tools',
data : data,
success : function(response) {
if (!response.outputs || response.outputs.length == 0) {
chart.state('failed', 'Job submission failed. No response.');
// create job
_submit: function(chart, settings_string, columns_string, success, error) {
// link this
var self = this;
// backup chart details
var chart_id = chart.id;
var chart_type = chart.get('type');
// get chart settings
var chart_definition = chart.definition;
// configure tool
data = {
'tool_id' : 'charts',
'inputs' : {
'input' : {
'id' : chart.get('dataset_id'),
'src' : 'hda'
},
'module' : chart_definition.execute,
'columns' : columns_string,
'settings' : settings_string
}
}
// set chart state
chart.state('wait', 'Sending job request...');
// post job
Utils.request({
type : 'POST',
url : config.root + 'api/tools',
data : data,
success : function(response) {
if (!response.outputs || response.outputs.length == 0) {
chart.state('failed', 'Job submission failed. No response.');
// call error
error && error();
} else {
// update galaxy history
self._refreshHdas();
// get dataset
var job = response.outputs[0];
// check dataset
chart.state('wait', 'Your job has been queued. You may close the browser window. The job will run in the background.');
// backup resulting dataset id
chart.set('dataset_id_job', job.id);
// save
self.app.storage.save();
// wait for job completion
self._wait(chart, success, error);
}
},
error : function(response) {
var message = '';
if (response && response.message && response.message.data && response.message.data.input) {
message = response.message.data.input + '.';
}
// update state
chart.state('failed', 'This visualization requires the \'charts\' tool. Please make sure it is installed. ' + message);
// call error
error && error();
} else {
// update galaxy history
self._refreshHdas();
}
});
},
// get dataset
var job = response.outputs[0];
// check dataset
chart.state('wait', 'Your job has been queued. You may close the browser window. The job will run in the background.');
// backup resulting dataset id
chart.set('dataset_id_job', job.id);
// save
self.app.storage.save();
// wait for job completion
self._wait(chart, success, error);
/** Request job details */
_wait: function( chart, success, error ) {
var self = this;
Utils.request({
type : 'GET',
url : config.root + 'api/datasets/' + chart.get( 'dataset_id_job' ),
data : {},
success : function( dataset ) {
var ready = false;
switch ( dataset.state ) {
case 'ok':
chart.state( 'wait', 'Job completed successfully...' );
success && success( dataset );
ready = true;
break;
case 'error':
chart.state( 'failed', 'Job has failed. Please check the history for details.' );
error && error( dataset );
ready = true;
break;
case 'running':
chart.state( 'wait', 'Your job is running. You may close the browser window. The job will continue in the background.' );
}
if ( !ready ) {
setTimeout( function() {
self._wait( chart, success, error );
}, self.app.config.get( 'query_timeout' ) );
}
}
},
error : function(response) {
var message = '';
if (response && response.message && response.message.data && response.message.data.input) {
message = response.message.data.input + '.';
}
// update state
chart.state('failed', 'This visualization requires the \'charts\' tool. Please make sure it is installed. ' + message);
// call error
error && error();
}
});
},
// request job details
_wait: function(chart, success, error) {
var self = this;
Utils.request({
type : 'GET',
url : config.root + 'api/datasets/' + chart.get('dataset_id_job'),
data : {},
success : function(dataset) {
// check dataset state
var ready = false;
switch (dataset.state) {
case 'ok':
// update state
chart.state('wait', 'Job completed successfully...');
// execute success
success && success(dataset);
// stop loop
ready = true;
break;
case 'error':
// update state
chart.state('failed', 'Job has failed. Please check the history for details.');
// call error
error && error(dataset);
// stop loop
ready = true;
break;
case 'running':
// wait
chart.state('wait', 'Your job is running. You may close the browser window. The job will continue in the background.');
}
// wait and try again
if (!ready) {
setTimeout(function() {
self._wait(chart, success, error);
}, self.app.config.get('query_timeout'));
}
}
});
},
// refresh history panel
_refreshHdas: function() {
// update galaxy history
if (Galaxy && Galaxy.currHistoryPanel) {
Galaxy.currHistoryPanel.refreshContents();
});
},
/** Refresh history panel */
_refreshHdas: function() {
Galaxy && Galaxy.currHistoryPanel && Galaxy.currHistoryPanel.refreshContents();
}
}
});
});
});
config/plugins/visualizations/charts/static/library/storage.js
+75 -135
View File
@@ -1,142 +1,82 @@
// dependencies
define(['utils/utils', 'plugin/models/chart', 'plugin/models/group', 'mvc/visualization/visualization-model'], function(Utils, Chart, Group) {
/**
* This class saves and loads a chart through the api.
*/
return Backbone.Model.extend({
// viz model
vis: null,
// initialize
initialize: function(app) {
// link app
this.app = app;
// link chart
this.chart = this.app.chart;
// link options
this.options = this.app.options;
// initialize parameters
this.id = this.options.id;
// create visualization
this.vis = new Visualization({
type : 'charts',
config : {
dataset_id : this.options.config.dataset_id,
chart_dict : {}
}
});
// add visualization id
if (this.id) {
this.vis.id = this.id;
}
// add charts
var chart_dict = this.options.config.chart_dict;
if (chart_dict) {
this.vis.get('config').chart_dict = chart_dict;
}
},
// pack and save nested chart model
save: function() {
// link chart
var chart = this.app.chart;
// reset
this.vis.get('config').chart_dict = {};
// set title
var title = chart.get('title');
if (title != '') {
this.vis.set('title', title);
}
// create chart dictionary
var chart_dict = {
attributes : chart.attributes,
settings : chart.settings.attributes,
groups : []
};
// append groups
chart.groups.each(function(group) {
chart_dict.groups.push(group.attributes);
});
// add chart to charts array
this.vis.get('config').chart_dict = chart_dict;
// save visualization
var self = this;
this.vis.save()
.fail(function(xhr, status, message) {
console.error(xhr, status, message);
})
.then(function(response) {
if (response && response.id) {
self.id = response.id;
define( [ 'utils/utils', 'plugin/models/chart', 'mvc/visualization/visualization-model' ],
function( Utils, Chart ) {
return Backbone.Model.extend({
vis: null,
initialize: function( app ) {
this.app = app;
this.chart = this.app.chart;
this.options = this.app.options;
this.id = this.options.id;
this.vis = new Visualization({
id : this.id,
type : 'charts',
config : {
dataset_id : this.options.config.dataset_id,
chart_dict : {}
}
});
},
// load nested models/collections from packed dictionary
load: function() {
// get charts array
var chart_dict = this.vis.get('config').chart_dict;
// check
if (!chart_dict.attributes) {
return false;
}
// get chart type
var chart_type = chart_dict.attributes['type'];
// check chart type
if (!chart_type) {
console.debug('Storage::load() - Chart type not provided. Invalid format.');
return false;
}
// get chart definition
var chart_definition = this.app.types.get(chart_type);
if (!chart_definition) {
console.debug('Storage::load() - Chart type not supported. Please re-configure the chart. Resetting chart.');
return false;
}
// initiating chart
console.debug('Storage::load() - Loading chart type ' + chart_type + '.');
// parse chart definition
this.chart.definition = chart_definition;
// main
this.chart.set(chart_dict.attributes);
// set state
this.chart.state('ok', 'Loading saved visualization...');
// get settings
this.chart.settings.set(chart_dict.settings);
// get groups
for (var j in chart_dict.groups) {
this.chart.groups.add(new Group(chart_dict.groups[j]));
}
// reset modified flag
this.chart.set('modified', false);
// return
return true;
}
});
var chart_dict = this.options.config.chart_dict;
if ( chart_dict ) {
this.vis.get( 'config' ).chart_dict = chart_dict;
}
},
/** Pack and save nested chart model */
save: function() {
var self = this;
var chart = this.app.chart;
var chart_dict = {
attributes : chart.attributes,
settings : chart.settings.attributes,
groups : []
};
chart.groups.each( function( group ) {
chart_dict.groups.push( group.attributes );
});
this.vis.set( 'title', chart.get( 'title' ) || '' );
this.vis.get( 'config' ).chart_dict = chart_dict;
this.vis.save().fail(function( xhr, status, message ) {
self.app.showModal( 'Saving failed.', 'An attempt to save this chart to the server failed. Please try again and contact the administrator.' );
})
.then( function( response ) {
if ( response && response.id ) {
self.id = response.id;
} else {
console.debug('Storage::save() - Unrecognized response. Saving may have failed.');
}
});
},
/** Load nested models/collections from packed dictionary */
load: function() {
var chart_dict = this.vis.get( 'config' ).chart_dict;
if ( !chart_dict.attributes ) {
console.debug('Storage::load() - Chart attributes not found. Invalid format.');
return false;
}
var chart_type = chart_dict.attributes[ 'type' ];
if ( !chart_type ) {
console.debug('Storage::load() - Chart type not provided. Invalid format.');
return false;
}
var chart_definition = this.app.types.get( chart_type );
if ( !chart_definition ) {
console.debug('Storage::load() - Chart type not supported. Please re-configure the chart. Resetting chart.');
return false;
}
this.chart.definition = chart_definition;
this.chart.set( chart_dict.attributes );
this.chart.state( 'ok', 'Loading saved visualization...' );
this.chart.settings.set( chart_dict.settings );
this.chart.groups.reset();
this.chart.groups.add( chart_dict.groups );
this.chart.set( 'modified', false );
console.debug('Storage::load() - Loading chart type ' + chart_type + '.');
return true;
}
});
});
config/plugins/visualizations/charts/static/library/ui-table-form.js
-244
View File
@@ -1,244 +0,0 @@
// dependencies
define(['mvc/ui/ui-table', 'mvc/ui/ui-misc', 'utils/utils'],
function(Table, Ui, Utils) {
/**
* This class takes a dictionary as input an creates an input form. It uses the Ui.Table element to organize and format the form elements.
*/
var View = Backbone.View.extend({
// options
optionsDefault: {
title : '',
content : '',
mode : ''
},
// elements
list: [],
// initialize
initialize: function(app, options) {
// link app
this.app = app;
// get options
this.options = Utils.merge(options, this.optionsDefault);
// ui elements
this.table_title = new Ui.Label({title: this.options.title});
this.table = new Table.View({content: this.options.content});
// create element
var $view = $('<div class="ui-form"/>');
if (this.options.title) {
$view.append(this.table_title.$el);
}
$view.append(this.table.$el);
// add element
this.setElement($view);
},
// title
title: function(new_title) {
this.table_title.title(new_title);
},
// update
update: function(settings, model) {
// reset table
this.table.delAll();
// reset list
this.list = [];
// load settings elements into table
for (var id in settings) {
this._add(settings[id].id || id, settings[id], model);
}
// trigger change
for (var id in this.list) {
this.list[id].trigger('change');
}
},
// add table row
_add: function(id, settings_def, model) {
// link this
var self = this;
// field wrapper
var field = null;
// create select field
var type = settings_def.type;
switch(type) {
// text input field
case 'text' :
field = new Ui.Input({
id : 'field-' + id,
placeholder : settings_def.placeholder,
value : model.get(id),
onchange : function(value) {
model.set(id, value);
}
});
break;
// radiobox field
case 'radiobutton' :
field = new Ui.RadioButton.View({
id : 'field-' + id,
data : settings_def.data,
value : model.get(id),
onchange : function(value) {
// set new value
model.set(id, value);
// find selected dictionary
var dict = _.findWhere(settings_def.data, {value: value});
if (dict && dict.operations) {
var operations = dict.operations;
for (var i in operations.show) {
var target = operations.show[i];
self.table.get(target).show();
}
for (var i in operations.hide) {
var target = operations.hide[i];
self.table.get(target).hide();
}
}
}
});
break;
// select field
case 'select' :
field = new Ui.Select.View({
id : 'field-' + id,
data : settings_def.data,
value : model.get(id),
onchange : function(value) {
// set new value
model.set(id, value);
// find selected dictionary
var dict = _.findWhere(settings_def.data, {value: value});
if (dict && dict.operations) {
var operations = dict.operations;
for (var i in operations.show) {
var target = operations.show[i];
self.table.get(target).show();
}
for (var i in operations.hide) {
var target = operations.hide[i];
self.table.get(target).hide();
}
}
}
});
break;
case 'dataset':
field = new Ui.Select.View({
id : 'field-' + id,
onchange : function(value) {
// set new value
model.set(id, value);
}
});
// link refresh event
self.app.datasets.on('all', function() {
// identify selectables
var selectable = [];
self.app.datasets.each(function(dataset) {
if (dataset.get('datatype_id') == settings_def.data) {
selectable.push({value: dataset.get('id'), label: dataset.get('name')});
}
});
// update select field
field.update(selectable);
// set value
if (!model.get(id)) {
model.set(id, field.first());
}
field.value(model.get(id));
});
// trigger change
self.app.datasets.trigger('all.datasets');
break;
// slider input field
case 'textarea' :
field = new Ui.Textarea({
id : 'field-' + id,
onchange : function() {
model.set(id, field.value());
}
});
break;
// separator
case 'separator' :
field = $('<div/>');
break;
// default
default:
field = new Ui.Input({
id : 'field-' + id,
placeholder : settings_def.placeholder,
type : settings_def.type,
onchange : function() {
model.set(id, field.value());
}
});
}
// set value
if (type != 'separator') {
if (!model.get(id)) {
model.set(id, settings_def.init);
}
field.value(model.get(id));
// add list
this.list[id] = field;
// combine field and info
var $input = $('<div/>');
$input.append(field.$el);
if (settings_def.info) {
$input.append('<div class="ui-form-info">' + settings_def.info + '</div>');
}
// add row to table
if (this.options.style == 'bold') {
this.table.add(new Ui.Label({title: settings_def.title, cls: 'form-label'}).$el);
this.table.add($input);
} else {
this.table.add('<span class="ui-form-title">' + settings_def.title + '</span>', '25%');
this.table.add($input);
}
} else {
this.table.add('<div class="ui-form-separator">' + settings_def.title + ':<div/>');
this.table.add($('<div/>'));
}
// add to table
this.table.append(id);
// show/hide
if (settings_def.hide) {
this.table.get(id).hide();
}
}
});
return {
View : View
}
});
config/plugins/visualizations/charts/static/models/chart.js
+55 -64
View File
@@ -1,67 +1,58 @@
// dependencies
define(['plugin/models/groups'], function(Groups) {
define( [], function() {
var Groups = Backbone.Collection.extend({
model: Backbone.Model.extend({
defaults : {
key : 'Data label',
date : ''
},
// model
return Backbone.Model.extend({
// defaults
defaults : {
id : null,
title : '',
type : '',
date : null,
state : '',
state_info : '',
modified : false,
dataset_id : '',
dataset_id_job : ''
},
// initialize
initialize: function(options) {
this.groups = new Groups();
this.settings = new Backbone.Model();
},
// reset
reset: function() {
this.clear({silent: true}).set(this.defaults);
this.groups.reset();
this.settings.clear();
this.trigger('reset', this);
},
// copy
copy: function(new_chart) {
// copy chart/groups
var current = this;
// set attributes
current.clear({silent: true}).set(this.defaults);
current.set(new_chart.attributes);
// set nested models/collections
current.settings = new_chart.settings.clone();
current.groups.reset();
new_chart.groups.each(function(group) {
current.groups.add(group.clone());
});
// trigger change
current.trigger('change', current);
},
// set state
state: function(value, info) {
// set status
this.set('state', value);
this.set('state_info', info);
// trigger set state
this.trigger('set:state');
// log status
console.debug('Chart:state() - ' + info + ' (' + value + ')');
}
});
reset: function(){
this.clear( { silent: true } ).set( this.defaults );
this.trigger( 'reset', this );
}
})
});
return Backbone.Model.extend({
defaults : {
id : null,
title : '',
type : '',
date : null,
state : '',
state_info : '',
modified : false,
dataset_id : '',
dataset_id_job : ''
},
initialize: function( options ) {
this.groups = new Groups();
this.settings = new Backbone.Model();
},
reset: function() {
this.clear( { silent: true } ).set( this.defaults );
this.groups.reset();
this.settings.clear();
this.trigger( 'reset', this );
},
copy: function( new_chart ) {
var current = this;
current.clear( { silent: true } ).set( this.defaults );
current.set( new_chart.attributes );
current.settings = new_chart.settings.clone();
current.groups.reset();
new_chart.groups.each( function( group ) {
current.groups.add( group.clone() );
});
current.trigger( 'change', current );
},
state: function( value, info ) {
this.set( { 'state': value, 'state_info': info } );
this.trigger( 'set:state' );
console.debug( 'Chart:state() - ' + info + ' (' + value + ')' );
}
});
});
config/plugins/visualizations/charts/static/models/config.js
+7 -12
View File
@@ -1,13 +1,8 @@
// dependencies
define([], function() {
// model
return Backbone.Model.extend({
// options
defaults : {
query_limit : 10000,
query_timeout : 100
}
});
define( [], function() {
return Backbone.Model.extend({
defaults : {
query_limit : 1000,
query_timeout : 100
}
});
});
config/plugins/visualizations/charts/static/models/group.js
-19
View File
@@ -1,19 +0,0 @@
// dependencies
define([], function() {
// model
return Backbone.Model.extend({
// options
defaults : {
key : 'Data label',
date : ''
},
// reset
reset: function(){
this.clear({silent: true}).set(this.defaults);
this.trigger('reset', this);
}
});
});
config/plugins/visualizations/charts/static/models/groups.js
-9
View File
@@ -1,9 +0,0 @@
// dependencies
define(['plugin/models/group'], function(Group) {
// collection
return Backbone.Collection.extend({
model: Group
});
});
config/plugins/visualizations/charts/static/plugins/nvd3/nv.d3.css
+1 -1
View File
File diff suppressed because one or more lines are too long
config/plugins/visualizations/charts/static/views/editor.js
+154 -312
View File
@@ -1,327 +1,169 @@
// dependencies
define(['mvc/ui/ui-tabs', 'mvc/ui/ui-misc', 'mvc/ui/ui-portlet', 'utils/utils',
'plugin/models/chart', 'plugin/models/group',
'plugin/views/group', 'plugin/views/settings', 'plugin/views/types'],
function(Tabs, Ui, Portlet, Utils, Chart, Group, GroupView, SettingsView, TypesView) {
/**
* The charts editor holds the tabs for selecting chart types, chart configuration
* and data group selections.
*/
return Backbone.View.extend({
// initialize
initialize: function(app, options){
// link this
var self = this;
// link application
this.app = app;
// get current chart object
this.chart = this.app.chart;
// message element
this.message = new Ui.Message();
// create portlet
this.portlet = new Portlet.View({
icon : 'fa-bar-chart-o',
title: 'Editor',
operations : {
'save' : new Ui.ButtonIcon({
icon : 'fa-save',
tooltip : 'Draw Chart',
title : 'Draw',
onclick : function() {
self._saveChart();
}
}),
'back' : new Ui.ButtonIcon({
icon : 'fa-caret-left',
tooltip : 'Return to Viewer',
title : 'Cancel',
onclick : function() {
// show viewer/viewport
self.app.go('viewer');
// reset chart
self.app.storage.load();
}
})
}
});
//
// grid with chart types
//
this.types = new TypesView(app, {
onchange : function(chart_type) {
// get chart definition
var chart_definition = self.app.types.get(chart_type);
if (!chart_definition) {
console.debug('FAILED - Editor::onchange() - Chart type not supported.');
define( [ 'mvc/ui/ui-tabs', 'mvc/ui/ui-misc', 'mvc/ui/ui-portlet', 'utils/utils',
'plugin/models/chart', 'plugin/views/settings', 'plugin/views/groups', 'plugin/views/types' ],
function( Tabs, Ui, Portlet, Utils, Chart, SettingsView, GroupsView, TypesView ) {
return Backbone.View.extend({
initialize: function( app, options ){
var self = this;
this.app = app;
this.chart = this.app.chart;
this.message = new Ui.Message( { cls: 'ui-margin-bottom' } );
this.portlet = new Portlet.View({
icon : 'fa-bar-chart-o',
title: 'Editor',
operations : {
'save' : new Ui.ButtonIcon({
icon : 'fa-save',
tooltip : 'Draw Chart',
title : 'Draw',
onclick : function() {
self._saveChart();
}
}),
'back' : new Ui.ButtonIcon({
icon : 'fa-caret-left',
tooltip : 'Return to Viewer',
title : 'Cancel',
onclick : function() {
self.app.go( 'viewer' );
self.app.storage.load();
}
})
}
// parse chart definition
self.chart.definition = chart_definition;
// reset type relevant chart content
self.chart.settings.clear();
// update chart type
self.chart.set({type: chart_type});
// set modified flag
self.chart.set('modified', true);
// log
console.debug('Editor::onchange() - Switched chart type.');
},
ondblclick : function(chart_id) {
self._saveChart();
}
});
//
// tabs
//
this.tabs = new Tabs.View({
title_new : 'Add Data',
onnew : function() {
var group = self._addGroupModel();
self.tabs.show(group.id);
}
});
//
// main/default tab
//
// construct elements
this.title = new Ui.Input({
placeholder: 'Chart title',
onchange: function() {
self.chart.set('title', self.title.value());
}
});
// append element
var $main = $('<div/>');
$main.append((new Ui.Label({ title : 'Provide a chart title:'}).$el));
$main.append(this.title.$el);
$main.append($('<div/>').addClass('ui-form-info').html('This title will appear in the list of \'Saved Visualizations\'. Charts are saved upon creation.'));
$main.append(this.types.$el.addClass('ui-margin-top'));
// add tab
this.tabs.add({
id : 'main',
title : 'Start',
$el : $main
});
//
// main settings tab
//
// create settings view
this.settings = new SettingsView(this.app);
// add tab
this.tabs.add({
id : 'settings',
title : 'Configuration',
$el : this.settings.$el
});
// append tabs
this.portlet.append(this.message.$el.addClass('ui-margin-top'));
this.portlet.append(this.tabs.$el.addClass('ui-margin-top'));
// elements
this.setElement(this.portlet.$el);
// hide back button on startup
this.tabs.hideOperation('back');
// chart events
var self = this;
this.chart.on('change:title', function(chart) {
self._refreshTitle();
});
this.chart.on('change:type', function(chart) {
self.types.value(chart.get('type'));
});
this.chart.on('reset', function(chart) {
self._resetChart();
});
this.app.chart.on('redraw', function(chart) {
self.portlet.showOperation('back');
});
// groups events
this.app.chart.groups.on('add', function(group) {
self._addGroup(group);
});
this.app.chart.groups.on('remove', function(group) {
self._removeGroup(group);
});
this.app.chart.groups.on('reset', function(group) {
self._removeAllGroups();
});
this.app.chart.groups.on('change:key', function(group) {
self._refreshGroupKey();
});
// reset
this._resetChart();
},
});
// hide
show: function() {
this.$el.show();
},
// hide
hide: function() {
this.$el.hide();
},
// grid with chart types
this.types = new TypesView(app, {
onchange : function(chart_type) {
var chart_definition = self.app.types.get( chart_type );
if ( !chart_definition ) {
console.debug('FAILED - Editor::onchange() - Chart type not supported.');
}
self.chart.definition = chart_definition;
self.chart.settings.clear();
self.chart.set( { type : chart_type, modified : true } );
console.debug( 'Editor::onchange() - Switched chart type.' );
},
ondblclick : function( chart_id ) {
self._saveChart();
}
});
// refresh title
_refreshTitle: function() {
var title = this.chart.get('title');
this.portlet.title(title);
if (this.title.value() != title) {
this.title.value(title);
}
},
// refresh group
_refreshGroupKey: function() {
var self = this;
var counter = 0;
this.chart.groups.each(function(group) {
var title = group.get('key', '');
if (title == '') {
title = 'Data label';
}
self.tabs.title(group.id, ++counter + ': ' + title);
});
},
// add group model
_addGroupModel: function() {
var group = new Group({
id : Utils.uid()
});
this.chart.groups.add(group);
return group;
},
// input field for chart title
this.title = new Ui.Input({
placeholder: 'Chart title',
onchange: function() {
self.chart.set( 'title', self.title.value() );
}
});
// add group tab
_addGroup: function(group) {
// link this
var self = this;
// create view
var group_view = new GroupView(this.app, {group: group});
// add new tab
this.tabs.add({
id : group.id,
$el : group_view.$el,
ondel : function() {
self.chart.groups.remove(group.id);
}
});
// update titles
this._refreshGroupKey();
// reset
this.chart.set('modified', true);
},
// remove group
_removeGroup: function(group) {
this.tabs.del(group.id);
// update titles
this._refreshGroupKey();
// reset
this.chart.set('modified', true);
},
// create tabs
this.tabs = new Tabs.View( {} );
this.tabs.add({
id : 'main',
title : 'Start',
icon : 'fa fa-bars',
tooltip : 'Start by selecting a chart type.',
$el : $( '<div/>' ).append( ( new Ui.Label( { title : 'Provide a chart title:' } ).$el ) )
.append( this.title.$el )
.append( $( '<div/>' ).addClass( 'ui-form-info' ).html( 'This title will appear in the list of \'Saved Visualizations\'. Charts are saved upon creation.' ) )
.append( this.types.$el.addClass( 'ui-margin-top' ) )
});
this.tabs.add({
id : 'settings',
title : 'Customize',
icon : 'fa-gear',
tooltip : 'Customize charts options.',
$el : ( new SettingsView( this.app ) ).$el
});
this.tabs.add({
id : 'groups',
title : 'Select data',
icon : 'fa-database',
tooltip : 'Specify your data columns.',
$el : ( new GroupsView( this.app ) ).$el
});
// remove group
_removeAllGroups: function(group) {
this.tabs.delRemovable();
},
// reset
_resetChart: function() {
// reset chart details
this.chart.set('id', Utils.uid());
this.chart.set('type', 'nvd3_bar');
this.chart.set('dataset_id', this.app.options.config.dataset_id);
this.chart.set('title', 'New Chart');
// reset back button
this.portlet.hideOperation('back');
},
// create chart
_saveChart: function() {
// update chart data
this.chart.set({
type : this.types.value(),
title : this.title.value(),
date : Utils.time()
});
// make sure that at least one data group is available
if (this.chart.groups.length == 0) {
this.message.update({message: 'Please select data columns before drawing the chart.'});
var group = this._addGroupModel();
this.tabs.show(group.id);
return;
}
// make sure that all necessary columns are assigned
var self = this;
var valid = true;
var chart_def = this.chart.definition;
this.chart.groups.each(function(group) {
if (!valid) {
// set elements
this.portlet.append( this.message.$el );
this.portlet.append( this.tabs.$el.addClass( 'ui-margin-top-large' ) );
this.setElement( this.portlet.$el );
this.tabs.hideOperation( 'back' );
// chart events
this.chart.on( 'change:title', function( chart ) { self._refreshTitle() } );
this.chart.on( 'change:type', function( chart ) { self.types.value( chart.get( 'type' ) ) } );
this.chart.on( 'reset', function( chart ) { self._resetChart() } );
this.app.chart.on( 'redraw', function( chart ) { self.portlet.showOperation( 'back' ) } );
this._resetChart();
},
/** Show editor */
show: function() {
this.$el.show();
},
/** Hide editor */
hide: function() {
this.$el.hide();
},
/** Refresh title handler */
_refreshTitle: function() {
var title = this.chart.get( 'title' );
this.portlet.title( title );
this.title.value( title );
},
/** Reset entire chart */
_resetChart: function() {
this.chart.set({
'id' : Utils.uid(),
'type' : 'nvd3_bar',
'dataset_id' : this.app.options.config.dataset_id,
'title' : 'New Chart'
});
this.chart.groups.reset();
this.chart.groups.add( { id : Utils.uid() } );
this.portlet.hideOperation( 'back' );
},
/** Save chart data */
_saveChart: function() {
var self = this;
this.chart.set({
type : this.types.value(),
title : this.title.value(),
date : Utils.time()
});
if ( this.chart.groups.length == 0 ) {
this.message.update( { message: 'Please specify data columns before drawing the chart.' } );
this.tabs.show( 'groups' );
return;
}
for (var key in chart_def.columns) {
if (group.attributes[key] === '__null__') {
self.message.update({status: 'danger', message: 'This chart type requires column types not found in your tabular file.'});
self.tabs.show(group.id);
valid = false;
return;
var valid = true;
var chart_def = this.chart.definition;
this.chart.groups.each( function( group ) {
if ( valid ) {
for ( var key in chart_def.columns ) {
if ( group.attributes[ key ] === null ) {
self.message.update( { status: 'danger', message: 'This chart type requires column types not found in your tabular file.' } );
self.tabs.show( 'groups' );
valid = false;
}
}
}
});
if ( valid ) {
this.app.go( 'viewer' );
this.app.deferred.execute( function() {
self.app.storage.save();
self.chart.trigger( 'redraw' );
});
}
});
// validate if columns have been selected
if (!valid) {
return;
}
// show viewport
this.app.go('viewer');
// wait until chart is ready
var self = this;
this.app.deferred.execute(function() {
// save
self.app.storage.save();
// trigger redraw
self.chart.trigger('redraw');
});
}
});
});
});
config/plugins/visualizations/charts/static/views/group.js
-254
View File
@@ -1,254 +0,0 @@
// dependencies
define(['mvc/ui/ui-table', 'mvc/ui/ui-misc', 'utils/utils'],
function(Table, Ui, Utils) {
/**
* This class renders the data group selection fields.
*/
return Backbone.View.extend({
// initialize
initialize: function(app, options) {
// link app
this.app = app;
// link this
var self = this;
// get current chart object
this.chart = this.app.chart;
// access group
this.group = options.group;
// ui elements
this.group_key = new Ui.Input({
placeholder: 'Data label',
onchange: function(value) {
self.group.set('key', value);
}
});
this.table = new Table.View({content: 'No data column.'});
// create element
var $view = $('<div/>');
$view.append((new Ui.Label({title: 'Provide a label:'})).$el);
$view.append(this.group_key.$el.addClass('ui-margin-bottom'));
$view.append((new Ui.Label({title: 'Select columns:'})).$el.addClass('ui-margin-top'));
$view.append(this.table.$el.addClass('ui-margin-bottom'));
// add element
this.setElement($view);
// change
var self = this;
this.chart.on('change:dataset_id', function() {
self._refreshTable();
});
this.chart.on('change:type', function() {
self._refreshTable();
});
this.group.on('change:key', function() {
self._refreshGroupKey();
});
this.group.on('change', function() {
self._refreshGroup();
});
// refresh
this._refreshTable();
this._refreshGroupKey();
this._refreshGroup();
},
// update group selection table
_refreshTable: function() {
// identify datasets
var dataset_id = this.chart.get('dataset_id');
var chart_type = this.chart.get('type');
// check if dataset is available
if (!dataset_id || !chart_type) {
return;
}
// link this
var self = this;
// configure chart type
var chart_definition = this.chart.definition;
// reset table
this.table.delAll();
// load list for select fields
var select_list = {};
for (var id in chart_definition.columns) {
// get definition
var data_def = chart_definition.columns[id];
if (!data_def) {
console.debug('Group::_refreshTable() - Skipping column definition.');
continue;
}
// create select field
var select = new Ui.Select.View({
id : 'select_' + id
});
// title
var title = data_def.title;
// is unique
if (data_def.is_unique) {
title += '&nbsp;(all data labels)';
}
// add row to table
this.table.add(title, '25%');
this.table.add(select.$el);
this.table.append(id);
// add select field to list
select_list[id] = select;
}
// loading
this.chart.state('wait', 'Loading metadata...');
// get dataset
this.app.deferred.execute(function(process) {
self.app.datasets.request({
id : dataset_id,
success : function(dataset) {
for (var id in select_list) {
self._addRow(id, dataset, select_list, chart_definition.columns[id])
}
self.chart.state('ok', 'Metadata initialized...');
process.resolve();
}
});
});
},
// add row
_addRow: function(id, dataset, select_list, column_definition) {
// link this
var self = this;
// available column options
var is_label = column_definition.is_label;
var is_auto = column_definition.is_auto;
var is_numeric = column_definition.is_numeric;
var is_unique = column_definition.is_unique;
var is_zero = column_definition.is_zero;
var is_text = column_definition.is_text;
// configure columns
var columns = [];
// get select
var select = select_list[id];
// add auto selection column
if (is_auto) {
columns.push({
'label' : 'Column: Row Number',
'value' : 'auto'
});
}
// add zero selection column
if (is_zero) {
columns.push({
'label' : 'Column: None',
'value' : 'zero'
});
}
// meta data
var meta = dataset.metadata_column_types;
for (var key in meta) {
// check type
var valid = false;
if (meta[key] == 'int' || meta[key] == 'float') {
valid = is_numeric;
} else {
valid = is_text || is_label;
}
// check type
if (valid) {
// add to selection
columns.push({
'label' : 'Column: ' + (parseInt(key) + 1),
'value' : key
});
}
}
// update selection list
select.update(columns);
// set initial value
if (is_unique && this.chart.groups.first()) {
this.group.set(id, this.chart.groups.first().get(id));
}
// update current value
if (!select.exists(this.group.get(id))) {
// get first value
var first = select.first();
// log
console.debug('Group()::_addRow() - Switching model value from "' + this.group.get(id) + '" to "' + first + '".');
// update model value
this.group.set(id, first);
}
// set group value
select.value(this.group.get(id));
// link group with select field
this.group.off('change:' + id);
this.group.on('change:' + id, function(){
select.value(self.group.get(id));
});
// link select field with group
select.setOnChange(function(value) {
// update model value
if (is_unique) {
// update all groups
self.chart.groups.each(function(group) {
group.set(id, value);
});
} else {
// only change this group
self.group.set(id, value);
}
self.chart.set('modified', true);
});
// show select field
select.show();
},
// update
_refreshGroup: function() {
this.group.set('date', Utils.time());
},
// update group key
_refreshGroupKey: function() {
var key_text = this.group.get('key');
if (key_text === undefined) {
key_text = '';
}
if (key_text != this.group_key.value()) {
this.group_key.value(key_text);
}
}
});
});
config/plugins/visualizations/charts/static/views/groups.js
+90
View File
@@ -0,0 +1,90 @@
/**
* This class renders the chart configuration form.
*/
define( [ 'mvc/ui/ui-table', 'mvc/ui/ui-misc', 'mvc/form/form-view', 'mvc/form/form-repeat', 'utils/utils' ], function( Table, Ui, Form, Repeat, Utils ) {
var GroupView = Backbone.View.extend({
initialize: function( app, options ) {
var self = this;
this.app = app;
this.chart = app.chart;
this.group = options.group;
this.setElement( $( '<div/>' ) );
this.chart.on( 'change:dataset_id change:type', function() { self.render() } );
this.group.on( 'change', function() { self.form && self.form.data.set( self.group ) } );
this.render();
},
render: function() {
var self = this;
var inputs = this.chart.definition.series && this.chart.definition.series.slice() || [];
_.each( this.chart.definition.columns, function( column, name ) {
inputs.push( { name: name, label: column.title, type: 'select' } );
});
this.form = new Form( {
inputs : inputs,
cls : 'ui-portlet-plain'
} );
this.$el.empty().append( this.form.$el );
var dataset_id = this.chart.get( 'dataset_id' );
var chart_type = this.chart.get( 'type' );
var chart_definition = this.chart.definition;
if ( dataset_id && chart_type ) {
this.chart.state( 'wait', 'Loading metadata...' );
this.app.deferred.execute( function( process ) {
self.app.datasets.request({
id : dataset_id,
success : function( dataset ) {
for ( var id in chart_definition.columns ) {
var index = self.form.data.match( id );
if ( index ) {
var columns = [];
var select = self.form.field_list[ index ];
var input_def = chart_definition.columns[ id ];
input_def.is_auto && columns.push( { 'label': 'Column: Row Number', 'value': 'auto' } );
input_def.is_zero && columns.push( { 'label' : 'Column: None', 'value' : 'zero' } );
var meta = dataset.metadata_column_types;
for ( var key in meta ) {
var valid = ( [ 'int', 'float' ].indexOf( meta[ key ] ) != -1 && input_def.is_numeric ) || input_def.is_text || input_def.is_label;
valid && columns.push( { 'label' : 'Column: ' + ( parseInt( key ) + 1 ), 'value' : key } );
}
select.update( columns );
self.group.set( id, select.value( self.group.get( id ) ) );
}
}
self.chart.state( 'ok', 'Metadata initialized...' );
self.form.data.set( self.group );
self.form.setOnChange( function() {
self.group.set( self.form.data.create() );
self.chart.set( 'modified', true );
});
process.resolve();
}
});
});
}
}
});
return Backbone.View.extend({
initialize: function( app ) {
var repeat = new Repeat.View({
title : 'Data series',
title_new : 'Data series',
min : 1,
onnew : function() { app.chart.groups.add( { id : Utils.uid() } ) }
});
this.setElement( repeat.$el );
app.chart.groups.on( 'add remove reset', function() { app.chart.set( 'modified', true ) } )
.on( 'remove', function( group ) { repeat.del( group.id ) } )
.on( 'reset', function() { repeat.delAll() } )
.on( 'add', function( group ) {
repeat.add({
id : group.id,
cls : 'ui-portlet-panel',
$el : ( new GroupView( app, { group: group } ) ).$el,
ondel : function() { app.chart.groups.remove( group ) }
});
});
}
});
});
config/plugins/visualizations/charts/static/views/settings.js
+71 -49
View File
@@ -1,54 +1,76 @@
// dependencies
define(['mvc/ui/ui-misc', 'plugin/library/ui-table-form', 'utils/utils'],
function(Ui, TableForm, Utils) {
/**
* This class renders the chart configuration form.
*/
return Backbone.View.extend({
// initialize
initialize: function(app, options) {
// link app
this.app = app;
// link this
var self = this;
// get current chart object
this.chart = this.app.chart;
// create settings
this.form = new TableForm.View(app, {
title : 'Configuration',
content : 'This chart type does not provide any options.'
});
// set element
this.setElement(this.form.$el);
// change
var self = this;
this.chart.on('change', function() {
self._refresh();
});
},
// update dataset
_refresh: function() {
// get settings
var chart_definition = this.chart.definition;
// check if chart definition is available
if (!chart_definition) {
return;
}
// set title
this.form.title(chart_definition.category + ' - ' + chart_definition.title + ':');
// update table form model
this.form.update(chart_definition.settings, this.chart.settings);
}
});
define( [ 'mvc/ui/ui-table', 'mvc/ui/ui-misc', 'mvc/form/form-parameters', 'utils/utils' ], function( Table, Ui, Parameters, Utils ) {
return Backbone.View.extend({
initialize: function( app, options ) {
var self = this;
this.app = app;
this.chart = this.app.chart;
this.list = [];
this.parameters = new Parameters();
this.table_title = new Ui.Label({});
this.table = new Table.View( { content: 'This chart type does not provide any options.' } );
this.setElement( $( '<div/>' ).addClass( 'ui-form' )
.append( this.table_title.$el )
.append( this.table.$el ) );
this.chart.on( 'change', function() { self.render() } );
},
render: function() {
var chart_definition = this.chart.definition;
if ( chart_definition ) {
this.table_title.title( chart_definition.category + ' - ' + chart_definition.title + ':' );
this.table.delAll();
this.list = [];
var settings = chart_definition.settings;
for ( var id in settings ) {
this._add( settings[ id ].id || id, settings[ id ], this.chart.settings );
}
for ( var id in this.list ) {
this.list[ id ].trigger('change');
}
}
},
/** Add table row */
_add: function( name, settings_def, model ) {
var self = this;
if ( settings_def.type !== 'separator' ) {
switch( settings_def.type ) {
case 'select':
settings_def.onchange = function( new_value ) {
model.set( name, new_value );
var dict = _.findWhere( settings_def.data, { value: new_value } );
if ( dict && dict.operations ) {
var operations = dict.operations;
for ( var i in operations.show ) {
self.table.get( operations.show[ i ] ).fadeIn( 'fast' );
}
for ( var i in operations.hide ) {
self.table.get( operations.hide[ i ] ).hide();
}
}
};
break;
default:
settings_def.onchange = function( new_value ) { model.set( name, new_value ) };
}
settings_def.id = name;
settings_def.value = model.get( name ) || settings_def.init;
model.set( name, settings_def.value );
var field = this.parameters.create( settings_def );
var $input = $( '<div/>' ).append( field.$el );
settings_def.info && $input.append( $( '<div/>' ).addClass( 'ui-form-info' ).append( settings_def.info ) );
this.table.add( $( '<span/>' ).addClass( 'ui-form-title' ).append( settings_def.title ), '20%' );
this.table.add( $input );
this.list[ name ] = field;
} else {
this.table.add( $( '<div/>' ).addClass( 'ui-form-separator' ).append( settings_def.title + ':' ) );
this.table.add( $( '<div/>' ) );
}
this.table.append( name );
settings_def.hide && this.table.get( name ).hide();
}
});
});
config/plugins/visualizations/charts/static/views/types.js
+131 -193
View File
@@ -1,202 +1,140 @@
// dependencies
define(['utils/utils', 'mvc/ui/ui-misc'], function(Utils, Ui) {
/**
* This class renders the chart type selection grid.
*/
return Backbone.View.extend({
// defaults options
optionsDefault: {
onchange : null,
ondblclick : null
},
// events
events : {
'click' : '_onclick',
'dblclick' : '_ondblclick'
},
// initialize
initialize : function(app, options) {
// link this
var self = this;
// link app
this.app = app;
// configure options
this.options = Utils.merge(options, this.optionsDefault);
// create new element
var $el = $('<div class="charts-grid"/>');
// add label
$el.append((new Ui.Label({ title : 'How many data points would you like to analyze?'})).$el);
// construct chart type subset selection buttons
this.library = new Ui.RadioButton.View({
data : [ { label: 'Few (<500)', value: 'small' },
{ label: 'Some (<10k)', value: 'medium' },
{ label: 'Many (>10k)', value: 'large' }],
onchange: function(value) {
self._filter(value);
define( [ 'utils/utils', 'mvc/ui/ui-misc' ], function( Utils, Ui ) {
return Backbone.View.extend({
optionsDefault: {
onchange : null,
ondblclick : null
},
events : {
'click' : '_onclick',
'dblclick' : '_ondblclick'
},
initialize : function( app, options ) {
var self = this;
this.app = app;
this.options = Utils.merge( options, this.optionsDefault );
this.library = new Ui.RadioButton.View({
data : [ { label: 'Few (<500)', value: 'small' },
{ label: 'Some (<10k)', value: 'medium' },
{ label: 'Many (>10k)', value: 'large' }],
onchange: function( value ) { self._filter( value ) }
});
this.setElement( $( '<div/>' ).addClass( 'charts-grid' )
.append( ( new Ui.Label( { title : 'How many data points would you like to analyze?' } ) ).$el )
.append( this.library.$el.addClass( 'ui-margin-bottom' ) ) );
this._render();
this.library.value( 'small' );
this.library.trigger( 'change' );
},
/** Set/Get selected chart type */
value: function( new_value ) {
var before = this.$el.find( '.current' ).attr( 'id' );
if ( new_value !== undefined ) {
this.$el.find( '.current' ).removeClass( 'current' );
this.$el.find( '#' + new_value ).addClass( 'current' );
}
});
$el.append(this.library.$el.addClass('ui-margin-bottom'));
// set element
this.setElement($el);
// render
this._render();
// set
this.library.value('small');
this.library.trigger('change');
},
// value
value: function(new_value) {
// get current element
var before = this.$el.find('.current').attr('id');
// check if new_value is defined
if (new_value !== undefined) {
// remove current class
this.$el.find('.current').removeClass('current');
// add current class
this.$el.find('#' + new_value).addClass('current');
}
// get current id/value
var after = this.$el.find('.current').attr('id');
if(after === undefined) {
return null;
} else {
// fire onchange
if (after != before && this.options.onchange) {
this.options.onchange(new_value);
}
// return current value
return after;
}
},
// filter
_filter: function(value) {
// hide all category headers
this.$el.find('.header').hide();
// show chart types
var types = this.app.types.attributes;
for (var id in types) {
var type = types[id];
var $el = this.$el.find('#' + id);
var $header = this.$el.find('#types-header-' + this.categories_index[type.category]);
var keywords = type.keywords || '';
if (keywords.indexOf(value) >= 0) {
$el.show();
$header.show();
} else {
$el.hide();
}
}
},
// render
_render: function() {
// load chart types into categories
this.categories = {};
this.categories_index = {};
// counter
var category_index = 0;
// identify categories
var types = this.app.types.attributes;
for (var id in types) {
var type = types[id];
var category = type.category;
if (!this.categories[category]) {
this.categories[category] = {};
this.categories_index[category] = category_index++;
}
this.categories[category][id] = type;
}
// add categories and charts to screen
for (var category in this.categories) {
// create empty element
var $el = $('<div style="clear: both;"/>')
// add header label
$el.append(this._template_header({
id : 'types-header-' + this.categories_index[category],
title : category
}));
// add chart types
for (var id in this.categories[category]) {
// get type
var type = this.categories[category][id];
// make title
var title = type.title + ' (' + type.library + ')';
if (type.zoomable) {
title = '<span class="fa fa-search-plus"/>' + title;
var after = this.$el.find( '.current' ).attr( 'id' );
if( after !== undefined ) {
if ( after != before && this.options.onchange ) {
this.options.onchange( new_value );
}
// append type to screen
$el.append(this._template_item({
id : id,
title : title,
url : config.app_root + 'charts/' + this.app.chartPath(id) + '/logo.png'
}));
return after;
}
// add to view
this.$el.append($el);
}
},
// onclick
_onclick: function(e) {
var old_value = this.value();
var new_value = $(e.target).closest('.item').attr('id');
if (new_value != '') {
if (new_value && old_value != new_value) {
this.value(new_value);
},
/** Filter chart types */
_filter: function( value ) {
this.$el.find( '.header' ).hide();
var types = this.app.types.attributes;
for ( var id in types ) {
var type = types[ id ];
var $el = this.$el.find( '#' + id );
var $header = this.$el.find( '#types-header-' + this.categories_index[ type.category ] );
var keywords = type.keywords || '';
if ( keywords.indexOf( value ) >= 0 ) {
$el.show();
$header.show();
} else {
$el.hide();
}
}
},
/** Render chart type view */
_render: function() {
var category_index = 0;
this.categories = {};
this.categories_index = {};
var types = this.app.types.attributes;
for ( var id in types ) {
var type = types[ id ];
var category = type.category;
if ( !this.categories[ category ] ) {
this.categories[ category ] = {};
this.categories_index[ category ] = category_index++;
}
this.categories[ category ][ id ] = type;
}
// add categories and charts to screen
for (var category in this.categories) {
var $el = $( '<div style="clear: both;"/>' ).append( this._template_header({
id : 'types-header-' + this.categories_index[ category ],
title : category
}));
for ( var id in this.categories[ category ] ) {
var type = this.categories[ category ][ id ];
var title = type.title + ' (' + type.library + ')';
if ( type.zoomable ) {
title = '<span class="fa fa-search-plus"/>' + title;
}
$el.append( this._template_item({
id : id,
title : title,
url : config.app_root + 'charts/' + this.app.chartPath( id ) + '/logo.png'
}));
}
this.$el.append( $el );
}
},
/** Add click handler */
_onclick: function( e ) {
var old_value = this.value();
var new_value = $( e.target ).closest( '.item' ).attr( 'id' );
if ( new_value != '' ) {
if ( new_value && old_value != new_value ) {
this.value( new_value );
}
}
},
/** Add double click handler */
_ondblclick: function( e ) {
var value = this.value();
if ( value && this.options.ondblclick ) {
this.options.ondblclick( value );
}
},
// header template
_template_header: function( options ) {
return '<div id="' + options.id + '" class="header">' +
'&bull; ' + options.title +
'<div>';
},
// item template
_template_item: function( options ) {
return '<div id="' + options.id + '" class="item">' +
'<img class="image" src="' + options.url + '">' +
'<div class="title">' + options.title + '</div>' +
'<div>';
}
},
// ondblclick
_ondblclick: function(e) {
var value = this.value();
if (value && this.options.ondblclick) {
this.options.ondblclick(value);
}
},
// template
_template_header: function(options) {
return '<div id="' + options.id + '" class="header">' +
'&bull; ' + options.title +
'<div>';
},
// template
_template_item: function(options) {
return '<div id="' + options.id + '" class="item">' +
'<img class="image" src="' + options.url + '">' +
'<div class="title">' + options.title + '</div>' +
'<div>';
}
});
});
});
config/plugins/visualizations/charts/static/views/viewer.js
+128 -150
View File
@@ -1,162 +1,140 @@
// dependencies
define(['utils/utils', 'mvc/ui/ui-misc', 'mvc/ui/ui-portlet',
'plugin/views/viewport', 'plugin/library/screenshot'],
function(Utils, Ui, Portlet, ViewportView, Screenshot) {
/**
* This class renders the chart viewer which encapsulates the chart viewport.
*/
return Backbone.View.extend({
// initialize
initialize: function(app, options){
// link app
this.app = app;
// link chart
this.chart = this.app.chart;
// create viewport
this.viewport_view = new ViewportView(app);
// link this
var self = this;
// message element
this.message = new Ui.Message();
// button menu
var picture_button_menu = new Ui.ButtonMenu({
icon : 'fa-camera',
title : 'Screenshot',
tooltip : 'Download as PNG, SVG or PDF file'
});
// add png option
picture_button_menu.addMenu({
id : 'button-png',
title : 'Save as PNG',
icon : 'fa-file',
onclick : function() {
self._wait (self.chart, function() {
Screenshot.createPNG({
$el : self.viewport_view.$el,
title : self.chart.get('title'),
error : function(err) {
self.message.update({ message: err, status: 'danger' });
}
define( [ 'utils/utils', 'mvc/ui/ui-misc', 'mvc/ui/ui-portlet',
'plugin/views/viewport', 'plugin/library/screenshot' ],
function( Utils, Ui, Portlet, ViewportView, Screenshot ) {
return Backbone.View.extend({
initialize: function(app, options){
var self = this;
this.app = app;
this.chart = this.app.chart;
this.viewport_view = new ViewportView( app );
this.message = new Ui.Message();
// button menu
var picture_button_menu = new Ui.ButtonMenu({
icon : 'fa-camera',
title : 'Screenshot',
tooltip : 'Download as PNG, SVG or PDF file'
});
// add png option
picture_button_menu.addMenu({
id : 'button-png',
title : 'Save as PNG',
icon : 'fa-file',
onclick : function() {
self._wait( self.chart, function() {
Screenshot.createPNG({
$el : self.viewport_view.$el,
title : self.chart.get( 'title' ),
error : function( err ) {
self.message.update( { message: err, status: 'danger' } );
}
});
});
});
}
});
// add png option
picture_button_menu.addMenu({
id : 'button-svg',
title : 'Save as SVG',
icon : 'fa-file-text-o',
onclick : function() {
self._wait (self.chart, function() {
Screenshot.createSVG({
$el : self.viewport_view.$el,
title : self.chart.get('title'),
error : function(err) {
self.message.update({ message: err, status: 'danger' });
}
}
});
// add svg option
picture_button_menu.addMenu({
id : 'button-svg',
title : 'Save as SVG',
icon : 'fa-file-text-o',
onclick : function() {
self._wait( self.chart, function() {
Screenshot.createSVG({
$el : self.viewport_view.$el,
title : self.chart.get( 'title' ),
error : function( err ) {
self.message.update( { message: err, status: 'danger' } );
}
});
});
});
}
});
// add png option
picture_button_menu.addMenu({
id : 'button-png',
title : 'Save as PDF',
icon : 'fa-file-o',
onclick : function() {
self.app.modal.show({
title : 'Send chart data for PDF creation',
body : 'Galaxy does not provide integrated PDF export scripts. You may click \'Continue\' to create the PDF by using a 3rd party service (https://export.highcharts.com).',
buttons : {
'Cancel' : function() {self.app.modal.hide()},
'Continue' : function() {
self.app.modal.hide();
self._wait (self.chart, function() {
Screenshot.createPDF({
$el : self.viewport_view.$el,
title : self.chart.get('title'),
error : function(err) {
self.message.update({ message: err, status: 'danger' });
}
}
});
// add pdf option
picture_button_menu.addMenu({
id : 'button-png',
title : 'Save as PDF',
icon : 'fa-file-o',
onclick : function() {
self.app.modal.show({
title : 'Send chart data for PDF creation',
body : 'Galaxy does not provide integrated PDF export scripts. You may click \'Continue\' to create the PDF by using a 3rd party service (https://export.highcharts.com).',
buttons : {
'Cancel' : function() { self.app.modal.hide() },
'Continue' : function() {
self.app.modal.hide();
self._wait( self.chart, function() {
Screenshot.createPDF({
$el : self.viewport_view.$el,
title : self.chart.get( 'title' ),
error : function( err ) {
self.message.update( { message: err, status: 'danger' } );
}
});
});
}
}
});
}
});
// create portlet
this.portlet = new Portlet.View({
icon : 'fa-bar-chart-o',
title: 'Viewport',
operations: {
edit_button: new Ui.ButtonIcon({
icon : 'fa-edit',
tooltip : 'Customize this chart',
title : 'Editor',
onclick : function() {
self._wait( self.chart, function() {
self.app.go( 'editor' );
});
}
}
});
}
});
}),
picture_button_menu: picture_button_menu
}
});
// create portlet
this.portlet = new Portlet.View({
icon : 'fa-bar-chart-o',
title: 'Viewport',
operations: {
edit_button: new Ui.ButtonIcon({
icon : 'fa-edit',
tooltip : 'Customize this chart',
title : 'Editor',
onclick : function() {
self._wait (self.chart, function() {
self.app.go('editor');
});
}
}),
picture_button_menu: picture_button_menu
}
});
// append portlet
this.portlet.append(this.message.$el.addClass('ui-margin-top'));
this.portlet.append(this.viewport_view.$el);
// set element
this.setElement(this.portlet.$el);
// events
var self = this;
this.chart.on('change:title', function() {
self._refreshTitle();
});
},
// append elements
this.portlet.append( this.message.$el.addClass( 'ui-margin-top' ) );
this.portlet.append( this.viewport_view.$el );
this.setElement( this.portlet.$el );
// show
show: function() {
// show element
this.$el.show();
// trigger resize to refresh d3 element
$(window).trigger('resize');
},
// hide
hide: function() {
this.$el.hide();
},
// refresh title
_refreshTitle: function() {
var title = this.chart.get('title');
this.portlet.title(title);
},
// wait for chart to be ready
_wait: function(chart, callback) {
// get chart
if (this.app.deferred.ready()) {
callback();
} else {
this.message.update({message: 'Your chart is currently being processed. Please wait and try again.'});
// events
this.chart.on( 'change:title', function() { self._refreshTitle() } );
},
/** Show and refresh viewer */
show: function() {
this.$el.show();
$( window ).trigger( 'resize' );
},
/** Hide viewer */
hide: function() {
this.$el.hide();
},
/** Change title */
_refreshTitle: function() {
var title = this.chart.get( 'title' );
this.portlet.title( title );
},
/** Check if chart is ready for export */
_wait: function(chart, callback) {
if ( this.app.deferred.ready() ) {
callback();
} else {
this.message.update( { message: 'Your chart is currently being processed. Please wait and try again.' } );
}
}
}
});
});
});
config/plugins/visualizations/charts/static/views/viewport.js
+182 -291
View File
@@ -1,303 +1,194 @@
// dependencies
define(['mvc/ui/ui-portlet', 'mvc/ui/ui-misc', 'utils/utils'],
function(Portlet, Ui, Utils) {
/**
* The viewport creates and manages the dom elements used by the visualization plugins to draw the chart.
* Additionally, this class creates default request strings and request dictionaries parsed to the visualization plugins.
* This is the last class of the charts core classes before handing control over to the visualization plugins.
*/
return Backbone.View.extend({
define( [ 'mvc/ui/ui-portlet', 'mvc/ui/ui-misc', 'utils/utils' ], function( Portlet, Ui, Utils ) {
return Backbone.View.extend({
// list of container/canvas elements
container_list: [],
canvas_list: [],
initialize: function( app, options ) {
var self = this;
this.app = app;
this.chart = this.app.chart;
this.options = options;
this.setElement( $( '<div/>' ).addClass( 'charts-viewport' )
.append( $( '<div/>' ).addClass( 'info' )
.append( $( '<span/>' ).addClass( 'icon' ) )
.append( $( '<span/>' ).addClass( 'text' ) ) ) );
this.$info = this.$( '.info' );
this.$icon = this.$( '.icon' );
this.$text = this.$( '.text' );
// list of canvas elements
container_list: [],
canvas_list: [],
// initialize
initialize: function(app, options) {
// link app
this.app = app;
// link chart
this.chart = this.app.chart;
// link options
this.options = Utils.merge(options, this.optionsDefault);
// create element
this.setElement($(this._template()));
// use full screen for viewer
this._fullscreen(this.$el, 50);
// prevent window scrolling
var initial_overflow = $('body').css('overflow');
this.$el.on('mouseover', function() {
$('body').css('overflow', 'hidden');
}).on('mouseout', function() {
$('body').css('overflow', initial_overflow);
});
// create container element
this._createContainer('div');
// events
var self = this;
this.chart.on('redraw', function() {
self.app.deferred.execute(function(process) {
self._draw(process, self.chart);
// use full screen for viewer
this._fullscreen( this.$el, 50 );
// prevent window scrolling
var initial_overflow = $( 'body' ).css( 'overflow' );
this.$el.on( 'mouseover', function() {
$( 'body' ).css( 'overflow', 'hidden' );
}).on('mouseout', function() {
$( 'body' ).css( 'overflow', initial_overflow );
});
});
// link status handler
this.chart.on('set:state', function() {
// get info element
var $info = self.$el.find('#info');
var $container = self.$el.find('.charts-viewport-container');
// get icon
var $icon = $info.find('#icon');
// remove icon
$icon.removeClass();
// show info
$info.show();
$info.find('#text').html(self.chart.get('state_info'));
// check status
var state = self.chart.get('state');
switch (state) {
case 'ok':
$info.hide();
$container.show();
break;
case 'failed':
$icon.addClass('icon fa fa-warning');
$container.hide();
break;
default:
$icon.addClass('icon fa fa-spinner fa-spin');
$container.show();
}
});
},
// show
show: function() {
this.$el.show();
},
// hide
hide: function() {
this.$el.hide();
},
// resize to fullscreen
_fullscreen: function($el, margin) {
// fix size
$el.css('height', $(window).height() - margin);
// catch window resize event
$(window).resize(function () {
$el.css('height', $(window).height() - margin);
});
},
// creates n canvas elements
_createContainer: function(tag, n) {
// check size of requested canvas elements
n = n || 1;
// clear previous canvas elements
for (var i in this.container_list) {
this.container_list[i].remove();
}
// reset lists
this.container_list = [];
this.canvas_list = [];
// create requested canvas elements
for (var i = 0; i < n; i++) {
// create element
var container_el = $(this._templateContainer(tag, parseInt(100 / n)));
// add to view
this.$el.append(container_el);
// add to list
this.container_list[i] = container_el;
// add a separate list for canvas elements
this.canvas_list[i] = container_el.find('.charts-viewport-canvas').attr('id');
}
},
// add
_draw: function(process, chart) {
// link this
var self = this;
// identify chart type
var chart_type = chart.get('type');
// load chart settings
this.chart_definition = chart.definition;
// determine number of svg/div-elements to create
var n_panels = 1;
if (chart.settings.get('use_panels') === 'true') {
n_panels = chart.groups.length;
}
// create canvas element and add to canvas list
this._createContainer(this.chart_definition.tag, n_panels);
// set chart state
chart.state('wait', 'Please wait...');
// clean up data if there is any from previous jobs
if (!this.chart_definition.execute ||
(this.chart_definition.execute && chart.get('modified'))) {
// reset jobs
this.app.jobs.cleanup(chart);
// reset modified flag
chart.set('modified', false);
}
// create chart view
var self = this;
require(['plugin/charts/' + this.app.chartPath(chart_type) + '/wrapper'], function(ChartView) {
if (self.chart_definition.execute) {
self.app.jobs.request(chart, self._defaultSettingsString(chart), self._defaultRequestString(chart),
function() {
var view = new ChartView(self.app, {
process : process,
chart : chart,
request_dictionary : self._defaultRequestDictionary(chart),
canvas_list : self.canvas_list
});
},
function() {
process.reject();
}
);
} else {
var view = new ChartView(self.app, {
process : process,
chart : chart,
request_dictionary : self._defaultRequestDictionary(chart),
canvas_list : self.canvas_list
// create container element
this._createContainer( 'div' );
// link redraw trigger
var self = this;
this.chart.on( 'redraw', function() {
self.app.deferred.execute( function( process ) {
self._draw( process, self.chart );
});
}
});
},
//
// REQUEST STRING FUNCTIONS
//
// create default chart request
_defaultRequestString: function(chart) {
// configure request
var request_string = '';
// add groups to data request
var group_index = 0;
var self = this;
chart.groups.each(function(group) {
// increase group counter
group_index++;
// add selected columns to column string
for (var key in self.chart_definition.columns) {
request_string += key + '_' + group_index + ':' + (parseInt(group.get(key)) + 1) + ', ';
}
});
// return
return request_string.substring(0, request_string.length - 2);
},
// create default chart request
_defaultSettingsString: function(chart) {
// configure settings
var settings_string = '';
// add settings to settings string
for (key in chart.settings.attributes) {
settings_string += key + ':' + chart.settings.get(key) + ', ';
};
// return
return settings_string.substring(0, settings_string.length - 2);
},
// create default chart request
_defaultRequestDictionary: function(chart) {
// configure request
var request_dictionary = {
groups : []
};
// update request dataset id
if (this.chart_definition.execute) {
request_dictionary.id = chart.get('dataset_id_job');
} else {
request_dictionary.id = chart.get('dataset_id');
}
// add groups to data request
var group_index = 0;
var self = this;
chart.groups.each(function(group) {
// add columns
var columns = {};
for (var column_key in self.chart_definition.columns) {
// get settings for column
var column_settings = self.chart_definition.columns[column_key];
// add to columns
columns[column_key] = Utils.merge ({
index : group.get(column_key)
}, column_settings);
}
// add group data
request_dictionary.groups.push({
key : (++group_index) + ':' + group.get('key'),
columns : columns
});
});
// return
return request_dictionary;
},
// template
_template: function() {
return '<div class="charts-viewport">' +
'<div id="info" class="info">' +
'<span id="icon" class="icon"/>' +
'<span id="text" class="text" />' +
'</div>' +
'</div>';
},
// template svg/div element
_templateContainer: function(tag, width) {
return '<div class="charts-viewport-container" style="width:' + width + '%;">' +
'<div id="menu"/>' +
'<' + tag + ' id="' + Utils.uid() + '" class="charts-viewport-canvas">' +
'</div>';
}
});
// link status handler
this.chart.on( 'set:state', function() {
var $container = self.$el.find( '.charts-viewport-container' );
var $info = self.$info;
var $icon = self.$icon;
var $text = self.$text;
$icon.removeClass();
$info.show();
$text.html( self.chart.get( 'state_info' ) );
var state = self.chart.get( 'state' );
switch ( state ) {
case 'ok':
$info.hide();
$container.show();
break;
case 'failed':
$icon.addClass( 'icon fa fa-warning' );
$container.hide();
break;
default:
$icon.addClass( 'icon fa fa-spinner fa-spin' );
$container.show();
}
});
},
/** Show rendered chart */
show: function() {
this.$el.show();
},
/** Hide chart */
hide: function() {
this.$el.hide();
},
/** Force resize to fullscreen */
_fullscreen: function( $el, margin ) {
$el.css( 'height', $( window ).height() - margin );
$( window ).resize( function() {
$el.css( 'height', $( window ).height() - margin );
});
},
/** A chart may contain multiple sub charts/containers which are created here */
_createContainer: function( tag, n ) {
n = n || 1;
for ( var i in this.container_list ) {
this.container_list[ i ].remove();
}
this.container_list = [];
this.canvas_list = [];
for ( var i = 0; i < n; i++ ) {
var container_id = Utils.uid();
var container_el = $( '<div/>' ).addClass( 'charts-viewport-container' )
.width( parseInt( 100 / n ) + '%' )
.append( $( '<div/>' ).attr( 'id', 'menu' ) )
.append( $( '<' + tag + ' class="charts-viewport-canvas" />' ).attr( 'id', container_id ) );
this.$el.append( container_el );
this.container_list[ i ] = container_el;
this.canvas_list[ i ] = container_id;
}
},
/** Draws a new chart by loading and executing the corresponding chart wrapper */
_draw: function( process, chart ) {
var self = this;
var chart_type = chart.get( 'type' );
this.chart_definition = chart.definition;
var n_panels = 1;
if ( chart.settings.get( 'use_panels' ) === 'true' ) {
n_panels = chart.groups.length;
}
this._createContainer( this.chart_definition.tag, n_panels );
chart.state( 'wait', 'Please wait...' );
if ( !this.chart_definition.execute || ( this.chart_definition.execute && chart.get( 'modified' ) ) ) {
this.app.jobs.cleanup( chart );
chart.set( 'modified', false );
}
require( [ 'plugin/charts/' + this.app.chartPath( chart_type ) + '/wrapper' ], function( ChartView ) {
if ( self.chart_definition.execute ) {
self.app.jobs.request( chart, self._defaultSettingsString( chart ), self._defaultRequestString( chart ),
function() {
var view = new ChartView( self.app, {
process : process,
chart : chart,
request_dictionary : self._defaultRequestDictionary( chart ),
canvas_list : self.canvas_list
});
},
function() {
process.reject();
}
);
} else {
var view = new ChartView( self.app, {
process : process,
chart : chart,
request_dictionary : self._defaultRequestDictionary( chart ),
canvas_list : self.canvas_list
});
}
});
},
/** Creates default chart request */
_defaultRequestString: function( chart ) {
var request_string = '';
var group_index = 0;
var self = this;
chart.groups.each( function( group ) {
group_index++;
for ( var key in self.chart_definition.columns ) {
request_string += key + '_' + group_index + ':' + ( parseInt( group.get( key ) ) + 1 ) + ', ';
}
});
return request_string.substring( 0, request_string.length - 2 );
},
/** Creates default settings string for charts which require a job execution */
_defaultSettingsString: function( chart ) {
var settings_string = '';
for ( key in chart.settings.attributes ) {
settings_string += key + ':' + chart.settings.get( key ) + ', ';
};
return settings_string.substring( 0, settings_string.length - 2 );
},
/** Create default data request dictionary */
_defaultRequestDictionary: function( chart ) {
var request_dictionary = { groups : [] };
request_dictionary.id = this.chart_definition.execute ? chart.get( 'dataset_id_job' ) : chart.get( 'dataset_id' );
var group_index = 0;
var self = this;
chart.groups.each( function( group ) {
var columns = {};
for ( var column_key in self.chart_definition.columns ) {
var column_settings = self.chart_definition.columns[ column_key ];
columns[ column_key ] = Utils.merge( { index : group.get( column_key ) }, column_settings );
}
request_dictionary.groups.push( Utils.merge({
key : ( ++group_index ) + ':' + group.get( 'key' ),
columns : columns
}, group.attributes ));
});
return request_dictionary;
}
});
});
config/plugins/visualizations/graphviz/config/graphviz.xml
+11 -1
View File
@@ -4,7 +4,17 @@
<data_sources>
<data_source>
<model_class>HistoryDatasetAssociation</model_class>
<test type="isinstance" test_attr="datatype" result_type="datatype">data.Text</test>
<test test_attr="ext" result_type="datatype">json</test>
<to_param param_attr="id">dataset_id</to_param>
</data_source>
<data_source>
<model_class>HistoryDatasetAssociation</model_class>
<test test_attr="ext" result_type="datatype">txt</test>
<to_param param_attr="id">dataset_id</to_param>
</data_source>
<data_source>
<model_class>HistoryDatasetAssociation</model_class>
<test type="has_dataprovider" test_attr="datatype">node-edge</test>
<to_param param_attr="id">dataset_id</to_param>
</data_source>
</data_sources>
config/plugins/visualizations/graphviz/static/js/graphVis.js
+126 -105
View File
@@ -1,38 +1,49 @@
var showNodeLabel;
var showEdgeLabel;
var showOut;
var showIn;
var demoNodes = [];
var demoEdges = [];
var shape;
var allcy;
var cy;
var showNodeLabel;
var showEdgeLabel;
var showOut;
var showIn;
var demoNodes = [];
var demoEdges = [];
var shape;
var allcy;
var cy;
// The key to hold down when brush/box selecting: 16 === shift
var BRUSH_KEY = 16;
$(function() {
// remove horizontal scrollbar
$("body").css("overflow-x", "hidden");
$(function() { // on dom ready
$("body").css("overflow-x", "hidden"); //// remove horizontal scrollbar
}); // END on dom ready
// set up a key that, when held down, allows box selection and turns panning off
$( document )
.on( 'keydown', function( ev ){
if( cy && ev.keyCode === BRUSH_KEY ){
cy.panningEnabled( false );
cy.boxSelectionEnabled( true );
}
})
.on( 'keyup', function( ev ){
if( cy && ev.keyCode === BRUSH_KEY ){
cy.panningEnabled( true );
cy.boxSelectionEnabled( false );
}
});
});
// shape of the graph
$("select option:selected").each(function() {
shape = $(this).val();
$("select option:selected").each(function() {
shape = $(this).val();
});
});
///////// create graph
function createGraph(data) {
///////// create graph
function createGraph(data) {
window.graphData = data;
allcy = cytoscape({
headless: true,
});
cy = cytoscape({
container: document.getElementById('cy'),
layout: {
@@ -45,121 +56,131 @@
motionBlur: true,
textureOnViewport: true,
// initially have click and drag pan the viewport
panningEnabled: true,
boxSelectionEnabled: false,
ready: function() {
window.cy = this;
cy.nodes().on("click", function(e) {
showNodeInfo(e.cyTarget);
});
},
style: cytoscape.stylesheet()
.selector('node')
.css({
'content': showNodeLabel,
'text-valign': 'center',
'background-color': '#888888',
'opacity': 0.8
'content': showNodeLabel,
'text-valign': 'center',
'background-color': '#888888',
'opacity': 0.8
})
})
.selector('edge')
.css({
'curve-style': 'unbundled-bezier',
'target-arrow-shape': 'triangle',
'width': 4,
'line-color': '#ddd',
'target-arrow-color': '#ddd',
'content': showEdgeLabel
'curve-style': 'unbundled-bezier',
'target-arrow-shape': 'triangle',
'width': 2,
'line-color': '#ddd',
'target-arrow-color': '#ddd',
'content': showEdgeLabel
})
})
.selector(':selected')
.css({
'background-color': '#FE2E64',
'line-color': '#FE2E64',
'target-arrow-color': '#FE2E64',
'source-arrow-color': '#FE2E64',
'opacity': 1
})
.selector('core')
.css({
'outside-texture-bg-color': 'white'
'background-color': '#FE2E64',
'line-color': '#FE2E64',
'target-arrow-color': '#FE2E64',
'source-arrow-color': '#FE2E64',
'opacity': 1
})
})
.selector('core')
.css({
'outside-texture-bg-color': 'white'
})
});
window.cy = cy;
allcy.load(data);
// if graph contains more than 50 nodes load only root nodes
if (allcy.nodes().length > 50 && allcy.nodes().roots().length != 0 ) {
var toAdd = allcy.nodes().roots().closedNeighborhood();
allcy.nodes().roots().addClass("roots");
showNodesToExpand(toAdd);
cy.add(toAdd);
cy.load(cy.elements('*').jsons());
// if graph contains more than 50 nodes load only root nodes
if (allcy.nodes().length > 50 && allcy.nodes().roots().length !== 0 ) {
var toAdd = allcy.nodes().roots().closedNeighborhood();
allcy.nodes().roots().addClass("roots");
showNodesToExpand(toAdd);
cy.add(toAdd);
cy.load(cy.elements('*').jsons());
cy.style()
.selector('.toBeExpaned')
.css({
'width': 50,
'height': 50
})
.update();
'width': 50,
'height': 50
})
.update();
} else {
cy.add(data);
cy.load(cy.elements('*').jsons());
cy.add(data);
cy.load(cy.elements('*').jsons());
}
cy.boxSelectionEnabled(true);
checkBoxes();
} // END create graph
} // END create graph
//// parse function for matrix data
function parseAndCreate(demoNodes, demoEdges) {
var data = {
nodes: demoNodes,
edges: demoEdges
};
createGraph(data);
}
//// parse function for matrix data
function parseTextMatrix(data) {
var nodes = [],
edges = [];
data.split( '\n' ).forEach( function( line ){
var columns = line.split( /,?\s+/ ),
sourceId = columns[0]; // split by comma or space
///// parsing function for json: for link/egde and "" issues
function parseJson(dataToParse) {
nodes.push({
data: { id: sourceId }
});
if (dataToParse.links == null) {
for( var i = 1; i < columns.length; i++ ){
edges.push({
data: {
source : sourceId,
target : columns[i]
}
});
}
});
var links = dataToParse.edges;
} else {
var links = dataToParse.links;
dataToParse.edges = links;
createGraph({
nodes: nodes,
edges: edges
});
}
}
///// parsing function for json: for link/egde and "" issues
function parseJson( data ) {
var nodes = dataToParse.nodes;
for (var i = 0; i < nodes.length; i++) {
dataToParse.nodes[i].data = nodes[i];
dataToParse.nodes[i].data.id = '"' + nodes[i].id + '"';
}
for (var i = 0; i < links.length; i++) {
dataToParse.edges[i].data = links[i];
dataToParse.edges[i].data.source = '"' + links[i].source + '"';
dataToParse.edges[i].data.target = '"' + links[i].target + '"';
}
createGraph(dataToParse);
}
if( data.hasOwnProperty( 'links' ) ){
data.edges = data.links;
delete data.links;
}
data.nodes = data.nodes.map( function _processNode( node ){
return {
data : $.extend( {}, node.data, {
id : node.id + ''
})
};
});
data.edges = data.edges.map( function _processEdge( edge ){
return {
data : $.extend( {}, edge.data, {
id : edge.id || undefined,
source : data.nodes[ edge.source ].data.id,
target : data.nodes[ edge.target ].data.id
})
};
});
createGraph( data );
}
config/plugins/visualizations/graphviz/static/js/toolPanelFunctions.js
+31 -53
View File
@@ -34,7 +34,7 @@ $(function() { // on dom ready
});
//////// change the shape of the graph
//////// change the shape of the graph
$('#selectShape').change(function() {
$("select option:selected").each(function() {
@@ -54,11 +54,11 @@ $(function() { // on dom ready
if ($('#showOutNode').is(":checked")) {
showOut = true;
cy.nodes().on("tapend", highlightOut);
} else {
showOut = false;
cy.nodes().off("tapend", highlightOut);
resetHighlightOut(showIn, showOut);
}
});
@@ -71,13 +71,13 @@ $(function() { // on dom ready
if ($('#showInNode').is(":checked")) {
showIn = true;
cy.nodes().on("tapend", highlightIn);
} else {
showIn = false;
cy.nodes().off("tapend", highlightIn);
resetHighlightIn(showIn, showOut);
cy.nodes().removeClass('connectedNodeIn');
}
@@ -101,12 +101,12 @@ $(function() { // on dom ready
});
//// disable/unable buttons
//// disable/unable buttons
$(document).on('click', function() {
var selectedNode = cy.nodes(':selected');
if (selectedNode.outgoers().length == 0) {
if (selectedNode.outgoers().length === 0) {
$('.btn.colNode').prop('disabled', true);
@@ -149,7 +149,7 @@ $(function() { // on dom ready
var selectedNode = cy.nodes(':selected');
if (e.which == 101 && selectedNode.size() != 0) { // 101 for 'e' = expand
if (e.which === 101 && selectedNode.size() !== 0) { // 101 for 'e' = expand
if (selectedNode.hasClass('toBeExpaned')) {
@@ -162,20 +162,20 @@ $(function() { // on dom ready
}
if (e.which == 99 && selectedNode.size() != 0 && !selectedNode.hasClass('superNode') && selectedNode.outgoers().length != 0) { // 99 for 'c' = collapse
if (e.which === 99 && selectedNode.size() !== 0 && !selectedNode.hasClass('superNode') && selectedNode.outgoers().length !== 0) { // 99 for 'c' = collapse
colNode();
}
if (e.which == 100 && selectedNode.size() != 0) { // 100 for 'd' = delete
if (e.which === 100 && selectedNode.size() !== 0) { // 100 for 'd' = delete
deleteSelectedNodes();
}
});
/////////////// tool Panel movement
/////////////// tool Panel movement
$(document).on('click', '.slider-arrow.show', function() {
$(".slider-arrow, .panel").animate({
@@ -234,13 +234,9 @@ $(function() { // on dom ready
// document.getElementById('cy').style.width="100%";
cy.resize();
});
}); // on dom ready END
}) // on dom ready END
///////// finding outgoing Nodes
///////// finding outgoing Nodes
function highlightOut() {
var selectedNode = this;
@@ -259,7 +255,7 @@ function highlightOut() {
})
.update();
} else {
cy.nodes().not(this).removeClass('connectedNodeOut');
cy.nodes().not(this).removeClass('selectedNodeOut');
cy.edges().not(this).removeClass('connectedNodeOut');
@@ -290,7 +286,7 @@ function highlightOut() {
}
}
/////// reset highlighting of outgoing nodes
/////// reset highlighting of outgoing nodes
function resetHighlightOut(showIn, showOut) {
cy.style()
@@ -370,7 +366,7 @@ function highlightIn() {
}
/////// reset highlighting of incoming nodes
/////// reset highlighting of incoming nodes
function resetHighlightIn(showIn, showOut) {
cy.style()
@@ -401,7 +397,7 @@ function resetHighlightIn(showIn, showOut) {
}
////// export PNG
////// export PNG
function exportFunction() {
var pngPic = cy.png();
@@ -418,7 +414,6 @@ function downloadURI(uri, fname) {
link.click();
// Cleanup the DOM
document.body.removeChild(link);
delete link;
}
///// restore Graph structure
@@ -449,7 +444,6 @@ function showNodeInfo(node) {
var j = 0;
for (var key in nodeContent) {
if (key == 'group' || key == 'data') {
continue;
}
@@ -458,10 +452,8 @@ function showNodeInfo(node) {
i++;
values[j] = nodeContent[key];
j++;
}
var childNodes = node.outgoers().nodes();
var childNum = childNodes.length;
@@ -472,16 +464,15 @@ function showNodeInfo(node) {
var table = createTable(fieldName, values, degree, parentNum, childNum);
$('#nodeInfoDiv').html("<p><strong>Node Description </p>" + table);
for (var i = 0; i < fieldName.length; i++) {
if (isValidUrl(values[i])) {
for (var strIndex = 0; strIndex < fieldName.length; strIndex++) {
if (isValidUrl(values[strIndex])) {
var e = document.createElement("div");
e.id = "linkDiv";
$('#nodeInfoDiv').append(e);
var url = values[i];
document.getElementById("linkDiv").innerHTML += "<br>" + fieldName[i] + "<br>";
$('<iframe id="iframeId" width = "240"/>').appendTo(document.getElementById("linkDiv")).prop('src', url);
@@ -490,40 +481,27 @@ function showNodeInfo(node) {
}
}
function isValidUrl(str) {
var pattern = new RegExp(/((([A-Za-z]{3,9}:(?:\/\/)?)(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+|(?:www\.|[\-;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)((?:\/[\+~%\/\.\w\-_]*)?\??(?:[\-\+=&;%@\.\w_]*)#?(?:[\.\!\/\\\w]*))?)/); // fragment locater
if (!pattern.test(str)) {
return false;
} else {
return true;
}
}
function createTable(fieldName, values, degree, parentNum, childNum) {
var table = "<table class='CSSTableGenerator'>";
for (var i = 0; i < fieldName.length; i++) {
if (isValidUrl(values[i])) {
i++;
}
table += "<tr>" + "<td>" + fieldName[i] + "</td>" + "<td>" + values[i] + "</td>" + "</tr>";
}
table += "<tr>" + "<td>" + "degree" + "</td>" + "<td>" + degree + "</td>" + "</tr>";
table += "<tr>" + "<td>" + "number of child nodes" + "</td>" + "<td>" + childNum + "</td>" + "</tr>";
table += "<tr>" + "<td>" + "number of parent nodes" + "</td>" + "<td>" + parentNum + "</td>" + "</tr>";
table += "</table>";
return table;
}
function isValidUrl(str) {
var pattern = new RegExp(/((([A-Za-z]{3,9}:(?:\/\/)?)(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+|(?:www\.|[\-;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)((?:\/[\+~%\/\.\w\-_]*)?\??(?:[\-\+=&;%@\.\w_]*)#?(?:[\.\!\/\\\w]*))?)/); // fragment locater
return pattern.test(str);
}
/////// delete selected nodes
@@ -539,7 +517,7 @@ function deleteSelectedNodes() {
cy.remove(edgesToRemove);
}
/////// restore Deleted Nodes
/////// restore Deleted Nodes
function restoreDeletedNodes() {
nodesToRemove.restore();
edgesToRemove.restore();
@@ -550,7 +528,7 @@ function expandNodes(selectedNode) {
cy.nodes().unbind("tapend");
var selectedNodeId = selectedNode.id();
selectedNodeId = selectedNodeId.replace(/[^0-9\.]+/g, ""); // removing ""
selectedNodeId = selectedNodeId.replace(/[^0-9\.]+/g, ""); // removing ""
var eles = allcy.nodes();
@@ -559,7 +537,7 @@ function expandNodes(selectedNode) {
showNodesToExpand(nodesToAdd);
cy.add(eles[selectedNodeId].outgoers());
selectedNode.removeClass('toBeExpaned');
selectedNode.removeClass('toBeExpaned');
$("select option:selected").each(function() {
shape = $(this).val();
@@ -583,7 +561,7 @@ function showNodesToExpand(toAdd) {
if (ele.outdegree() > 0 && !ele.hasClass('roots')) {
ele.addClass('toBeExpaned');
}
}
});
}
@@ -604,7 +582,7 @@ function checkBoxes() {
resetHighlightIn(showIn, showOut);
cy.nodes().removeClass('connectedNodeIn');
}
if ($('#showOutNode').is(":checked")) {
showOut = true;
cy.nodes().on("tapend", highlightOut);
config/plugins/visualizations/graphviz/templates/graphviz.mako
+59 -57
View File
@@ -6,12 +6,14 @@
<meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
<title>${hda.name} | ${visualization_name}</title>
<%
root = h.url_for( '/' )
%>
<script type="text/javascript" src="/static/scripts/libs/jquery/jquery.js"></script>
<%
root = h.url_for( '/' )
%>
<script type="text/javascript" src="/static/scripts/libs/jquery/jquery.js"></script>
${h.stylesheet_link( root + 'plugins/visualizations/graphviz/static/css/style.css' )}
${h.javascript_link( root + 'plugins/visualizations/graphviz/static/js/jquery.qtip.js' )}
${h.javascript_link( root + 'plugins/visualizations/graphviz/static/js/cytoscape.min.js' )}
${h.javascript_link( root + 'plugins/visualizations/graphviz/static/js/collapse.js' )}
@@ -25,63 +27,62 @@
${h.javascript_link( root + 'plugins/visualizations/graphviz/static/js/wz_tooltip.js' )}
<script>
function parseNodeEdge( data ){
data = data.data[0];
parseJson( data );
}
$(document).ready(function() {
var hdaId = '${trans.security.encode_id( hda.id )}',
hdaExt = '${hda.ext}',
dataUrl = "${h.url_for( controller='/datasets', action='index')}/" + hdaId + "/display?to_ext=" + hdaExt;
// var reader = new FileReader();
$.ajax({
'async': true,
'global': false,
'url': dataUrl,
'dataType': "json",
'success': function(data) {
parseJson(data);
// createGraph(data);
},
'error': function(data) {
jQuery.get(dataUrl, function(data) {
var lines = data.split('\n');
var chars;
for (var line = 0; line < lines.length; line++) {
chars = lines[line].split(/,?\s+/); // split by comma or space
var hdaName = '${ hda.name | h }',
hdaId = '${trans.security.encode_id( hda.id )}',
hdaExt = '${hda.ext}',
rawUrl = '${h.url_for( controller="/datasets", action="index" )}',
apiUrl = '${h.url_for( "/" ) + "api/datasets"}',
dataUrl;
demoNodes.push({
data: {
id: chars[0],
label: chars[0]
}
});
function errorHandler( xhr, status, message ){
console.error(x, s, m);
alert("error loading data:\n" + m);
}
for (var i = 1; i < chars.length; i++) {
demoEdges.push({
data: {
source: chars[0],
target: chars[i],
id: chars[0] + chars[i],
switch( hdaExt ){
case 'txt':
dataUrl = rawUrl + '/' + hdaId + '/display?to_ext=txt';
$.ajax(dataUrl, {
dataType : 'text',
success : parseTextMatrix,
error : errorHandler
});
break;
}
})
}
}
parseAndCreate(demoNodes, demoEdges);
});
}
});
case 'json':
dataUrl = rawUrl + '/' + hdaId + '/display?to_ext=json';
$.ajax(dataUrl, {
dataType : 'json',
success : parseJson,
error : errorHandler
});
break;
} );
default:
dataUrl = apiUrl + '/' + hdaId;
$.ajax(dataUrl, {
dataType : 'json',
success : parseNodeEdge,
error : errorHandler,
data : {
data_type : 'raw_data',
provider : 'node-edge'
}
});
}
});
</script>
<div id="cy"></div>
<!-- left control panel for rendering controls, hiding nodes, etc. - initially hidden -->
<div class="panel">
<br>
<div id="mainselection">
@@ -139,12 +140,10 @@
<br>
<br>
<input type="button" class="btn delNode" name="deleteNodes" id="deleteNodes" value="Delete Selected Nodes " onclick="deleteSelectedNodes()" disabled="disabled">
<br>
<br>
<input type="button" class="btn" name="restoreNodes" id="restoreNodes" value="Restore Deleted Nodes" onclick="restoreDeletedNodes()">
<br>
<br>
@@ -158,13 +157,16 @@
<br>
<br>
</div>
<!-- button to show above panel -->
<a href="javascript:void(0);" class="slider-arrow show">&raquo;</a>
<div id="nodeInfoDiv" class="nodePanel">
<!-- right control panel for displaying node data - initially hidden -->
<div id="nodeInfoDiv" class="nodePanel">
<p> <strong>Node Description </strong></p>
<br><br>
<p> Please select a node </p>
</div>
<!-- button to show above panel -->
<a href="javascript:void(0);" class="slider-arrow-forNode show">&laquo;</a>
</body>
config/tool_destinations.yml.sample
+311
View File
@@ -0,0 +1,311 @@
# Dynamic Tool Destinations config has four main levels (or key properties) - tools,
# default_destination, users, and verbose.
#
# Under tools is where you put the IDs of the tools. Tool IDs must be
# distinct. In each tool is a list of rules, each of which contain rule-specific
# paremeters. Dynamic Tool Destination is set up to allow and fix a few light errors
# in the config, but it's best to follow the specified template.
#
# default_destination, the second key property, specifies which global destination
# to default to in case none of the rules apply.
#
# users, the third key property, can be used to assign a priority to a user. Three priorities
# are available low, med, and high. By default all users are assumed to be med priority. If
# a rule has priority destinations the user will be set to the destination which matches their priority
# or if their priority isn't available they will be matched to the next closest one.
#
# A fourth key property is verbose. When this is set to True, Dynamic Tool Destination
# gives much more descriptive output regarding the steps it's taking in mapping your tools to
# the appropriate destinations, including config validation and any potential errors found in the
# config.
#
# Note: Configurations can be validated with the following command:
#
# python lib/galaxy/jobs/dynamic_tool_destination.py -c
#
# or
#
# python lib/galaxy/jobs/dynamic_tool_destination.py -c /path/to/tool_destinations.yml
#
# The general template is as follows (note that this template
# does not use quote symbols), using spades and smalt as an example
# (spades for showing what each field is for, and smalt
# to give a fairly real-world example):
#
# Ex:
#
# tools:
# spades:
# rules:
# - rule_type: what kind of rule is it?
# nice_value: what kind of priority does this rule have over others?
# destination: how should this tool be run?
# lower_bound: what's the max file size?
# upper_bound: what's the minimum file size?
# users: #(optional) which users does this rule apply to
# - user@example.com
# default_destination: this tool-specific field is optional
# smalt_map:
# rules:
# - rule_type: file_size
# nice_value: 0
# lower_bound: 0
# upper_bound: 2 GB
# destination: cluster_low_4
# - rule_type: file_size
# nice_value: 0
# lower_bound: 2 GB
# upper_bound: 4 GB
# destination: cluster_low_8
# - rule_type: file_size
# nice_value: 0
# lower_bound: 4 GB
# upper_bound: Infinity
# destination: cluster_low_16
# default_destination: cluster_default
# users:
# user_low@example.com:
# priority: low
# user_med@example.com:
# priority: med
# user_high@example.com:
# priority: high
# default_destination: this global field is mandatory
# verbose: True
#
#
#
# Looking at this example, some things must be clarified: each entry in the list of
# rules per tool is specified by '-'. Per rule, regardless of rule type,
# the following fields are mandatory:
# rule_type, nice_value, and destination.
#
# Some of the other fields are mandatory only for specific rule types, which will be
# further discussed below.
#
# Starting with rule_type, there are currently 3 rule types: file_size, records,
# and arguments.
#
# file_size and records rules are based on how large the files are: if they fall
# within specified limits, then the rule is satisfied, and the tool may proceed
# with the appropriate destination.
#
# file_size and records rules have the following required parameters on top of the base
# mandatory parameters:
# upper_bound
# lower_bound
#
# Bounds are allowed to be specified in bytes (48000 for example) or a higher size unit,
# including the unit abbreviation (4 GB or 10 TB for example). Additionally, upper_bound
# is allowed to be Infinite; simply specify Infinite in order to do so.
#
# **The rule will allow the lower_bound, up to but not including the upper_bound
#
# The third rule_type is arguments, which has arguments as a mandatory parameter ontop of
# the base mandatory parameters. The arguments parameter is specified using the following
# template:
#
#
# arguments:
# argument_name: the_argument
#
#
# A real world example is shown below:
#
#
# tools:
# spades:
# rules:
# - rule_type: arguments
# nice_value: -19
# destination: fail
# fail_message: Don't do that
# arguments:
# careful: true
# default_destination: cluster_low
# default_destination: cluster
# verbose: False
#
#
# Next up, nice_value is used for prioritizing rules over others in case two rules
# match. nice_value basically translates to, "the higher the nice_value, the 'nicer'
# the tool is about being picked last". So based off of that idea, a rule with a nice
# value of -5 is guaranteed to be picked over a rule with a nice value of 10. nice_value
# is allowed to go from -20 to 20. If two rules have the same nice value and both were
# satisfied, the first rule in the config file will be picked. In summary, first-come-
# first-serve basis unless nice_value overrides that.
#
#
# Finally, destination simply refers to the specific way the tool will run. Each
# destination ID refers to a specific configuration to run the tool with.
#
# Some rules may call for the job to fail if certain conditions are encountered. In
# this case, destination simply refers to 'fail'.
#
# For example, the following rule is set to fail the job if a file that is too large
# (more than 4GB) is encountered:
#
#
# tools:
# spades:
# rules:
# - rule_type: file_size
# nice_value: 0
# destination: fail
# fail_message: Data too large
# lower_bound: 4 GB
# upper_bound: Infinity
#
#
# As shown above, a rule with 'fail' as the destination requires an additional
# parameter, 'fail_message', which DynamicToolDestination uses to print a helpful error
# message to the user indicating why the job failed (showing up inside the job log in
# Galaxy's history panel).
tools:
spades:
rules:
- rule_type: file_size
nice_value: 0
lower_bound: 0
upper_bound: 10 MB
fail_message: Too few reads for spades to work
destination: fail
- rule_type: file_size
nice_value: 0
lower_bound: 10 MB
upper_bound: 2 GB
destination:
priority:
low: slurm_low_24
med: slurm_med_24
high: slurm_high_24
- rule_type: file_size
nice_value: 0
lower_bound: 2 GB
upper_bound: 4 GB
destination:
priority:
low: slurm_low_48
med: slurm_med_48
high: slurm_high_48
- rule_type: file_size
nice_value: 0
lower_bound: 4 GB
upper_bound: Infinity
fail_message: Too much data, shoudn't run
destination: fail
- rule_type: arguments
nice_value: 0
arguments:
careful: true
destination:
priority:
low: slurm_low_48
med: slurm_med_48
high: slurm_high_48
default_destination:
priority:
low: slurm_low_16
med: slurm_med_16
high: slurm_high_16
srst2:
rules:
- rule_type: records
nice_value: 0
lower_bound: 0
upper_bound: 100
destination:
priority:
low: slurm_low_4
med: slurm_med_4
high: slurm_high_4
- rule_type: records
nice_value: 0
lower_bound: 100
upper_bound: 1000
destination:
priority:
low: slurm_low_8
med: slurm_med_8
high: slurm_high_8
- rule_type: records
nice_value: 0
lower_bound: 1000
upper_bound: 10000
destination:
priority:
low: slurm_low_8
med: slurm_med_8
high: slurm_high_8
- rule_type: records
nice_value: 0
lower_bound: 1000000
upper_bound: Infinity
fail_message: Using the wrong tool
destination: fail
smalt_map:
rules:
- rule_type: file_size
nice_value: 0
lower_bound: 0
upper_bound: 2 GB
destination:
priority:
low: slurm_low_4
med: slurm_med_4
high: slurm_high_4
- rule_type: file_size
nice_value: 0
lower_bound: 2 GB
upper_bound: 4 GB
destination:
priority:
low: slurm_low_8
med: slurm_med_8
high: slurm_high_8
- rule_type: file_size
nice_value: 0
lower_bound: 4 GB
upper_bound: Infinity
destination:
priority:
low: slurm_low_16
med: slurm_med_16
high: slurm_high_16
fastqc:
rules:
- rule_type: file_size
nice_value: 0
lower_bound: 0
upper_bound: 400 MB
fail_message: File size too small
destination: fail
- rule_type: file_size
nice_value: 0
lower_bound: 400 MB
upper_bound: Infinity
destination:
priority:
low: slurm_low_4
med: slurm_med_4
high: slurm_high_4
default_destination:
priority:
low: slurm_low_16
med: slurm_med_16
high: slurm_high_16
default_destination:
priority:
low: slurm_low_default
med: slurm_med_default
high: slurm_high_default
users:
user@email.com:
priority: high
verbose: True
doc/source/admin/grt.rst
+41
View File
@@ -0,0 +1,41 @@
Galactic Radio Telescope
========================
This is an opt-in service which Galaxy admins can configure to contribute their
job run data back to the community. We hope that by collecting this information
we can build accurate models of tool CPU/memory/time requirements. In turn,
admins will be able to use this analyzed data to optimize their job
distribution across highly heterogenous clusters.
Registration
------------
You will need to register your Galaxy instance with the Galactic Radio
Telescope (GRT). This can be done `https://radio-telescope.galaxyproject.org
<https://radio-telescope.galaxyproject.org>`__.
Submitting Data
---------------
Once you've registered your Galaxy instance, you'll receive an instance ID and
an API key which are used to run ``scripts/grt.py``. The tool itself is very simple
to run. It collects the last 7 days (by default) of data from your Galaxy
server, and sends them to the GRT for processing and display. Additionally
it collects the total number of users, and the number of users who ran
jobs in the last N days.
Running the tool is simple:
.. code-block:: shell
python scripts/grt.py \
<INSTANCE_UUID> \
<API_KEY> \
-c config/galaxy.ini \
--grt-url https://radio-telescope.galaxyproject.org/api/v1/upload/
--days 7
The only required parameters are the instance ID and API key. As you can see in
the example command, the GRT URL is configurable. If you do not wish to
participate in the public version of this experiment you can host your own
radio telescope to collect Galactic information.
doc/source/admin/index.rst
+2
View File
@@ -15,3 +15,5 @@ documentation. These resources should be used together.
framework_dependencies.rst
useful_scripts.rst
grt.rst
doc/source/admin/interactive_environments.rst
+63 -516
View File
@@ -3,9 +3,10 @@ Galaxy Interactive Environments (GIEs)
GIEs were a new feature back in Galaxy 15.05, leading with the release of the
IPython IE. They were presented at GCC2015, and the RStudio IE was released as
part of 15.07.
part of 15.07. The IPython IE was superceded by a Project Jupyter IE in
Galaxy XXXXX.
A GIE is a docker container, launched by Galaxy, proxied by Galaxy, with some
A GIE is a Docker container, launched by Galaxy, proxied by Galaxy, with some
extra sugar inside the container to allow users to interact easily with their
Galaxy histories.
@@ -13,11 +14,12 @@ How GIEs Work
-------------
A GIE is primarily composed of a Docker container, and the Galaxy visualization
component. GIE's are essentially visualization plugins which abuse the fact
that they're rendered with mako templates. The mako templating language used in
Viz plugins allows running python code, which in turn allows for us to launch
docker containers. Once this container is launched, we notify a proxy built
into Galaxy which helps coordinate a 1:1 mapping of users and their docker containers.
component. Galaxy vizualisation plugins are rendered using Mako templates and
Mako templates in turn can run Python code. GIEs build upon visualization plugins,
adding features to allow for container management and proxying. This Python code
in the Mako templates is used to launch the Docker container within which a GIE
runs. Once this container is launched, we notify a proxy built into Galaxy which
helps coordinate a 1:1 mapping of users and their docker containers.
Here's a simple diagram recapping the above:
@@ -74,9 +76,10 @@ Running ``node lib/main.js --help`` should produce some useful help text
--verbose
There are two ways to handle actually running the proxy. The first is to have
Galaxy automatically launch the proxy as needed, the second is to manage it
with something like Supervisord. The command for launching the proxy manually
(or via supervisord) is
Galaxy automatically launches the proxy as needed. This is the default configuration
as of 2014. Alternately, the proxy can be stated manually or via a system such as
Supervisord. Assuming that the ``$GALAXY_ROOT`` environment variable refers to the location of
the Galaxy installation, the command for launching the proxy is:
.. code-block:: console
@@ -84,10 +87,28 @@ with something like Supervisord. The command for launching the proxy manually
--port 8800 --sessions $GALAXY_ROOT/database/session_map.sqlite \
--cookie galaxysession --verbose
And this can be configured in your supervisord config by adding:
.. code-block:: console
[program:galaxy_nodejs_proxy]
directory = GALAXY_ROOT
command = GALAXY_ROOT/lib/galaxy/web/proxy/js/lib/main.js --sessions database/session_map.sqlite --ip 0.0.0.0 --port 8800
autostart = true
autorestart = unexpected
user = GALAXY_USER
startsecs = 5
redirect_stderr = true
where ``GALAXY_ROOT`` is the location of your Galaxy installation and ``GALAXY_USER`` is the username of the user that
Galaxy runs as.
Configuring the Proxy
^^^^^^^^^^^^^^^^^^^^^
Configuration is all managed in ``galaxy.ini``
Configuration is all managed in ``galaxy.ini``. The default arguments used
for the proxy are:
.. code-block:: ini
@@ -98,42 +119,46 @@ Configuration is all managed in ``galaxy.ini``
dynamic_proxy_debug=True
As you can see most of these variables map directly to the command line
arguments to the NodeJS script. There are a few extra parameters which merit
individual discussion
arguments to the NodeJS script. There are a few extra parameters which will
be needed if you run Galaxy behind an upstream proxy like nginx or
Apache:
.. code-block:: ini
dynamic_proxy_external_proxy=True
dynamic_proxy_prefix=gie_proxy
The first option says that you have Galaxy and the Galaxy NodeJS proxy wrapped
in an upstream proxy like Apache or NGINX. If you're using an upstream proxy, then
you'll need to set this to true. This will cause Galaxy to connect users to the
same port as Galaxy is being served on (so 80/443), rather than directing them
to ``:8800``.
in an upstream proxy like Apache or NGINX. This will cause Galaxy to connect
users to the same port as Galaxy is being served on (so 80/443), rather than
directing them to port 8800.
The second option is closely entertwined with the first option. When Galaxy is
accessed, it sets a cookie. This cookie generally cannot be sent with requests
accessed, it sets a cookie called ``galaxysession``. This cookie generally cannot be sent with requests
to different domains and different ports, so Galaxy and the dynamic proxy must
be accessible on the same port and protocol. A further restriction is the
cookie path; if you're running Galaxy under a URL like
``https://f.q.d.n/galaxy/``, the cookie is only accessible to URLs that look
like ``https://f.q.d.n/galaxy/*``. Galaxy and the Galaxy NodeJS proxy will take
care of most of this for you, but you should be aware of how it functions. If
your ``galaxysession`` cookie is not available in your request to the proxy
URL, it will through up an error and you won't be able to connect. Back on
topic, specifically, the second option sets the URL path that's used to
be accessible on the same port and protocol. In addition, the cookie is only
accessible to URLs that share the same prefix as the Galaxy URL. For example,
if you're running Galaxy under a URL like ``https://f.q.d.n/galaxy/``, the cookie
is only accessible to URLs that look like ``https://f.q.d.n/galaxy/*``. The
second (``dynamic_proxy_prefix``) option sets the URL path that's used to
differentiate requests that should go through the proxy to those that should go
to Galaxy. You will need to add special upstream proxy configuration to handle
this, and you'll need to use the same ``dynamic_proxy_prefix`` in your
``galaxy.ini`` that you use in your URL routes.
In the examples below, we assume that your Galaxy installation is available
at a URL such as ``https://f.q.d.n/galaxy``. If instead it is available at a
URL like ``https://f.q.d.n``, remove the ``/galaxy`` prefix from the examples.
For example ``/galaxy/gie_proxy`` would become ``/gie_proxy``. Remember that
``gie_proxy`` is the value you use for the ``dynamic_proxy_prefix`` option. If
you use a different value in that option you should change the examples
accordingly.
**Apache**
.. code-block:: apache
# IPython specific. Other IEs may require their own routes.
ProxyPass /galaxy/gie_proxy/ipython/api/kernels ws://localhost:8800/galaxy/gie_proxy/ipython/api/kernels
# Project Jupyter specific. Other IEs may require their own routes.
ProxyPass /galaxy/gie_proxy/jupyter/ipython/api/kernels ws://localhost:8800/galaxy/gie_proxy/jupyter/ipython/api/kernels
# Global GIE configuration
ProxyPass /galaxy/gie_proxy http://localhost:8800/galaxy/gie_proxy
@@ -155,9 +180,9 @@ Please note you will need to be using apache2.4 with ``mod_proxy_wstunnel``.
proxy_redirect off;
}
# IPython specific. Other IEs may require their own routes.
location /galaxy/gie_proxy/ipython/api/kernels {
proxy_pass http://localhost:8800/galaxy/gie_proxy/ipython/api/kernels;
# Project Jupyter / IPython specific. Other IEs may require their own routes.
location ~ ^/galaxy/gie_proxy/jupyter/(?<nbtype>[^/]+)/api/kernels(?<rest>.*?)$ {
proxy_pass http://localhost:8800/galaxy/gie_proxy/$nbtype/api/kernels$rest;
proxy_redirect off;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
@@ -176,13 +201,15 @@ proxying to GIE and other visualization plugin static content.
Docker on Another Host
^^^^^^^^^^^^^^^^^^^^^^
There are many reasons to run Interactive Environments on a separate host and
not on your webserver, serving Galaxy. This feature has been available since
15.07 and is used in production at the University of Freiburg.
You might want to run your IEs on a host different to the one that hosts your
Galaxy webserver, since IEs on the same host as the webserver compete for
resources with that webserver and introduce some security considerations which
could be mitigated by moving containers to a separate host. This feature has
been available since 15.07 and is used in production at the University of Freiburg.
First you need to configure a second host to be Docker enabled. In the
following we call this host ``gx-docker`` You need to start the Docker daemon
and bind it to a TCP port, not to a socket as is the default. For example
and bind it to a TCP port, not to a Unix socket as is the default. For example
you can start the daemon with
.. code-block:: console
@@ -229,483 +256,3 @@ NFS yet. You could for example use a sshfs mount with the following script
fi
This will let Galaxy and the Docker host share temporary files.
Interactive Environments in Detail (and How to Build Your Own)
--------------------------------------------------------------
Unfortunately building a GIE isn't completely straightforward, and it's
certainly not as simple as picking out an existing container and plugging it
in. Here we'll go through build a "Hello, World" GIE which just displays a file
from a user's history.
Directory Layout
^^^^^^^^^^^^^^^^
The GIE directory layout looks identical to that of normal visualization
plugins, for those familiar with developing those
.. code-block:: console
$ tree $GALAXY_ROOT/config/plugins/interactive_environments/ipython/
config/plugins/interactive_environments/ipython/
├── config
│ ├── ipython.ini
│ ├── ipython.ini.sample
│ └── ipython.xml
├── static
│ └── js
│ └── ipython.js
└── templates
├── ipython.mako
└── notebook.ipynb
We'll use the variable ``{gie}`` to stand in for the name of your GIE. It
should match ``[a-z]+``, like ``ipython`` or ``rstudio``. Here you can see the
``config/`` directory with a ``{gie}.ini.sample`` providing docker and image
configuration, and then ``{gie}.xml`` which outlines that it is a GIE.
The static directory can hold resources such as javascript and css files. If
you are actively developing a GIE, you'll need to restart Galaxy after adding
any resources to that file, before they can be accessed in the browser.
Lastly, and most importantly, there's the templates folder. This normally just
contains ``{gie}.mako``, however the IPython file needs an extra template file.
First Steps, Configuration
^^^^^^^^^^^^^^^^^^^^^^^^^^
We will name our GIE "helloworld", but you are free to name your's differently.
We'll first need to create the directory structure and set up our
configuration
.. code-block:: console
$ mkdir -p $GALAXY_ROOT/config/plugins/interactive_environments/helloworld/{config,static,templates}
$ cd $GALAXY_ROOT/config/plugins/interactive_environments/helloworld/
Next, you'll need to create the GIE plugin XML file ``config/helloworld.xml``
.. code-block:: xml
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE interactive_environment SYSTEM "../../interactive_environments.dtd">
<!-- This is the name which will show up in the User's Browser -->
<interactive_environment name="HelloWorld">
<data_sources>
<data_source>
<model_class>HistoryDatasetAssociation</model_class>
<!-- here you filter out which types of datasets are
appropriate for this GIE -->
<test type="isinstance" test_attr="datatype" result_type="datatype">tabular.Tabular</test>
<test type="isinstance" test_attr="datatype" result_type="datatype">data.Text</test>
<to_param param_attr="id">dataset_id</to_param>
</data_source>
</data_sources>
<params>
<param type="dataset" var_name_in_template="hda" required="true">dataset_id</param>
</params>
<!-- Be sure that your entrypoint name is correct! -->
<entry_point entry_point_type="mako">helloworld.mako</entry_point>
</interactive_environment>
Once this is done, we can set up our INI file, ``config/helloworld.ini.sample`` which controlls docker interaction
.. code-block:: ini
[main]
# Unused
[docker]
# Command to execute docker. For example `sudo docker` or `docker-lxc`.
#command = docker {docker_args}
# The docker image name that should be started.
image = hello-ie
# Additional arguments that are passed to the `docker run` command.
#command_inject = --sig-proxy=true -e DEBUG=false
# URL to access the Galaxy API with from the spawn Docker container, if empty
# this falls back to galaxy.ini's galaxy_infrastructure_url and finally to the
# Docker host of the spawned container if that is also not set.
#galaxy_url =
# The Docker hostname. It can be useful to run the Docker daemon on a different
# host than Galaxy.
#docker_hostname = localhost
We've named our image ``hello-ie``, we'll get to creating that in a minute.
Mako Templates
^^^^^^^^^^^^^^
Mako templates are very easy to use, and they allow significantly more
flexibility than many other templating languages. It's because of this
flexibility (and ability to write plain python code in them) that GIEs were
possible to develop easily.
In our ``templates/helloworld.mako``, we'll add the following
.. code-block:: html+mako
<%namespace name="ie" file="ie.mako" />
This line says to inherit from the ``ie.mako`` file that's available in
``$GALAXY_ROOT/config/plugins/interactive_environments/common/templates/ie.mako``.
Next we'll add the following
.. code-block:: html+mako
<%
# Sets ID and sets up a lot of other variables
ie_request.load_deploy_config()
# Define a volume that will be mounted into the container.
# This is a useful way to provide access to large files in the container,
# if the user knows ahead of time that they will need it.
user_file = ie_request.volume(
hda.file_name, '/import/file.dat', how='ro')
# Launch the IE. This builds and runs the docker command in the background.
ie_request.launch(
volumes=[user_file],
env_override={
'custom': '42'
}
)
# Only once the container is launched can we template our URLs. The ie_request
# doesn't have all of the information needed until the container is running.
url = ie_request.url_template('${PROXY_URL}/helloworld/')
%>
That mako snippet loaded the configuration from the INI files, launched the
docker container, and then built a URL to the correct endpoint, through the
Galaxy NodeJS proxy. Additionally we've set an environment variable named ``CUSTOM`` with the value ``42`` to be passed to the container, and we've attached the dataset the user selected (available in ``hda``) to the container as a read-only volume.
We'll continue appending to our ``helloworld.mako`` the HTML code that's actually displayed to the user, when this template is rendered
.. code-block:: html+mako
<html>
<head>
<!-- Loads some necessary javascript libraries. Specifically jquery,
toastr, and requirejs -->
${ ie.load_default_js() }
</head>
<body>
<script type="text/javascript">
// see $GALAXY_ROOT/config/plugins/interactive_environments/common/templates/ie.mako to learn what this does
${ ie.default_javascript_variables() }
var notebook_login_url = 'unused';
var notebook_access_url = '${ notebook_access_url }';
// Load code with require.js
${ ie.plugin_require_config() }
// Load notebook
// This will load code from static/helloworld.js, often used to handle
// things like Login. The load_notebook function will eventually append
// an IFrame to the <div id="main" /> below.
requirejs(['interactive_environments', 'plugin/helloworld'], function(){
load_notebook(notebook_access_url);
});
</script>
<div id="main" width="100%" height="100%">
</div>
</body>
</html>
We've glossed over some of the features of this file, but most IEs do a significant amount of "magic" in the top half of the mako template. For instance, the IPython notebook:
- If the user is trying to run the IPython GIE Visualization on an existing notebook in their history, then that gets loaded into the docker container via the temp directory and set as the default notebook
- Otherwise a default notebook is built for the user.
The RStudio notebook:
- generates a random password and configures the image to use this password
- Copies in an RData file if the user has loaded one
- sets some custom environment variables.
Connecting the User to the Container via Javascript
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
With the mako template above finished, if you were to load this in your
browser, not a lot would happen because we haven't built the hello-ie image,
and we haven't used Javascript to connect the user with the container. In the
tail end of the template, we set a variable ``notebook_access_url``. These are
partially a legacy of how things used to be done and you're welcome to clean up
your code according to your desires. Galaxy's NodeJS proxy handles the
authentication of users, so you don't have to worry about it, and can just
assume that only the correct user will have access to a given notebook.
In the ``static/`` directory, we generally create a ``js/`` directory below that, and create a ``{gie}.js`` (so, ``static/js/helloworld.js``) file in there. That file will have a function, ``load_notebook`` which will check if the GIE is available, and when it is, display it to the user.
We start by writing the load notebook function, which is pretty generic
.. code-block:: javascript
// Load an interactive environment (IE) from a remote URL
// @param {String} notebook_access_url: the URL embeded in the page and loaded
function load_notebook(notebook_access_url){
// When the page has completely loaded...
$( document ).ready(function() {
// Test if we can access the GIE, and if so, execute the function
// to load the GIE for the user.
test_ie_availability(notebook_access_url, function(){
_handle_notebook_loading(notebook_access_url);
});
});
}
This function will display a spinner to the user to indicate process, and then make multiple requests to ``notebook_access_url``. That MUST return a 200 OK for the ``_handle_notebook_loading`` function to ever be called. 302s do not count!
With that, we've almost completed the Javascript portion, just need to implement the function to display the GIE to the user in an iframe
.. code-block:: javascript
function _handle_notebook_loading(notebook_access_url){
append_notebook(notebook_access_url);
}
This function is very short. Historically, the GIE process involved a complex dance of:
- generating a random password in the mako template
- setting it as a javascript variable
- passing it to the docker container
- once the container was available, have the javascript automatically log a
user in (something browsers try to prevent since that's otherwise an XSS
vulnerability.)
- hope everything worked
Since the NodeJS proxy takes care of authentication/authorization, we can
reduce the helloworld ``_handle_notebook_loading`` function to a simple
``append_notebook`` call. You may wish to look at the IPython and RStudio GIEs
for examples of the complex things that can be done at every step.
The GIE Container
^^^^^^^^^^^^^^^^^
We'll build a simple container that just displays the dataset a user has
selected to them. Remember when we attached a volume to the container? We'll
make use of that now.
GIE Containers (often) consist of:
- Dockerfile
- NGINX Proxy configuration
- A custom startup script/entrypoint
- A script to monitor traffic and kill unused containers
We have to monitor the container's traffic and kill off unused containers,
bceause no one is watching them. The user launches the container in Galaxy, and
Galaxy immediately forgets the container exists. Thus, we say that if a
container has no connections to TCP connections to itself, then it should
commit suicide by killing the root process.
Here's an example Dockerfile for our helloworld container
.. code-block:: dockerfile
FROM ubuntu:14.04
# These environment variables are passed from Galaxy to the container
# and help you enable connectivity to Galaxy from within the container.
# This means your user can import/export data from/to Galaxy.
ENV DEBIAN_FRONTEND=noninteractive \
API_KEY=none \
DEBUG=false \
PROXY_PREFIX=none \
GALAXY_URL=none \
GALAXY_WEB_PORT=10000 \
HISTORY_ID=none \
REMOTE_HOST=none
RUN apt-get -qq update && \
apt-get install --no-install-recommends -y \
wget procps nginx python python-pip net-tools nginx
# Our very important scripts. Make sure you've run `chmod +x startup.sh
# monitor_traffic.sh` outside of the container!
ADD ./startup.sh /startup.sh
ADD ./monitor_traffic.sh /monitor_traffic.sh
# /import will be the universal mount-point for IPython
# The Galaxy instance can copy in data that needs to be present to the
# container
RUN mkdir /import
# Nginx configuration
COPY ./proxy.conf /proxy.conf
VOLUME ["/import"]
WORKDIR /import/
# EXTREMELY IMPORTANT! You must expose a SINGLE port on your container.
EXPOSE 80
CMD /startup.sh
If you have questions on this, please feel free to contact us on IRC
(`irc.freenode.net#galaxyproject <https://webchat.freenode.net/?channels=galaxyproject>`__).
The proxy configuration is interesting, here we'll point NGINX to reverse proxy
a service running on ``:8000`` inside the container. That port will be hosting
a python process which serves up the directory contents of ``/import``, i.e.
the file the user selected which was mounted as a volume into
``/import/file.dat``
.. code-block:: nginx
server {
listen 80;
server_name localhost;
access_log /var/log/nginx/localhost.access.log;
# Note the trailing slash used everywhere!
location PROXY_PREFIX/helloworld/ {
proxy_buffering off;
proxy_pass http://127.0.0.1:8000/;
proxy_redirect http://127.0.0.1:8000/ PROXY_PREFIX/helloworld/;
}
}
And here we'll run that service in our ``startup.sh`` file
.. code-block:: bash
#!/bin/bash
# First, replace the PROXY_PREFIX value in /proxy.conf with the value from
# the environment variable.
sed -i "s|PROXY_PREFIX|${PROXY_PREFIX}|" /proxy.conf;
# Then copy into the default location for ubuntu+nginx
cp /proxy.conf /etc/nginx/sites-enabled/default;
# Here you would normally start whatever service you want to start. In our
# example we start a simple directory listing service on port 8000
cd /import/ && python -mSimpleHTTPServer &
# Launch traffic monitor which will automatically kill the container if
# traffic stops
/monitor_traffic.sh &
# And finally launch nginx in foreground mode. This will make debugging
# easier as logs will be available from `docker logs ...`
nginx -g 'daemon off;'
Lastly, our ``monitor_traffic.sh`` file is often re-used between containers, the only adjustment being the port that is looked at
.. code-block:: bash
#!/bin/bash
while true; do
sleep 60
if [ `netstat -t | grep -v CLOSE_WAIT | grep ':80' | wc -l` -lt 3 ]
then
pkill nginx
fi
done
With those files, ``monitor_traffic.sh``, ``Dockerfile``, ``startup.sh``, and ``proxy.conf``, you should be able to build your ``hello-ie`` container
.. code-block:: bash
$ cd hello-ie
$ docker build -t hello-ie .
Now, if everything went smoothly, you should be able to restart Galaxy and try out your new GIE on a tabular or text file!
Debugging
^^^^^^^^^
When you launch your new GIE in Galaxy, your Galaxy logs should show something like the following:
.. code-block:: console
Starting docker container for IE helloworld with command [docker run --sig-proxy=true -e DEBUG=false -e "GALAXY_URL=http://localhost/galaxy/" -e "CORS_ORIGIN=http://localhost" -e "GALAXY_WEB_PORT=8000" -e "HISTORY_ID=f2db41e1fa331b3e" -e "CUSTOM=42" -e "GALAXY_PASTER_PORT=8000" -e "PROXY_PREFIX=/galaxy/gie_proxy" -e "API_KEY=1712364174a0ff79b34e9a78fee3ca1c" -e "REMOTE_HOST=127.0.0.1" -e "USER_EMAIL=hxr@local.host" -d -P -v "/home/hxr/work/galaxy/database/tmp/tmp5HaqZy:/import/" -v "/home/hxr/work/galaxy/database/files/000/dataset_68.dat:/import/file.dat:ro" hello-ie]
Here's the docker command written out in a more readable manner:
.. code-block:: console
$ docker run --sig-proxy=true \
-d -P \
-e "API_KEY=1712364174a0ff79b34e9a78fee3ca1c" \
-e "CORS_ORIGIN=http://localhost" \
-e "CUSTOM=42" \
-e "DEBUG=false" \
-e "GALAXY_PASTER_PORT=8000" \
-e "GALAXY_URL=http://localhost/galaxy/" \
-e "GALAXY_WEB_PORT=8000" \
-e "HISTORY_ID=f2db41e1fa331b3e" \
-e "PROXY_PREFIX=/galaxy/gie_proxy" \
-e "REMOTE_HOST=127.0.0.1" \
-e "USER_EMAIL=hxr@local.host" \
-v "/home/hxr/work/galaxy/database/tmp/tmp5HaqZy:/import/" \
-v "/home/hxr/work/galaxy/database/files/000/dataset_68.dat:/import/file.dat:ro" \
hello-ie
As you can see, a LOT is going on! We'll break it down further:
- ``-d`` runs the container in daemon mode, it launches and the client
submitting the container finished
- ``-P`` randomly assigns an unused port to the container for each ``EXPOSE``d
port from our ``Dockerfile``. This is why you must expose a port, and only
one port.
- A large number of environment variables are set:
- The user's API key is provided, allowing you to access datasets and
submit jobs on their behalf. If you have an environment like
IPython/RStudio, it is **highly recommended** that you provide some magic
by which the user can use their API key without embedding it in the
ntoebook. If you do embed it somehow in a document that gets saved to
their history, anyone can impersonate that user if they get a hold of it.
In the IPython GIE we have a variable that just runs
``os.environ.get('API_KEY')`` to avoid embedding it in their notebook.
- A CORS Origin is provided for very strict servers, but it may be easier
to simply void CORS requirements within the nginx proxy in your
container.
- Custom variables specified in your ``launch()`` command are available
- A ``DEBUG`` environment variable should be used to help admins debug
existing containers. You should use it to increase logging, not cleanup
temporary files, and so on.
- ``GALAXY_PASTER_PORT`` (deprecated) and ``GALAXY_WEB_PORT`` are the raw
port that Galaxy is listening on. You can use this to help decide how to
takl to Galaxy.
- ``GALAXY_URL`` is the URL that Galaxy should be accessible at. For
various reasons this may not be true. We recommend looking at our
implementation of `galaxy.py
<https://github.com/bgruening/docker-ipython-notebook/blob/15.07/galaxy.py>`__
which is a small utility script to provide API access to Galaxy to get
and fetch data, based on those environment variables.
- The ``HISTORY_ID`` of the current history the user is on is provided. In
the IPython/RStudio containers, we provide a dead simple method for users
to download datasets from their current history which will be visible to
them on the right hand side of their screen.
- A ``PROXY_PREFIX`` is provided which should be used in the nginx conf.
- ``REMOTE_HOST`` is another component used to test for a possible Galaxy
access path
- The user's email is made available, very convenient for webservices like
Entrez which require the user's email address. You can pre-fill it out
for them, making their life easier.
- Two volumes are mounted, one a temporary directory from Galaxy (rw), and one
the dataset the user selected (ro).
- and finally the image is specified.
Most of this information is usually required to build friendly, easy-to-use
GIEs. One of the strong points of GIEs is their magic interaction with Galaxy.
Here we've mounted a volume read-only, but in real life you may wish to provide
connectivity like IPython and RStudio provide, allowing the user to load
datasets on demand for interactive analysis, and then to store analysis
artefacts (and a log of what was done inside the container, à la IPython's
"notebooks") back to their current history.
If everything went well, at this point you should see a directory listing show up:
.. image:: interactive_environments_success.png
If you find yourself encountering difficulties, the "Hello, World" IE is
available in a `GitHub repo <https://github.com/erasche/hello-world-interactive-environment/releases/tag/v15.10>`__, and there are people on the IRC channel who can help debug.
doc/source/dev/index.rst
+2
View File
@@ -5,9 +5,11 @@ There are two primary areas of documentation developers will be interested in:
- `Codebase Documentation`_ for developers of Galaxy
- `API Documentation`_ for developers of third party tools interacting with Galaxy
- `Interactive Environments`_ for developers of Galaxy Interactive Environments
Additionally there is an `FAQ`_ document for new Galaxy developers answering common "How Do I..." questions.
.. _Codebase Documentation: ../lib/modules.html
.. _API Documentation: ../api_doc.html
.. _Interactive Environments: interactive_environments.html
.. _FAQ: faq.html
doc/source/dev/interactive_environments.rst
+479
View File
@@ -0,0 +1,479 @@
Interactive Environments in Detail (and How to Build Your Own)
==============================================================
Unfortunately building a GIE isn't completely straightforward, and it's
certainly not as simple as picking out an existing container and plugging it
in. Here we'll go through build a "Hello, World" GIE which just displays a file
from a user's history.
Directory Layout
----------------
The GIE directory layout looks identical to that of normal visualization
plugins, for those familiar with developing those
.. code-block:: console
$ tree $GALAXY_ROOT/config/plugins/interactive_environments/ipython/
config/plugins/interactive_environments/ipython/
├── config
│ ├── ipython.ini
│ ├── ipython.ini.sample
│ └── ipython.xml
├── static
│ └── js
│ └── ipython.js
└── templates
├── ipython.mako
└── notebook.ipynb
We'll use the variable ``{gie}`` to stand in for the name of your GIE. It
should match ``[a-z]+``, like ``ipython`` or ``rstudio``. Here you can see the
``config/`` directory with a ``{gie}.ini.sample`` providing docker and image
configuration, and then ``{gie}.xml`` which outlines that it is a GIE.
The static directory can hold resources such as javascript and css files. If
you are actively developing a GIE, you'll need to restart Galaxy after adding
any resources to that file, before they can be accessed in the browser.
Lastly, and most importantly, there's the templates folder. This normally just
contains ``{gie}.mako``, however the IPython file needs an extra template file.
First Steps, Configuration
--------------------------
We will name our GIE "helloworld", but you are free to name your's differently.
We'll first need to create the directory structure and set up our
configuration
.. code-block:: console
$ mkdir -p $GALAXY_ROOT/config/plugins/interactive_environments/helloworld/{config,static,templates}
$ cd $GALAXY_ROOT/config/plugins/interactive_environments/helloworld/
Next, you'll need to create the GIE plugin XML file ``config/helloworld.xml``
.. code-block:: xml
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE interactive_environment SYSTEM "../../interactive_environments.dtd">
<!-- This is the name which will show up in the User's Browser -->
<interactive_environment name="HelloWorld">
<data_sources>
<data_source>
<model_class>HistoryDatasetAssociation</model_class>
<!-- here you filter out which types of datasets are
appropriate for this GIE -->
<test type="isinstance" test_attr="datatype" result_type="datatype">tabular.Tabular</test>
<test type="isinstance" test_attr="datatype" result_type="datatype">data.Text</test>
<to_param param_attr="id">dataset_id</to_param>
</data_source>
</data_sources>
<params>
<param type="dataset" var_name_in_template="hda" required="true">dataset_id</param>
</params>
<!-- Be sure that your entrypoint name is correct! -->
<entry_point entry_point_type="mako">helloworld.mako</entry_point>
</interactive_environment>
Once this is done, we can set up our INI file, ``config/helloworld.ini.sample`` which controlls docker interaction
.. code-block:: ini
[main]
# Unused
[docker]
# Command to execute docker. For example `sudo docker` or `docker-lxc`.
#command = docker {docker_args}
# The docker image name that should be started.
image = hello-ie
# Additional arguments that are passed to the `docker run` command.
#command_inject = --sig-proxy=true -e DEBUG=false
# URL to access the Galaxy API with from the spawn Docker container, if empty
# this falls back to galaxy.ini's galaxy_infrastructure_url and finally to the
# Docker host of the spawned container if that is also not set.
#galaxy_url =
# The Docker hostname. It can be useful to run the Docker daemon on a different
# host than Galaxy.
#docker_hostname = localhost
We've named our image ``hello-ie``, we'll get to creating that in a minute.
Mako Templates
--------------
Mako templates are very easy to use, and they allow significantly more
flexibility than many other templating languages. It's because of this
flexibility (and ability to write plain python code in them) that GIEs were
possible to develop easily.
In our ``templates/helloworld.mako``, we'll add the following
.. code-block:: html+mako
<%namespace name="ie" file="ie.mako" />
This line says to inherit from the ``ie.mako`` file that's available in
``$GALAXY_ROOT/config/plugins/interactive_environments/common/templates/ie.mako``.
Next we'll add the following
.. code-block:: html+mako
<%
# Sets ID and sets up a lot of other variables
ie_request.load_deploy_config()
# Define a volume that will be mounted into the container.
# This is a useful way to provide access to large files in the container,
# if the user knows ahead of time that they will need it.
user_file = ie_request.volume(
hda.file_name, '/import/file.dat', how='ro')
# Launch the IE. This builds and runs the docker command in the background.
ie_request.launch(
volumes=[user_file],
env_override={
'custom': '42'
}
)
# Only once the container is launched can we template our URLs. The ie_request
# doesn't have all of the information needed until the container is running.
url = ie_request.url_template('${PROXY_URL}/helloworld/')
%>
That mako snippet loaded the configuration from the INI files, launched the
docker container, and then built a URL to the correct endpoint, through the
Galaxy NodeJS proxy. Additionally we've set an environment variable named ``CUSTOM`` with the value ``42`` to be passed to the container, and we've attached the dataset the user selected (available in ``hda``) to the container as a read-only volume.
We'll continue appending to our ``helloworld.mako`` the HTML code that's actually displayed to the user, when this template is rendered
.. code-block:: html+mako
<html>
<head>
<!-- Loads some necessary javascript libraries. Specifically jquery,
toastr, and requirejs -->
${ ie.load_default_js() }
</head>
<body>
<script type="text/javascript">
// see $GALAXY_ROOT/config/plugins/interactive_environments/common/templates/ie.mako to learn what this does
${ ie.default_javascript_variables() }
var notebook_login_url = 'unused';
var notebook_access_url = '${ notebook_access_url }';
// Load code with require.js
${ ie.plugin_require_config() }
// Load notebook
// This will load code from static/helloworld.js, often used to handle
// things like Login. The load_notebook function will eventually append
// an IFrame to the <div id="main" /> below.
requirejs(['interactive_environments', 'plugin/helloworld'], function(){
load_notebook(notebook_access_url);
});
</script>
<div id="main" width="100%" height="100%">
</div>
</body>
</html>
We've glossed over some of the features of this file, but most IEs do a significant amount of "magic" in the top half of the mako template. For instance, the IPython notebook:
- If the user is trying to run the IPython GIE Visualization on an existing notebook in their history, then that gets loaded into the docker container via the temp directory and set as the default notebook
- Otherwise a default notebook is built for the user.
The RStudio notebook:
- generates a random password and configures the image to use this password
- Copies in an RData file if the user has loaded one
- sets some custom environment variables.
Connecting the User to the Container via Javascript
---------------------------------------------------
With the mako template above finished, if you were to load this in your
browser, not a lot would happen because we haven't built the hello-ie image,
and we haven't used Javascript to connect the user with the container. In the
tail end of the template, we set a variable ``notebook_access_url``. These are
partially a legacy of how things used to be done and you're welcome to clean up
your code according to your desires. Galaxy's NodeJS proxy handles the
authentication of users, so you don't have to worry about it, and can just
assume that only the correct user will have access to a given notebook.
In the ``static/`` directory, we generally create a ``js/`` directory below that, and create a ``{gie}.js`` (so, ``static/js/helloworld.js``) file in there. That file will have a function, ``load_notebook`` which will check if the GIE is available, and when it is, display it to the user.
We start by writing the load notebook function, which is pretty generic
.. code-block:: javascript
// Load an interactive environment (IE) from a remote URL
// @param {String} notebook_access_url: the URL embeded in the page and loaded
function load_notebook(notebook_access_url){
// When the page has completely loaded...
$( document ).ready(function() {
// Test if we can access the GIE, and if so, execute the function
// to load the GIE for the user.
test_ie_availability(notebook_access_url, function(){
_handle_notebook_loading(notebook_access_url);
});
});
}
This function will display a spinner to the user to indicate process, and then make multiple requests to ``notebook_access_url``. That MUST return a 200 OK for the ``_handle_notebook_loading`` function to ever be called. 302s do not count!
With that, we've almost completed the Javascript portion, just need to implement the function to display the GIE to the user in an iframe
.. code-block:: javascript
function _handle_notebook_loading(notebook_access_url){
append_notebook(notebook_access_url);
}
This function is very short. Historically, the GIE process involved a complex dance of:
- generating a random password in the mako template
- setting it as a javascript variable
- passing it to the docker container
- once the container was available, have the javascript automatically log a
user in (something browsers try to prevent since that's otherwise an XSS
vulnerability.)
- hope everything worked
Since the NodeJS proxy takes care of authentication/authorization, we can
reduce the helloworld ``_handle_notebook_loading`` function to a simple
``append_notebook`` call. You may wish to look at the IPython and RStudio GIEs
for examples of the complex things that can be done at every step.
The GIE Container
-----------------
We'll build a simple container that just displays the dataset a user has
selected to them. Remember when we attached a volume to the container? We'll
make use of that now.
GIE Containers (often) consist of:
- Dockerfile
- NGINX Proxy configuration
- A custom startup script/entrypoint
- A script to monitor traffic and kill unused containers
We have to monitor the container's traffic and kill off unused containers,
bceause no one is watching them. The user launches the container in Galaxy, and
Galaxy immediately forgets the container exists. Thus, we say that if a
container has no connections to TCP connections to itself, then it should
commit suicide by killing the root process.
Here's an example Dockerfile for our helloworld container
.. code-block:: dockerfile
FROM ubuntu:14.04
# These environment variables are passed from Galaxy to the container
# and help you enable connectivity to Galaxy from within the container.
# This means your user can import/export data from/to Galaxy.
ENV DEBIAN_FRONTEND=noninteractive \
API_KEY=none \
DEBUG=false \
PROXY_PREFIX=none \
GALAXY_URL=none \
GALAXY_WEB_PORT=10000 \
HISTORY_ID=none \
REMOTE_HOST=none
RUN apt-get -qq update && \
apt-get install --no-install-recommends -y \
wget procps nginx python python-pip net-tools nginx
# Our very important scripts. Make sure you've run `chmod +x startup.sh
# monitor_traffic.sh` outside of the container!
ADD ./startup.sh /startup.sh
ADD ./monitor_traffic.sh /monitor_traffic.sh
# /import will be the universal mount-point for IPython
# The Galaxy instance can copy in data that needs to be present to the
# container
RUN mkdir /import
# Nginx configuration
COPY ./proxy.conf /proxy.conf
VOLUME ["/import"]
WORKDIR /import/
# EXTREMELY IMPORTANT! You must expose a SINGLE port on your container.
EXPOSE 80
CMD /startup.sh
If you have questions on this, please feel free to contact us on IRC
(`irc.freenode.net#galaxyproject <https://webchat.freenode.net/?channels=galaxyproject>`__).
The proxy configuration is interesting, here we'll point NGINX to reverse proxy
a service running on ``:8000`` inside the container. That port will be hosting
a python process which serves up the directory contents of ``/import``, i.e.
the file the user selected which was mounted as a volume into
``/import/file.dat``
.. code-block:: nginx
server {
listen 80;
server_name localhost;
access_log /var/log/nginx/localhost.access.log;
# Note the trailing slash used everywhere!
location PROXY_PREFIX/helloworld/ {
proxy_buffering off;
proxy_pass http://127.0.0.1:8000/;
proxy_redirect http://127.0.0.1:8000/ PROXY_PREFIX/helloworld/;
}
}
And here we'll run that service in our ``startup.sh`` file
.. code-block:: bash
#!/bin/bash
# First, replace the PROXY_PREFIX value in /proxy.conf with the value from
# the environment variable.
sed -i "s|PROXY_PREFIX|${PROXY_PREFIX}|" /proxy.conf;
# Then copy into the default location for ubuntu+nginx
cp /proxy.conf /etc/nginx/sites-enabled/default;
# Here you would normally start whatever service you want to start. In our
# example we start a simple directory listing service on port 8000
cd /import/ && python -mSimpleHTTPServer &
# Launch traffic monitor which will automatically kill the container if
# traffic stops
/monitor_traffic.sh &
# And finally launch nginx in foreground mode. This will make debugging
# easier as logs will be available from `docker logs ...`
nginx -g 'daemon off;'
Lastly, our ``monitor_traffic.sh`` file is often re-used between containers, the only adjustment being the port that is looked at
.. code-block:: bash
#!/bin/bash
while true; do
sleep 60
if [ `netstat -t | grep -v CLOSE_WAIT | grep ':80' | wc -l` -lt 3 ]
then
pkill nginx
fi
done
With those files, ``monitor_traffic.sh``, ``Dockerfile``, ``startup.sh``, and ``proxy.conf``, you should be able to build your ``hello-ie`` container
.. code-block:: bash
$ cd hello-ie
$ docker build -t hello-ie .
Now, if everything went smoothly, you should be able to restart Galaxy and try out your new GIE on a tabular or text file!
Debugging
---------
When you launch your new GIE in Galaxy, your Galaxy logs should show something like the following:
.. code-block:: console
Starting docker container for IE helloworld with command [docker run --sig-proxy=true -e DEBUG=false -e "GALAXY_URL=http://localhost/galaxy/" -e "CORS_ORIGIN=http://localhost" -e "GALAXY_WEB_PORT=8000" -e "HISTORY_ID=f2db41e1fa331b3e" -e "CUSTOM=42" -e "GALAXY_PASTER_PORT=8000" -e "PROXY_PREFIX=/galaxy/gie_proxy" -e "API_KEY=1712364174a0ff79b34e9a78fee3ca1c" -e "REMOTE_HOST=127.0.0.1" -e "USER_EMAIL=hxr@local.host" -d -P -v "/home/hxr/work/galaxy/database/tmp/tmp5HaqZy:/import/" -v "/home/hxr/work/galaxy/database/files/000/dataset_68.dat:/import/file.dat:ro" hello-ie]
Here's the docker command written out in a more readable manner:
.. code-block:: console
$ docker run --sig-proxy=true \
-d -P \
-e "API_KEY=1712364174a0ff79b34e9a78fee3ca1c" \
-e "CORS_ORIGIN=http://localhost" \
-e "CUSTOM=42" \
-e "DEBUG=false" \
-e "GALAXY_PASTER_PORT=8000" \
-e "GALAXY_URL=http://localhost/galaxy/" \
-e "GALAXY_WEB_PORT=8000" \
-e "HISTORY_ID=f2db41e1fa331b3e" \
-e "PROXY_PREFIX=/galaxy/gie_proxy" \
-e "REMOTE_HOST=127.0.0.1" \
-e "USER_EMAIL=hxr@local.host" \
-v "/home/hxr/work/galaxy/database/tmp/tmp5HaqZy:/import/" \
-v "/home/hxr/work/galaxy/database/files/000/dataset_68.dat:/import/file.dat:ro" \
hello-ie
As you can see, a LOT is going on! We'll break it down further:
- ``-d`` runs the container in daemon mode, it launches and the client
submitting the container finished
- ``-P`` randomly assigns an unused port to the container for each ``EXPOSE``d
port from our ``Dockerfile``. This is why you must expose a port, and only
one port.
- A large number of environment variables are set:
- The user's API key is provided, allowing you to access datasets and
submit jobs on their behalf. If you have an environment like
IPython/RStudio, it is **highly recommended** that you provide some magic
by which the user can use their API key without embedding it in the
notebook. If you do embed it somehow in a document that gets saved to
their history, anyone can impersonate that user if they get a hold of it.
In the IPython GIE we have a variable that just runs
``os.environ.get('API_KEY')`` to avoid embedding it in their notebook.
- A CORS Origin is provided for very strict servers, but it may be easier
to simply void CORS requirements within the nginx proxy in your
container.
- Custom variables specified in your ``launch()`` command are available
- A ``DEBUG`` environment variable should be used to help admins debug
existing containers. You should use it to increase logging, not cleanup
temporary files, and so on.
- ``GALAXY_PASTER_PORT`` (deprecated) and ``GALAXY_WEB_PORT`` are the raw
port that Galaxy is listening on. You can use this to help decide how to
takl to Galaxy.
- ``GALAXY_URL`` is the URL that Galaxy should be accessible at. For
various reasons this may not be true. We recommend looking at our
implementation of `galaxy.py
<https://github.com/bgruening/docker-ipython-notebook/blob/15.07/galaxy.py>`__
which is a small utility script to provide API access to Galaxy to get
and fetch data, based on those environment variables.
- The ``HISTORY_ID`` of the current history the user is on is provided. In
the IPython/RStudio containers, we provide a dead simple method for users
to download datasets from their current history which will be visible to
them on the right hand side of their screen.
- A ``PROXY_PREFIX`` is provided which should be used in the nginx conf.
- ``REMOTE_HOST`` is another component used to test for a possible Galaxy
access path
- The user's email is made available, very convenient for webservices like
Entrez which require the user's email address. You can pre-fill it out
for them, making their life easier.
- Two volumes are mounted, one a temporary directory from Galaxy (rw), and one
the dataset the user selected (ro).
- and finally the image is specified.
Most of this information is usually required to build friendly, easy-to-use
GIEs. One of the strong points of GIEs is their magic interaction with Galaxy.
Here we've mounted a volume read-only, but in real life you may wish to provide
connectivity like IPython and RStudio provide, allowing the user to load
datasets on demand for interactive analysis, and then to store analysis
artefacts (and a log of what was done inside the container, à la IPython's
"notebooks") back to their current history.
If everything went well, at this point you should see a directory listing show up:
.. image:: interactive_environments_success.png
If you find yourself encountering difficulties, the "Hello, World" IE is
available in a `GitHub repo <https://github.com/erasche/hello-world-interactive-environment/releases/tag/v15.10>`__, and there are people on the IRC channel who can help debug.
lib/galaxy/config.py
+3 -1
View File
@@ -113,6 +113,7 @@ class Configuration( object ):
self.expose_user_name = kwargs.get( "expose_user_name", False )
self.expose_user_email = kwargs.get( "expose_user_email", False )
self.password_expiration_period = timedelta( days=int( kwargs.get( "password_expiration_period", 0 ) ) )
# Check for tools defined in the above non-shed tool configs (i.e., tool_conf.xml) tht have
# been migrated from the Galaxy code distribution to the Tool Shed.
@@ -162,7 +163,7 @@ class Configuration( object ):
self.require_login = string_as_bool( kwargs.get( "require_login", "False" ) )
self.allow_user_creation = string_as_bool( kwargs.get( "allow_user_creation", "True" ) )
self.allow_user_deletion = string_as_bool( kwargs.get( "allow_user_deletion", "False" ) )
self.allow_user_dataset_purge = string_as_bool( kwargs.get( "allow_user_dataset_purge", "False" ) )
self.allow_user_dataset_purge = string_as_bool( kwargs.get( "allow_user_dataset_purge", "True" ) )
self.allow_user_impersonation = string_as_bool( kwargs.get( "allow_user_impersonation", "False" ) )
self.new_user_dataset_access_role_default_private = string_as_bool( kwargs.get( "new_user_dataset_access_role_default_private", "False" ) )
self.collect_outputs_from = [ x.strip() for x in kwargs.get( 'collect_outputs_from', 'new_file_path,job_working_directory' ).lower().split(',') ]
@@ -525,6 +526,7 @@ class Configuration( object ):
datatypes_config_file=[ 'config/datatypes_conf.xml', 'datatypes_conf.xml', 'config/datatypes_conf.xml.sample' ],
external_service_type_config_file=[ 'config/external_service_types_conf.xml', 'external_service_types_conf.xml', 'config/external_service_types_conf.xml.sample' ],
job_config_file=[ 'config/job_conf.xml', 'job_conf.xml' ],
tool_destinations_config_file=[ 'config/tool_destinations.yml', 'config/tool_destinations.yml.sample' ],
job_metrics_config_file=[ 'config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample' ],
dependency_resolvers_config_file=[ 'config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml' ],
job_resource_params_file=[ 'config/job_resource_params_conf.xml', 'job_resource_params_conf.xml' ],
lib/galaxy/dataset_collections/structure.py
+2 -2
View File
@@ -119,12 +119,12 @@ class Tree( object ):
return Tree( new_children, new_collection_type )
def clone( self ):
cloned_children = map( lambda (identifier, element): (identifier, element.clone()), self.children )
cloned_children = [(_[0], _[1].clone()) for _ in self.children]
return Tree( cloned_children, self.collection_type_description )
def dict_map( func, input_dict ):
return dict( [ ( k, func(v) ) for k, v in input_dict.iteritems() ] )
return dict( [ ( k, func(v) ) for k, v in input_dict.items() ] )
def get_structure( dataset_collection_instance, collection_type_description, leaf_subcollection_type=None ):
lib/galaxy/datatypes/binary.py
+8 -7
View File
@@ -1,4 +1,5 @@
"""Binary classes"""
from __future__ import print_function
import binascii
import gzip
@@ -72,7 +73,7 @@ class Binary( data.Data ):
"""Returns the mime type of the datatype"""
return 'application/octet-stream'
def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, size=None, offset=None, **kwd):
def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, **kwd):
trans.response.set_content_type(dataset.get_mime())
trans.log_event( "Display dataset id: %s" % str( dataset.id ) )
trans.response.headers['Content-Length'] = int( os.stat( dataset.file_name ).st_size )
@@ -234,7 +235,7 @@ class Bam( Binary ):
shutil.rmtree(tmp_dir) # clean up
raise Exception( "Error merging BAM files: %s" % stderr )
else:
print stderr
print(stderr)
os.unlink(stderr_name)
os.rmdir(tmp_dir)
@@ -322,7 +323,7 @@ class Bam( Binary ):
shutil.rmtree( tmp_dir) # clean up
raise Exception( "Error Grooming BAM file contents: %s" % stderr )
else:
print stderr
print(stderr)
# Move samtools_created_sorted_file_name to our output dataset location
shutil.move( samtools_created_sorted_file_name, file_name )
# Remove temp file and empty temporary directory
@@ -365,7 +366,7 @@ class Bam( Binary ):
os.unlink( stderr_name ) # clean up
raise Exception( "Error Setting BAM Metadata: %s" % stderr )
else:
print stderr
print(stderr)
dataset.metadata.bam_index = index_file
# Remove temp file
os.unlink( stderr_name )
@@ -932,7 +933,7 @@ class GeminiSQLite( SQlite ):
c = conn.cursor()
tables_query = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
result = c.execute( tables_query ).fetchall()
result = map( lambda x: x[0], result )
result = [_[0] for _ in result]
for table_name in gemini_table_names:
if table_name not in result:
return False
@@ -971,7 +972,7 @@ class MzSQlite( SQlite ):
c = conn.cursor()
tables_query = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
result = c.execute( tables_query ).fetchall()
result = map( lambda x: x[0], result )
result = [_[0] for _ in result]
for table_name in mz_table_names:
if table_name not in result:
return False
@@ -1006,7 +1007,7 @@ class IdpDB( SQlite ):
c = conn.cursor()
tables_query = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
result = c.execute( tables_query ).fetchall()
result = map( lambda x: x[0], result )
result = [_[0] for _ in result]
for table_name in mz_table_names:
if table_name not in result:
return False
lib/galaxy/datatypes/converters/bed_to_gff_converter.py
+3 -1
View File
@@ -1,5 +1,7 @@
#!/usr/bin/env python
# This code exists in 2 places: ~/datatypes/converters and ~/tools/filters
from __future__ import print_function
import sys
assert sys.version_info[:2] >= ( 2, 4 )
@@ -69,7 +71,7 @@ def __main__():
info_msg = "%i lines converted to GFF version 2. " % ( i + 1 - skipped_lines )
if skipped_lines > 0:
info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( skipped_lines, first_skipped_line )
print info_msg
print(info_msg)
if __name__ == "__main__":
__main__()
lib/galaxy/datatypes/converters/fastq_to_fqtoc.py
+3 -1
View File
@@ -1,6 +1,8 @@
#!/usr/bin/env python
from __future__ import print_function
import sys
from galaxy.util.checkers import is_gzip
@@ -18,7 +20,7 @@ def main():
"""
input_fname = sys.argv[1]
if is_gzip(input_fname):
print 'Conversion is only possible for uncompressed files'
print('Conversion is only possible for uncompressed files')
sys.exit(1)
out_file = open(sys.argv[2], 'w')
lib/galaxy/datatypes/converters/gff_to_bed_converter.py
+3 -1
View File
@@ -1,4 +1,6 @@
#!/usr/bin/env python
from __future__ import print_function
import sys
assert sys.version_info[:2] >= ( 2, 4 )
@@ -38,7 +40,7 @@ def __main__():
info_msg = "%i lines converted to BED. " % ( i + 1 - skipped_lines )
if skipped_lines > 0:
info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( skipped_lines, first_skipped_line )
print info_msg
print(info_msg)
if __name__ == "__main__":
__main__()
lib/galaxy/datatypes/converters/interval_to_bed_converter.py
+4 -2
View File
@@ -1,5 +1,7 @@
#!/usr/bin/env python
# Dan Blankenberg
from __future__ import print_function
import sys
import bx.intervals.io
@@ -55,9 +57,9 @@ def __main__():
if not first_skipped_line:
first_skipped_line = count + 1
out.close()
print "%i regions converted to BED." % ( count + 1 - skipped_lines )
print("%i regions converted to BED." % ( count + 1 - skipped_lines ))
if skipped_lines > 0:
print "Skipped %d blank or invalid lines starting with line # %d." % ( skipped_lines, first_skipped_line )
print("Skipped %d blank or invalid lines starting with line # %d." % ( skipped_lines, first_skipped_line ))
if __name__ == "__main__":
__main__()
lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py
+5 -3
View File
@@ -1,5 +1,7 @@
#!/usr/bin/env python
# Dan Blankenberg
from __future__ import print_function
import sys
import bx.intervals.io
@@ -138,7 +140,7 @@ def __main__():
except:
name = "region_%i" % count
try:
fields = map( str, [ region.chrom, region.start, region.end, name, 0, region.strand ] )
fields = [str(item) for item in (region.chrom, region.start, region.end, name, 0, region.strand)]
if force_num_columns is not None and len( fields ) != force_num_columns:
fields = force_bed_field_count( fields, count, force_num_columns )
out.write( "%s\n" % '\t'.join( fields ) )
@@ -147,9 +149,9 @@ def __main__():
if first_skipped_line is None:
first_skipped_line = count + 1
out.close()
print "%i regions converted to BED." % ( count + 1 - skipped_lines )
print("%i regions converted to BED." % ( count + 1 - skipped_lines ))
if skipped_lines > 0:
print "Skipped %d blank or invalid lines starting with line # %d." % ( skipped_lines, first_skipped_line )
print("Skipped %d blank or invalid lines starting with line # %d." % ( skipped_lines, first_skipped_line ))
if __name__ == "__main__":
__main__()
lib/galaxy/datatypes/converters/interval_to_coverage.py
+9 -9
View File
@@ -6,7 +6,7 @@ usage: %prog bed_file out_file
-1, --cols1=N,N,N,N: Columns for chrom, start, end, strand in interval file
-2, --cols2=N,N,N,N: Columns for chrom, start, end, strand in coverage file
"""
import commands
import subprocess
import tempfile
from bisect import bisect
from os import environ
@@ -42,11 +42,11 @@ def main( interval, coverage ):
for record in interval:
chrom = record.chrom
if lastchrom and not lastchrom == chrom and partitions:
for partition in xrange(0, len(partitions) - 1):
for partition in range(0, len(partitions) - 1):
forward = forward_covs[partition]
reverse = reverse_covs[partition]
if forward + reverse > 0:
coverage.write(chrom=chrom, position=xrange(partitions[partition], partitions[partition + 1]),
coverage.write(chrom=chrom, position=range(partitions[partition], partitions[partition + 1]),
forward=forward, reverse=reverse)
partitions = []
forward_covs = []
@@ -64,7 +64,7 @@ def main( interval, coverage ):
forward_covs.insert(start_index, forward_base)
reverse_covs.insert(start_index, reverse_base)
end_index = bisect(partitions, record.end)
for index in xrange(start_index, end_index):
for index in range(start_index, end_index):
forward_covs[index] += forward
reverse_covs[index] += reverse
partitions.insert(end_index, record.end)
@@ -72,11 +72,11 @@ def main( interval, coverage ):
reverse_covs.insert(end_index, reverse_covs[end_index - 1] - reverse )
if partitions:
for partition in xrange(0, start_index):
for partition in range(0, start_index):
forward = forward_covs[partition]
reverse = reverse_covs[partition]
if forward + reverse > 0:
coverage.write(chrom=chrom, position=xrange(partitions[partition], partitions[partition + 1]),
coverage.write(chrom=chrom, position=range(partitions[partition], partitions[partition + 1]),
forward=forward, reverse=reverse)
partitions = partitions[start_index:]
forward_covs = forward_covs[start_index:]
@@ -86,11 +86,11 @@ def main( interval, coverage ):
# Finish the last chromosome
if partitions:
for partition in xrange(0, len(partitions) - 1):
for partition in range(0, len(partitions) - 1):
forward = forward_covs[partition]
reverse = reverse_covs[partition]
if forward + reverse > 0:
coverage.write(chrom=chrom, position=xrange(partitions[partition], partitions[partition + 1]),
coverage.write(chrom=chrom, position=range(partitions[partition], partitions[partition + 1]),
forward=forward, reverse=reverse)
@@ -133,7 +133,7 @@ if __name__ == "__main__":
temp_file = tempfile.NamedTemporaryFile(mode="r")
environ['LC_ALL'] = 'POSIX'
commandline = "sort -f -n -k %d -k %d -k %d -o %s %s" % (chr_col_1 + 1, start_col_1 + 1, end_col_1 + 1, temp_file.name, in_fname)
errorcode, stdout = commands.getstatusoutput(commandline)
subprocess.check_call(commandline, shell=True)
coverage = CoverageWriter( out_stream=open(out_fname, "a"),
chromCol=chr_col_2, positionCol=position_col_2,
lib/galaxy/datatypes/converters/lped_to_fped_converter.py
+4 -4
View File
@@ -2,12 +2,12 @@
# recode to numeric fbat version
# much slower so best to always
# use numeric alleles internally
from __future__ import print_function
import sys
import os
import sys
import time
prog = os.path.split(sys.argv[0])[-1]
myversion = 'Oct 10 2009'
@@ -63,7 +63,7 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath):
if i == 0:
lrow = row.split()
try:
x = [int(x) for x in lrow[10:50]] # look for non numeric codes
[int(x) for x in lrow[10:50]] # look for non numeric codes
except:
dorecode = 1
if dorecode:
@@ -100,7 +100,7 @@ def main():
flist = os.listdir(outfilepath)
with open(outhtmlname, 'w') as f:
f.write(galhtmlprefix % prog)
print '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog, timenow()) # becomes info
print('## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog, timenow())) # becomes info
f.write('<div>## Rgenetics: http://rgenetics.org Galaxy Tools %s %s\n<ol>' % (prog, timenow()))
for i, data in enumerate( flist ):
f.write('<li><a href="%s">%s</a></li>\n' % (os.path.split(data)[-1], os.path.split(data)[-1]))
lib/galaxy/datatypes/converters/lped_to_pbed_converter.py
+6 -4
View File
@@ -4,10 +4,12 @@
# eg lped/eigen/fbat/snpmatrix all to pbed
# and pbed to lped/eigen/fbat/snpmatrix ?
# that's a lot of converters
import sys
from __future__ import print_function
import os
import time
import subprocess
import sys
import time
prog = os.path.split(sys.argv[0])[-1]
myversion = 'Oct 10 2009'
@@ -68,7 +70,7 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath, plink):
outroot = os.path.join(outfilepath, basename)
missval = getMissval(inped=pedf)
if not missval:
print '### lped_to_pbed_converter.py cannot identify missing value in %s' % pedf
print('### lped_to_pbed_converter.py cannot identify missing value in %s' % pedf)
missval = '0'
cl = '%s --noweb --file %s --make-bed --out %s --missing-genotype %s' % (plink, inpedfilepath, outroot, missval)
p = subprocess.Popen(cl, shell=True, cwd=outfilepath)
@@ -100,7 +102,7 @@ def main():
with open(outhtmlname, 'w') as f:
f.write(galhtmlprefix % prog)
s = '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog, timenow()) # becomes info
print s
print(s)
f.write('<div>%s\n<ol>' % (s))
for i, data in enumerate( flist ):
f.write('<li><a href="%s">%s</a></li>\n' % (os.path.split(data)[-1], os.path.split(data)[-1]))
lib/galaxy/datatypes/converters/maf_to_fasta_converter.py
+3 -1
View File
@@ -1,5 +1,7 @@
#!/usr/bin/env python
# Dan Blankenberg
from __future__ import print_function
import sys
import bx.align.maf
@@ -26,7 +28,7 @@ def __main__():
out.write( "%s\n" % c.text )
out.write( "\n" )
out.close()
print "%i MAF blocks converted to FASTA." % ( count )
print("%i MAF blocks converted to FASTA." % ( count ))
if __name__ == "__main__":
lib/galaxy/datatypes/converters/maf_to_interval_converter.py
+4 -2
View File
@@ -1,5 +1,7 @@
#!/usr/bin/env python
# Dan Blankenberg
from __future__ import print_function
import sys
import bx.align.maf
@@ -24,9 +26,9 @@ def __main__():
out.write( "%s\t%i\t%i\t%s\n" % ( maf_utilities.src_split( c.src )[-1], c.get_forward_strand_start(), c.get_forward_strand_end(), c.strand ) )
count += 1
except Exception as e:
print >> sys.stderr, "There was a problem processing your input: %s" % e
print("There was a problem processing your input: %s" % e, file=sys.stderr)
out.close()
print "%i MAF blocks converted to Genomic Intervals for species %s." % ( count, species )
print("%i MAF blocks converted to Genomic Intervals for species %s." % ( count, species ))
if __name__ == "__main__":
lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py
+2 -2
View File
@@ -1,6 +1,6 @@
# converter for ldreduced rgenetics datatype
# used for grr and eigenstrat - shellfish if we get around to it
#
from __future__ import print_function
import os
import sys
@@ -104,7 +104,7 @@ def main():
f.write(galhtmlprefix % prog)
s1 = '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog, timenow()) # becomes info
s2 = 'Input %s, winsize=%s, winmove=%s, r2thresh=%s' % (base_name, winsize, winmove, r2thresh)
print '%s %s' % (s1, s2)
print('%s %s' % (s1, s2))
f.write('<div>%s\n%s\n<ol>' % (s1, s2))
for i, data in enumerate( flist ):
f.write('<li><a href="%s">%s</a></li>\n' % (os.path.split(data)[-1], os.path.split(data)[-1]))
lib/galaxy/datatypes/converters/pbed_to_lped_converter.py
+3 -1
View File
@@ -4,6 +4,8 @@
# eg lped/eigen/fbat/snpmatrix all to pbed
# and pbed to lped/eigen/fbat/snpmatrix ?
# that's a lot of converters
from __future__ import print_function
import os
import subprocess
import sys
@@ -68,7 +70,7 @@ def main():
with open(outhtmlname, 'w') as f:
f.write(galhtmlprefix % prog)
s = '## Rgenetics: http://bitbucket.org/rgalaxy Galaxy Tools %s %s' % (prog, timenow()) # becomes info
print s
print(s)
f.write('<div>%s\n<ol>' % (s))
for i, data in enumerate( flist ):
f.write('<li><a href="%s">%s</a></li>\n' % (os.path.split(data)[-1], os.path.split(data)[-1]))
lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py
+3 -2
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python
# Dan Blankenberg
from __future__ import print_function
import sys
@@ -25,7 +26,7 @@ def __main__():
elems = line.split( '\t' )
out.write( '%s\t%s\t%s\t%s\t0\t%s\n' % ( elems[0], int(elems[1]) - 1, elems[2], elems[4], elems[3] ) )
except Exception as e:
print e
print(e)
skipped_lines += 1
if not first_skipped_line:
first_skipped_line = i + 1
@@ -37,7 +38,7 @@ def __main__():
info_msg = "%i lines converted to BED. " % ( i + 1 - skipped_lines )
if skipped_lines > 0:
info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( skipped_lines, first_skipped_line )
print info_msg
print(info_msg)
if __name__ == "__main__":
__main__()
lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.py
+2 -2
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env python
"""
convert a ref.taxonommy file to a seq.taxonomy file
convert a ref.taxonomy file to a seq.taxonomy file
Usage:
%python ref_to_seq_taxonomy_converter.py <ref.taxonom> <seq.taxonomy>
"""
@@ -14,7 +14,7 @@ assert sys.version_info[:2] >= (2, 4)
def __main__():
infile_name = sys.argv[1]
outfile = open(sys.argv[2], 'w')
for i, line in enumerate(file(infile_name)):
for i, line in enumerate(open(infile_name)):
line = line.rstrip()
if line and not line.startswith('#'):
fields = line.split('\t')
lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py
+3 -2
View File
@@ -1,11 +1,12 @@
#!/usr/bin/env python
# code is same as ~/tools/stats/wiggle_to_simple.py
"""
Read a wiggle track and print out a series of lines containing
"chrom position score". Ignores track lines, handles bed, variableStep
and fixedStep wiggle lines.
"""
from __future__ import print_function
import sys
import bx.wiggle
@@ -34,7 +35,7 @@ def main():
out_file.write( "%s\n" % "\t".join( map( str, fields ) ) )
except UCSCLimitException:
# Wiggle data was truncated, at the very least need to warn the user.
print 'Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.'
print('Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.')
except ValueError as e:
in_file.close()
out_file.close()
lib/galaxy/datatypes/data.py
+7 -12
View File
@@ -312,7 +312,7 @@ class Data( object ):
trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (dataset.hid, fname, to_ext)
return open( dataset.file_name )
def display_data(self, trans, data, preview=False, filename=None, to_ext=None, size=None, offset=None, **kwd):
def display_data(self, trans, data, preview=False, filename=None, to_ext=None, **kwd):
""" Old display method, for transition - though still used by API and
test framework. Datatypes should be very careful if overridding this
method and this interface between datatypes and Galaxy will likely
@@ -974,29 +974,24 @@ def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skip
file_type = None
data_checked = False
temp = open( file_name, "U" )
last_line = ''
while count <= LINE_COUNT:
line = last_line + temp.readline( WIDTH - len( last_line ) )
while count < LINE_COUNT:
line = temp.readline( WIDTH )
if line and not is_multi_byte and not data_checked:
# See if we have a compressed or binary file
if line[0:2] == util.gzip_magic:
file_type = 'gzipped'
break
else:
for char in line:
if ord( char ) > 128:
file_type = 'binary'
break
data_checked = True
if file_type in [ 'gzipped', 'binary' ]:
break
if file_type in [ 'gzipped', 'binary' ]:
break
if not line_wrap:
if '\n' in line:
i = line.index( '\n' )
last_line = line[i:]
line = line[:i]
if line.endswith('\n'):
line = line[:-1]
else:
last_line = ''
while True:
i = temp.read(1)
if not i or i == '\n':
lib/galaxy/datatypes/dataproviders/column.py
+5 -5
View File
@@ -3,10 +3,10 @@ Providers that provide lists of lists generally where each line of a source
is further subdivided into multiple data (e.g. columns from a line).
"""
import urllib
from six.moves.urllib.parse import unquote_plus
import re
import line
from . import line
_TODO = """
move ColumnarDataProvider parsers to more sensible location
@@ -96,7 +96,7 @@ class ColumnarDataProvider( line.RegexLineDataProvider ):
self.column_count = len( self.column_types )
# if no indeces given, infer from column_count
if not self.selected_column_indeces and self.column_count:
self.selected_column_indeces = list( xrange( self.column_count ) )
self.selected_column_indeces = list( range( self.column_count ) )
self.deliminator = deliminator
@@ -182,7 +182,7 @@ class ColumnarDataProvider( line.RegexLineDataProvider ):
elif 'has' == op:
return lambda d: val in d[column]
elif 're' == op:
val = urllib.unquote_plus( val )
val = unquote_plus( val )
val = re.compile( val )
return lambda d: val.match( d[column] ) is not None
return None
@@ -258,7 +258,7 @@ class ColumnarDataProvider( line.RegexLineDataProvider ):
# TODO: too much going on in this loop - the above should all be precomputed AMAP...
all_columns = line.split( self.deliminator )
# if no indeces were passed to init, return all columns
selected_indeces = self.selected_column_indeces or list( xrange( len( all_columns ) ) )
selected_indeces = self.selected_column_indeces or list( range( len( all_columns ) ) )
parsed_columns = []
for parser_index, column_index in enumerate( selected_indeces ):
parsed_columns.append( self.parse_column_at_index( all_columns, parser_index, column_index ) )
lib/galaxy/datatypes/dataproviders/line.py
+2 -2
View File
@@ -4,7 +4,7 @@ Dataproviders that iterate over lines from their sources.
import collections
import re
import base
from . import base
import logging
log = logging.getLogger( __name__ )
@@ -241,7 +241,7 @@ class BlockDataProvider( base.LimitedOffsetDataProvider ):
Called per block (just before providing).
"""
# empty block_lines and assemble block
return list( ( self.block_lines.popleft() for i in xrange( len( self.block_lines ) ) ) )
return list( ( self.block_lines.popleft() for i in range( len( self.block_lines ) ) ) )
def filter_block( self, block ):
"""
lib/galaxy/datatypes/molecules.py
+1 -1
View File
@@ -393,7 +393,7 @@ class OBFS(Binary):
return "OpenBabel Fastsearch Index"
def display_data(self, trans, data, preview=False, filename=None,
to_ext=None, size=None, offset=None, **kwd):
to_ext=None, **kwd):
"""Apparently an old display method, but still gets called.
This allows us to format the data shown in the central pane via the "eye" icon.
lib/galaxy/datatypes/sequence.py
+4
View File
@@ -10,6 +10,8 @@ import re
import string
from cgi import escape
from six import PY3
from galaxy import util
from galaxy.datatypes import metadata
from galaxy.util.checkers import is_gzip
@@ -21,6 +23,8 @@ from . import data
import bx.align.maf
if PY3:
long = int
log = logging.getLogger(__name__)
lib/galaxy/datatypes/sniff.py
+2 -2
View File
@@ -198,7 +198,7 @@ def get_headers( fname, sep, count=60, is_multi_byte=False ):
[['chr7', '127475281', '127491632', 'NM_000230', '0', '+', '127486022', '127488767', '0', '3', '29,172,3225,', '0,10713,13126,'], ['chr7', '127486011', '127488900', 'D49487', '0', '+', '127486022', '127488767', '0', '2', '155,490,', '0,2399']]
"""
headers = []
for idx, line in enumerate(file(fname)):
for idx, line in enumerate(open(fname)):
line = line.rstrip('\n\r')
if is_multi_byte:
# TODO: fix this - sep is never found in line
@@ -470,7 +470,7 @@ DECOMPRESSION_FUNCTIONS = dict( gzip=gzip.GzipFile )
COMPRESSION_CHECK_FUNCTIONS = [ ( 'gzip', is_gzip ) ]
COMPRESSION_DATATYPES = dict( gzip=[ 'bam' ] )
COMPRESSED_EXTENSIONS = []
for exts in COMPRESSION_DATATYPES.itervalues():
for exts in COMPRESSION_DATATYPES.values():
COMPRESSED_EXTENSIONS.extend( exts )
lib/galaxy/datatypes/tabular.py
+26 -24
View File
@@ -13,6 +13,8 @@ import tempfile
from cgi import escape
from json import dumps
from six import PY3
from galaxy import util
from galaxy.datatypes import data, metadata
from galaxy.datatypes.metadata import MetadataElement
@@ -21,6 +23,9 @@ from galaxy.util.checkers import is_gzip
from . import dataproviders
if PY3:
long = int
log = logging.getLogger(__name__)
@@ -53,26 +58,23 @@ class TabularData( data.Text ):
except:
return False
def get_chunk(self, trans, dataset, chunk):
ck_index = int(chunk)
f = open(dataset.file_name)
f.seek(ck_index * trans.app.config.display_chunk_size)
# If we aren't at the start of the file, seek to next newline. Do this better eventually.
if f.tell() != 0:
cursor = f.read(1)
while cursor and cursor != '\n':
def get_chunk(self, trans, dataset, offset=0, ck_size=None):
with open(dataset.file_name) as f:
f.seek(offset)
ck_data = f.read(ck_size or trans.app.config.display_chunk_size)
if ck_data and ck_data[-1] != '\n':
cursor = f.read(1)
ck_data = f.read(trans.app.config.display_chunk_size)
cursor = f.read(1)
while cursor and ck_data[-1] != '\n':
ck_data += cursor
cursor = f.read(1)
return dumps( { 'ck_data': util.unicodify( ck_data ), 'ck_index': ck_index + 1 } )
while cursor and cursor != '\n':
ck_data += cursor
cursor = f.read(1)
last_read = f.tell()
return dumps( { 'ck_data': util.unicodify( ck_data ),
'offset': last_read } )
def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None, **kwd):
def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, offset=None, ck_size=None, **kwd):
preview = util.string_as_bool( preview )
if chunk:
return self.get_chunk(trans, dataset, chunk)
if offset is not None:
return self.get_chunk(trans, dataset, offset, ck_size)
elif to_ext or not preview:
to_ext = to_ext or dataset.extension
return self._serve_raw(trans, dataset, to_ext)
@@ -853,10 +855,10 @@ class Eland( Tabular ):
dataset.metadata.comment_lines = 0
dataset.metadata.columns = 21
dataset.metadata.column_types = ['str', 'int', 'int', 'int', 'int', 'int', 'str', 'int', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str']
dataset.metadata.lanes = lanes.keys()
dataset.metadata.lanes = list(lanes.keys())
dataset.metadata.tiles = ["%04d" % int(t) for t in tiles.keys()]
dataset.metadata.barcodes = filter(lambda x: x != '0', barcodes.keys()) + ['NoIndex' for x in barcodes.keys() if x == '0']
dataset.metadata.reads = reads.keys()
dataset.metadata.barcodes = [_ for _ in barcodes.keys() if _ != '0'] + ['NoIndex' for _ in barcodes.keys() if _ == '0']
dataset.metadata.reads = list(reads.keys())
class ElandMulti( Tabular ):
@@ -919,7 +921,7 @@ class BaseCSV( TabularData ):
# check the dialect works
reader = csv.reader(open(filename, 'r'), self.dialect)
# Check we can read header and get columns
header_row = reader.next()
header_row = next(reader)
if len(header_row) < 2:
# No columns so not separated by this dialect.
return False
@@ -937,7 +939,7 @@ class BaseCSV( TabularData ):
if not found_second_line:
return False
else:
data_row = reader.next()
data_row = next(reader)
if len(data_row) < 2:
# No columns so not separated by this dialect.
return False
@@ -975,8 +977,8 @@ class BaseCSV( TabularData ):
data_row = None
header_row = None
try:
header_row = reader.next()
data_row = reader.next()
header_row = next(reader)
data_row = next(reader)
for row in reader:
pass
except csv.Error as e:
lib/galaxy/datatypes/text.py
+6 -8
View File
@@ -127,19 +127,17 @@ class Ipynb( Json ):
except:
return False
def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None, **kwd):
def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, **kwd):
config = trans.app.config
trust = getattr( config, 'trust_ipython_notebook_conversion', False )
if trust:
return self._display_data_trusted(trans, dataset, preview=preview, filename=filename, to_ext=to_ext, chunk=chunk, **kwd)
return self._display_data_trusted(trans, dataset, preview=preview, filename=filename, to_ext=to_ext, **kwd)
else:
return super(Ipynb, self).display_data( trans, dataset, preview=preview, filename=filename, to_ext=to_ext, chunk=chunk, **kwd )
return super(Ipynb, self).display_data( trans, dataset, preview=preview, filename=filename, to_ext=to_ext, **kwd )
def _display_data_trusted(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None, **kwd):
def _display_data_trusted(self, trans, dataset, preview=False, filename=None, to_ext=None, **kwd):
preview = string_as_bool( preview )
if chunk:
return self.get_chunk(trans, dataset, chunk)
elif to_ext or not preview:
if to_ext or not preview:
return self._serve_raw(trans, dataset, to_ext)
else:
ofile_handle = tempfile.NamedTemporaryFile(delete=False)
@@ -426,7 +424,7 @@ class SnpEffDb( Text ):
# search data_dir/genome_version for files
regulation_pattern = 'regulation_(.+).bin'
# annotation files that are included in snpEff by a flag
annotations_dict = {'nextProt.bin' : '-nextprot', 'motif.bin': '-motif'}
annotations_dict = {'nextProt.bin': '-nextprot', 'motif.bin': '-motif'}
regulations = []
annotations = []
genome_version = None
lib/galaxy/datatypes/util/gff_util.py
+1
View File
@@ -149,6 +149,7 @@ class GFFReaderWrapper( NiceReaderWrapper ):
self.default_strand, fix_strand=self.fix_strand )
return interval
# For Python3 this needs to be changed to __next__() after bx-python library is ported too
def next( self ):
""" Returns next GFFFeature. """
lib/galaxy/dependencies/pinned-requirements.txt
+1
View File
@@ -29,6 +29,7 @@ nose==1.3.7
Parsley==1.3
six==1.9.0
Whoosh==2.7.4
testfixtures==4.10.0
# Cheetah and dependencies
Cheetah==2.4.4
lib/galaxy/jobs/actions/post.py
+10 -3
View File
@@ -427,13 +427,20 @@ class DeleteIntermediatesAction(DefaultJobAction):
else:
log.debug("No job found yet for wfi_step %s, (step %s)" % (wfi_step, wfi_step.workflow_step))
for j2c in jobs_to_check:
sa_session.refresh(j2c)
creating_jobs = []
for input_dataset in j2c.input_datasets:
if not input_dataset.dataset:
log.debug("PJA Async Issue: No dataset attached to input_dataset %s during handling of workflow invocation %s" % (input_dataset.id, wfi))
elif not input_dataset.dataset.creating_job:
log.debug("PJA Async Issue: No creating job attached to dataset %s during handling of workflow invocation %s" % (input_dataset.dataset.id, wfi))
else:
creating_jobs.append((input_dataset, input_dataset.dataset.creating_job))
for (input_dataset, creating_job) in creating_jobs:
sa_session.refresh(creating_job)
sa_session.refresh(input_dataset)
sa_session.refresh(input_dataset.dataset.creating_job)
creating_jobs.append( (input_dataset, input_dataset.dataset.creating_job) )
for input_dataset in [x.dataset for (x, creating_job) in creating_jobs if creating_job.workflow_invocation_step and creating_job.workflow_invocation_step.workflow_invocation == wfi]:
# note that the above input_dataset is a reference to a
# job.input_dataset.dataset at this point
safe_to_delete = True
for job_to_check in [d_j.job for d_j in input_dataset.dependent_jobs]:
if job_to_check != job and job_to_check.state not in [job.states.OK, job.states.DELETED]:
lib/galaxy/jobs/dynamic_tool_destination.py
Executable
+1413
View File
File diff suppressed because it is too large Load Diff
lib/galaxy/jobs/mapper.py
+2
View File
@@ -5,6 +5,7 @@ import sys
import galaxy.jobs.rules
from galaxy.jobs import stock_rules
from galaxy.jobs.dynamic_tool_destination import map_tool_to_destination
from .rule_helper import RuleHelper
log = logging.getLogger( __name__ )
@@ -33,6 +34,7 @@ STOCK_RULES = dict(
choose_one=stock_rules.choose_one,
burst=stock_rules.burst,
docker_dispatch=stock_rules.docker_dispatch,
dtd=map_tool_to_destination,
)
lib/galaxy/jobs/runners/kubernetes.py
+70 -15
View File
@@ -7,6 +7,7 @@ import logging
from galaxy import model
from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner
from os import environ as os_environ
from six import text_type
# pykube imports:
try:
@@ -38,6 +39,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
assert operator is not None, K8S_IMPORT_MESSAGE
runner_param_specs = dict(
k8s_config_path=dict(map=str, default=os_environ.get('KUBECONFIG', None)),
k8s_use_service_account=dict(map=bool, default=False),
k8s_persistent_volume_claim_name=dict(map=str),
k8s_persistent_volume_claim_mount_path=dict(map=str),
k8s_namespace=dict(map=str, default="default"),
@@ -52,8 +54,10 @@ class KubernetesJobRunner(AsynchronousJobRunner):
# self.cli_interface = CliInterface()
# here we need to fetch the default kubeconfig path from the plugin defined in job_conf...
self._pykube_api = HTTPClient(KubeConfig.from_file(self.runner_params["k8s_config_path"]))
if "k8s_use_service_account" in self.runner_params and self.runner_params["k8s_use_service_account"]:
self._pykube_api = HTTPClient(KubeConfig.from_service_account())
else:
self._pykube_api = HTTPClient(KubeConfig.from_file(self.runner_params["k8s_config_path"]))
self._galaxy_vol_name = "pvc-galaxy" # TODO this needs to be read from params!!
self._init_monitor_thread()
@@ -71,7 +75,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
job_destination = job_wrapper.job_destination
# Construction of the Kubernetes Job object follows: http://kubernetes.io/docs/user-guide/persistent-volumes/
k8s_job_name = self.__produce_unique_k8s_job_name(job_wrapper)
k8s_job_name = self.__produce_unique_k8s_job_name(job_wrapper.get_id_tag())
k8s_job_obj = {
"apiVersion": "extensions/v1beta1",
"kind": "Job",
@@ -87,7 +91,14 @@ class KubernetesJobRunner(AsynchronousJobRunner):
"spec": self.__get_k8s_job_spec(job_wrapper)
}
# Checks if job exists
job = Job(self._pykube_api, k8s_job_obj)
if job.exists():
job.delete()
# Creates the Kubernetes Job
# TODO if a job with that ID exists, what should we do?
# TODO do we trust that this is the same job and use that?
# TODO or create a new job as we cannot make sure
Job(self._pykube_api, k8s_job_obj).create()
# define job attributes in the AsyncronousJobState for follow-up
@@ -99,9 +110,9 @@ class KubernetesJobRunner(AsynchronousJobRunner):
external_runjob_script = None
return external_runjob_script
def __produce_unique_k8s_job_name(self, job_wrapper):
def __produce_unique_k8s_job_name(self, galaxy_internal_job_id):
# wrapper.get_id_tag() instead of job_id for compatibility with TaskWrappers.
return "galaxy-" + job_wrapper.get_id_tag()
return "galaxy-" + galaxy_internal_job_id
def __get_k8s_job_spec(self, job_wrapper):
"""Creates the k8s Job spec. For a Job spec, the only requirement is to have a .spec.template."""
@@ -114,7 +125,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
(see pod selector) and an appropriate restart policy."""
k8s_spec_template = {
"metadata": {
"labels": {"app": self.__produce_unique_k8s_job_name(job_wrapper)}
"labels": {"app": self.__produce_unique_k8s_job_name(job_wrapper.get_id_tag())}
},
"spec": {
"volumes": self.__get_k8s_mountable_volumes(job_wrapper),
@@ -155,6 +166,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
# to assemble the command.
# TODO possibly shell needs to be set by job_wrapper
"command": ["/bin/bash", "-c", job_wrapper.runner_command_line],
"workingDir": job_wrapper.working_directory,
"volumeMounts": [{
"mountPath": self.runner_params['k8s_persistent_volume_claim_mount_path'],
"name": self._galaxy_vol_name
@@ -200,9 +212,17 @@ class KubernetesJobRunner(AsynchronousJobRunner):
jobs = Job.objects(self._pykube_api).filter(selector="app=" + job_state.job_id)
if len(jobs.response['items']) == 1:
job = Job(self._pykube_api, jobs.response['items'][0])
job_destination = job_state.job_wrapper.job_destination
succeeded = 0
active = 0
failed = 0
max_pod_retrials = 1
if 'k8s_pod_retrials' in self.runner_params:
max_pod_retrials = int(self.runner_params['k8s_pod_retrials'])
if 'max_pod_retrials' in job_destination.params:
max_pod_retrials = int(job_destination.params['max_pod_retrials'])
if 'succeeded' in job.obj['status']:
succeeded = job.obj['status']['succeeded']
if 'active' in job.obj['status']:
@@ -219,16 +239,16 @@ class KubernetesJobRunner(AsynchronousJobRunner):
job_state.running = False
self.mark_as_finished(job_state)
return None
elif active > 0 or succeeded + active + failed == 0:
elif active > 0 and failed <= max_pod_retrials:
job_state.running = True
return job_state
elif failed > job_state.job_destination.params['max_pod_retrials']:
elif failed > max_pod_retrials:
self.__produce_log_file(job_state)
error_file = open(job_state.error_file, 'w')
error_file.write("Exceeded max number of Kubernetes pod retrials allowed for job\n")
error_file.close()
job_state.running = False
job_state.fail_message = "More pods failed than allowed. See stdout for pods details."
self.mark_as_failed(job_state)
job.scale(replicas=0)
return None
@@ -256,16 +276,50 @@ class KubernetesJobRunner(AsynchronousJobRunner):
self.mark_as_failed(job_state)
return job_state
def fail_job(self, job_state):
"""
Kubernetes runner overrides fail_job (called by mark_as_failed) to rescue the pod's log files which are left as
stdout (pods logs are the natural stdout and stderr of the running processes inside the pods) and are
deleted in the parent implementation as part of the failing the job process.
:param job_state:
:return:
"""
# First we rescue the pods logs
with open(job_state.output_file, 'r') as outfile:
stdout_content = outfile.read()
if getattr(job_state, 'stop_job', True):
self.stop_job(self.sa_session.query(self.app.model.Job).get(job_state.job_wrapper.job_id))
self._handle_runner_state('failure', job_state)
# Not convinced this is the best way to indicate this state, but
# something necessary
if not job_state.runner_state_handled:
job_state.job_wrapper.fail(
message=getattr(job_state, 'fail_message', 'Job failed'),
stdout=stdout_content, stderr='See stdout for pod\'s stderr.'
)
if job_state.job_wrapper.cleanup_job == "always":
job_state.cleanup()
def __produce_log_file(self, job_state):
pod_r = Pod.objects(self._pykube_api).filter(selector="app=" + job_state.job_id)
logs = ""
for pod_obj in pod_r.response['items']:
pod = Pod(self._pykube_api, pod_obj)
logs += "\n\n==== Pod " + pod.name + " log start ====\n\n"
logs += pod.logs(timestamps=True)
logs += "\n\n==== Pod " + pod.name + " log end ===="
try:
pod = Pod(self._pykube_api, pod_obj)
logs += "\n\n==== Pod " + pod.name + " log start ====\n\n"
logs += pod.logs(timestamps=True)
logs += "\n\n==== Pod " + pod.name + " log end ===="
except Exception as detail:
log.info("Could not write pod\'s " + pod_obj['metadata']['name'] +
" log file due to HTTPError " + str(detail))
logs_file_path = job_state.output_file
logs_file = open(logs_file_path, mode="w")
if isinstance(logs, text_type):
logs = logs.encode('utf8')
logs_file.write(logs)
logs_file.close()
return logs_file_path
@@ -273,8 +327,9 @@ class KubernetesJobRunner(AsynchronousJobRunner):
def stop_job(self, job):
"""Attempts to delete a dispatched job to the k8s cluster"""
try:
jobs = Job.objects(self._pykube_api).filter(selector="app=" + job.job_runner_external_id)
if jobs.response['items'].len() >= 0:
jobs = Job.objects(self._pykube_api).filter(selector="app=" +
self.__produce_unique_k8s_job_name(job.get_id_tag()))
if len(jobs.response['items']) >= 0:
job_to_delete = Job(self._pykube_api, jobs.response['items'][0])
job_to_delete.scale(replicas=0)
# TODO assert whether job parallelism == 0
lib/galaxy/managers/base.py
+8 -7
View File
@@ -25,16 +25,17 @@ attribute change to a model object.
# instead of the three separate classes. With no 'apparent' perfect scheme
# I'm opting to just keep them separate.
import datetime
import logging
import re
import sqlalchemy
import routes
import sqlalchemy
from six import string_types
from galaxy import exceptions
from galaxy import model
from galaxy.model import tool_shed_install
import logging
log = logging.getLogger( __name__ )
@@ -375,7 +376,7 @@ class ModelManager( object ):
"""
# cpu-expensive
for item in items:
filter_results = map( lambda f: f( item ), filters )
filter_results = [f( item ) for f in filters]
if all( filter_results ):
yield item
@@ -735,7 +736,7 @@ class ModelDeserializer( HasAModelManager ):
"""
# TODO:?? a larger question is: which should be first? Deserialize then validate - or - validate then deserialize?
def __init__( self, app, **kwargs ):
def __init__( self, app, validator=None, **kwargs ):
"""
Set up deserializers and validator.
"""
@@ -746,7 +747,7 @@ class ModelDeserializer( HasAModelManager ):
self.deserializable_keyset = set([])
self.add_deserializers()
# a sub object that can validate incoming values
self.validate = ModelValidator( self.app )
self.validate = validator or ModelValidator( self.app )
def add_deserializers( self ):
"""
@@ -837,7 +838,7 @@ class ModelValidator( HasAModelManager ):
# validators for primitives and compounds of primitives
def basestring( self, key, val ):
return self.type( key, val, basestring )
return self.type( key, val, string_types )
def bool( self, key, val ):
return self.type( key, val, bool )
@@ -849,7 +850,7 @@ class ModelValidator( HasAModelManager ):
"""
Must be a basestring or None.
"""
return self.type( key, val, ( basestring, type( None ) ) )
return self.type( key, val, ( string_types, type( None ) ) )
def int_range( self, key, val, min=None, max=None ):
"""
lib/galaxy/managers/context.py
+4 -3
View File
@@ -1,9 +1,10 @@
"""
Mixins for transaction-like objects.
"""
from json import dumps
import string
from json import dumps
from six import text_type
from galaxy.util import bunch
@@ -20,7 +21,7 @@ class ProvidesAppContext( object ):
Application-level logging of user actions.
"""
if self.app.config.log_actions:
action = self.app.model.UserAction(action=action, context=context, params=unicode( dumps( params ) ) )
action = self.app.model.UserAction(action=action, context=context, params=text_type( dumps( params ) ) )
try:
if user:
action.user = user
lib/galaxy/managers/users.py
+23 -1
View File
@@ -11,6 +11,7 @@ from galaxy import util
from galaxy.managers import base
from galaxy.managers import deletable
from galaxy.managers import api_keys
from galaxy.security import validate_user_input
import logging
log = logging.getLogger( __name__ )
@@ -26,7 +27,6 @@ class UserManager( base.ModelManager, deletable.PurgableManagerMixin ):
# TODO: incorp BaseAPIController.validate_in_users_and_groups
# TODO: incorp CreatesUsersMixin
# TODO: incorp CreatesApiKeysMixin
# TODO: incorporate security/validate_user_input.py
# TODO: incorporate UsesFormDefinitionsMixin?
def create( self, webapp_name=None, **kwargs ):
@@ -288,6 +288,28 @@ class UserSerializer( base.ModelSerializer, deletable.PurgableSerializerMixin ):
})
class UserDeserializer( base.ModelDeserializer ):
"""
Service object for validating and deserializing dictionaries that
update/alter users.
"""
model_manager_class = UserManager
def add_deserializers( self ):
super( UserDeserializer, self ).add_deserializers()
self.deserializers.update({
'username' : self.deserialize_username,
})
def deserialize_username( self, item, key, username, trans=None, **context ):
# TODO: validate_user_input requires trans and should(?) raise exceptions
# move validation to UserValidator and use self.app, exceptions instead
validation_error = validate_user_input.validate_publicname( trans, username, user=item )
if validation_error:
raise base.ModelDeserializingError( validation_error )
return self.default_deserializer( item, key, username, trans=trans, **context )
class CurrentUserSerializer( UserSerializer ):
model_manager_class = UserManager
lib/galaxy/model/__init__.py
+35 -26
View File
@@ -14,35 +14,32 @@ import os
import socket
import time
from datetime import datetime, timedelta
from itertools import ifilter, imap
from string import Template
from uuid import UUID, uuid4
from six import string_types
from sqlalchemy import and_, func, not_, or_, true, join, select
from sqlalchemy.orm import joinedload, object_session, aliased
from sqlalchemy import (and_, func, join, not_, or_, select, true, type_coerce,
types)
from sqlalchemy.ext import hybrid
from sqlalchemy import types
from sqlalchemy import type_coerce
from sqlalchemy.orm import aliased, joinedload, object_session
import galaxy.model.orm.now
import galaxy.model.metadata
import galaxy.model.orm.now
import galaxy.security.passwords
import galaxy.util
from galaxy.model.item_attrs import UsesAnnotations
from galaxy.util.dictifiable import Dictifiable
from galaxy.security import get_permitted_actions
from galaxy.util import Params, restore_text, send_mail
from galaxy.util import ready_name_for_url, unique_id
from galaxy.util import unicodify, directory_hash_id
from galaxy.util.multi_byte import is_multi_byte
from galaxy.util.hash_util import new_secure_hash
from galaxy.util import (directory_hash_id, Params, ready_name_for_url,
restore_text, send_mail, unicodify, unique_id)
from galaxy.util.bunch import Bunch
from galaxy.util.dictifiable import Dictifiable
from galaxy.util.hash_util import new_secure_hash
from galaxy.util.multi_byte import is_multi_byte
from galaxy.util.sanitize_html import sanitize_html
from galaxy.web.framework.helpers import to_unicode
from galaxy.web.form_builder import (AddressField, CheckboxField, HistoryField,
PasswordField, SelectField, TextArea, TextField, WorkflowField,
WorkflowMappingField)
from galaxy.web.framework.helpers import to_unicode
log = logging.getLogger( __name__ )
@@ -166,6 +163,7 @@ class User( object, Dictifiable ):
self.active = False
self.activation_token = None
self.username = None
self.last_password_change = None
# Relationships
self.histories = []
self.credentials = []
@@ -179,6 +177,7 @@ class User( object, Dictifiable ):
self.password = galaxy.security.passwords.hash_password( cleartext )
else:
self.password = new_secure_hash( text_type=cleartext )
self.last_password_change = datetime.now()
def check_password( self, cleartext ):
"""
@@ -1184,7 +1183,7 @@ class History( object, Dictifiable, UsesAnnotations, HasName ):
""" Optimized version of add_dataset above that minimizes database
interactions when adding many datasets to history at once.
"""
all_hdas = all( imap( is_hda, datasets ) )
all_hdas = all( is_hda(_) for _ in datasets )
optimize = len( datasets) > 1 and parent_id is None and all_hdas and set_hid
if optimize:
self.__add_datasets_optimized( datasets, genome_build=genome_build )
@@ -1442,7 +1441,7 @@ class History( object, Dictifiable, UsesAnnotations, HasName ):
if len(ids) < max_in_filter_length:
query = query.filter( content_class.id.in_(ids) )
else:
query = ifilter(lambda content: content.id in ids, query)
query = (content for content in query if content.id in ids)
return query
def __collection_contents_iter( self, **kwds ):
@@ -2064,7 +2063,7 @@ class DatasetInstance( object ):
raise NoConverterException("A dependency (%s) is missing a converter." % dependency)
except KeyError:
pass # No deps
new_dataset = self.datatype.convert_dataset( trans, self, target_ext, return_output=True, visible=False, deps=deps, set_output_history=True ).values()[0]
new_dataset = next(iter(self.datatype.convert_dataset( trans, self, target_ext, return_output=True, visible=False, deps=deps, set_output_history=True ).values()))
assoc = ImplicitlyConvertedDatasetAssociation( parent=self, file_type=target_ext, dataset=new_dataset, metadata_safe=False )
session = trans.sa_session
session.add( new_dataset )
@@ -2213,7 +2212,7 @@ class DatasetInstance( object ):
"""
data_sources_dict = {}
msg = None
for source_type, source_list in self.datatype.data_sources.iteritems():
for source_type, source_list in self.datatype.data_sources.items():
data_source = None
if source_type == "data_standalone":
# Nothing to do.
@@ -2566,9 +2565,9 @@ class Library( object, Dictifiable, HasName ):
# (seq[i].attr, i, seq[i]) and sort it. The second item of tuple is needed not
# only to provide stable sorting, but mainly to eliminate comparison of objects
# (which can be expensive or prohibited) in case of equal attribute values.
intermed = map( None, map( getattr, seq, ( attr, ) * len( seq ) ), xrange( len( seq ) ), seq )
intermed = map( None, (getattr(_, attr) for _ in seq), range( len( seq ) ), seq )
intermed.sort()
return map( operator.getitem, intermed, ( -1, ) * len( intermed ) )
return [_[-1] for _ in intermed]
if folders is None:
active_folders = [ folder ]
for active_folder in folder.active_folders:
@@ -3110,11 +3109,17 @@ class DatasetCollection( object, Dictifiable, UsesAnnotations ):
@property
def populated( self ):
return self.populated_state == DatasetCollection.populated_states.OK
top_level_populated = self.populated_state == DatasetCollection.populated_states.OK
if top_level_populated and self.has_subcollections:
return all(map(lambda e: e.child_collection.populated, self.elements))
return top_level_populated
@property
def waiting_for_elements( self ):
return self.populated_state == DatasetCollection.populated_states.NEW
top_level_waiting = self.populated_state == DatasetCollection.populated_states.NEW
if not top_level_waiting and self.has_subcollections:
return any(map(lambda e: e.child_collection.waiting_for_elements, self.elements))
return top_level_waiting
def mark_as_populated( self ):
self.populated_state = DatasetCollection.populated_states.OK
@@ -3179,6 +3184,10 @@ class DatasetCollection( object, Dictifiable, UsesAnnotations ):
# Nothing currently editable in this class.
return {}
@property
def has_subcollections(self):
return ":" in self.collection_type
class DatasetCollectionInstance( object, HasName ):
"""
@@ -3232,7 +3241,7 @@ class DatasetCollectionInstance( object, HasName ):
changed = self.collection.set_from_dict( new_data )
# unknown keys are ignored here
for key in [ k for k in new_data.keys() if k in self.editable_keys ]:
for key in ( k for k in new_data.keys() if k in self.editable_keys ):
new_val = new_data[ key ]
old_val = self.__getattribute__( key )
if new_val == old_val:
@@ -3696,7 +3705,7 @@ class WorkflowStep( object ):
outputs[output_name] = workflow_output
else:
outputs[output_name] = workflow_output
return outputs.values()
return list(outputs.values())
@property
def content_id( self ):
@@ -3954,7 +3963,7 @@ class WorkflowInvocation( object, Dictifiable ):
).filter( and_( *and_conditions ) )
# Immediately just load all ids into memory so time slicing logic
# is relatively intutitive.
return map( lambda wi: wi.id, query.all() )
return [wi.id for wi in query.all()]
def to_dict( self, view='collection', value_mapper=None, step_details=False ):
rval = super( WorkflowInvocation, self ).to_dict( view=view, value_mapper=value_mapper )
@@ -5163,4 +5172,4 @@ def copy_list(lst, *args, **kwds):
if lst is None:
return lst
else:
return list(map(lambda el: el.copy(*args, **kwds), lst))
return [el.copy(*args, **kwds) for el in lst]
lib/galaxy/model/mapping.py
+1
View File
@@ -33,6 +33,7 @@ model.User.table = Table(
Column( "email", TrimmedString( 255 ), index=True, nullable=False ),
Column( "username", TrimmedString( 255 ), index=True, unique=True ),
Column( "password", TrimmedString( 255 ), nullable=False ),
Column( "last_password_change", DateTime, default=now ),
Column( "external", Boolean, default=False ),
Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ),
Column( "deleted", Boolean, index=True, default=False ),
lib/galaxy/model/migrate/versions/0006_change_qual_datatype.py
+6 -4
View File
@@ -2,6 +2,8 @@
This migration script changes certain values in the history_dataset_association.extension
column, specifically 'qual' is chaged to be 'qual454'.
"""
from __future__ import print_function
import logging
import sys
@@ -20,10 +22,10 @@ metadata = MetaData()
def display_migration_details():
print "========================================"
print "This migration script changes certain values in the history_dataset_association.extension"
print "column, specifically 'qual' is chaged to be 'qual454'."
print "========================================"
print("========================================")
print("This migration script changes certain values in the history_dataset_association.extension")
print("column, specifically 'qual' is chaged to be 'qual454'.")
print("========================================")
def upgrade(migrate_engine):
lib/galaxy/model/migrate/versions/0007_sharing_histories.py
+7 -5
View File
@@ -3,6 +3,8 @@ This migration script creates the new history_user_share_association table, and
a new boolean type column to the history table. This provides support for sharing
histories in the same way that workflows are shared.
"""
from __future__ import print_function
import logging
import sys
@@ -21,11 +23,11 @@ metadata = MetaData()
def display_migration_details():
print "========================================"
print "This migration script creates the new history_user_share_association table, and adds"
print "a new boolean type column to the history table. This provides support for sharing"
print "histories in the same way that workflows are shared."
print "========================================"
print("========================================")
print("This migration script creates the new history_user_share_association table, and adds")
print("a new boolean type column to the history table. This provides support for sharing")
print("histories in the same way that workflows are shared.")
print("========================================")
HistoryUserShareAssociation_table = Table( "history_user_share_association", metadata,
Column( "id", Integer, primary_key=True ),
lib/galaxy/model/migrate/versions/0008_galaxy_forms.py
+13 -11
View File
@@ -9,6 +9,8 @@ This migration script adds the following new tables for supporting Galaxy forms:
7) sample_state
8) sample_event
"""
from __future__ import print_function
import datetime
import logging
import sys
@@ -31,17 +33,17 @@ metadata = MetaData()
def display_migration_details():
print "========================================"
print "This migration script adds the following new tables for supporting Galaxy forms:"
print "1) form_definition_current"
print "2) form_definition"
print "3) form_values"
print "4) request_type"
print "5) request"
print "6) sample"
print "7) sample_state"
print "8) sample_event"
print "========================================"
print("========================================")
print("This migration script adds the following new tables for supporting Galaxy forms:")
print("1) form_definition_current")
print("2) form_definition")
print("3) form_values")
print("4) request_type")
print("5) request")
print("6) sample")
print("7) sample_state")
print("8) sample_event")
print("========================================")
FormDefinitionCurrent_table = Table('form_definition_current', metadata,
Column( "id", Integer, primary_key=True),
lib/galaxy/model/migrate/versions/0009_request_table.py
+7 -5
View File
@@ -3,6 +3,8 @@ This migration script adds a new column to 2 tables:
1) a new boolean type column named 'submitted' to the 'request' table
2) a new string type column named 'bar_code' to the 'sample' table
"""
from __future__ import print_function
import logging
import sys
@@ -22,11 +24,11 @@ metadata = MetaData()
def display_migration_details():
print "========================================"
print "This migration script adds a new column to 2 tables:"
print "1) a new boolean type column named 'submitted' to the 'request' table"
print "2) a new string type column named 'bar_code' to the 'sample' table"
print "========================================"
print("========================================")
print("This migration script adds a new column to 2 tables:")
print("1) a new boolean type column named 'submitted' to the 'request' table")
print("2) a new string type column named 'bar_code' to the 'sample' table")
print("========================================")
def upgrade(migrate_engine):
lib/galaxy/model/migrate/versions/0010_hda_display_at_authz_table.py
+11 -9
View File
@@ -8,6 +8,8 @@ history_dataset_association_display_at_authorization table failed: (Operational
(1059, "Identifier name 'ix_history_dataset_association_display_at_authorization_update_time'
is too long
"""
from __future__ import print_function
import datetime
import logging
import sys
@@ -29,15 +31,15 @@ metadata = MetaData()
def display_migration_details():
print "========================================"
print "This migration script adds the history_dataset_association_display_at_authorization table, which"
print "allows 'private' datasets to be displayed at external sites without making them public."
print ""
print "If using mysql, this script will display the following error, which is corrected in the next migration"
print "script: history_dataset_association_display_at_authorization table failed: (OperationalError)"
print "(1059, 'Identifier name 'ix_history_dataset_association_display_at_authorization_update_time'"
print "is too long."
print "========================================"
print("========================================")
print("This migration script adds the history_dataset_association_display_at_authorization table, which")
print("allows 'private' datasets to be displayed at external sites without making them public.")
print("")
print("If using mysql, this script will display the following error, which is corrected in the next migration")
print("script: history_dataset_association_display_at_authorization table failed: (OperationalError)")
print("(1059, 'Identifier name 'ix_history_dataset_association_display_at_authorization_update_time'")
print("is too long.")
print("========================================")
HistoryDatasetAssociationDisplayAtAuthorization_table = Table( "history_dataset_association_display_at_authorization", metadata,
Column( "id", Integer, primary_key=True ),

Some files were not shown because too many files have changed in this diff Show More