Merge branch 'dev' of https://github.com/galaxyproject/galaxy into smash_eggs

2026-09-24 16:30:27 +08:00 · 2015-10-13 16:54:18 -04:00
parent 744835bbfe 9808acf5ae
commit 433b79de2b
471 changed files with 13197 additions and 8397 deletions
@@ -3,10 +3,7 @@ client/
 database/
 doc/patch.py
 doc/source/conf.py
-lib/galaxy/model/migrate/versions/
-lib/galaxy/model/tool_shed_install/migrate/versions/
 lib/galaxy/util/jstree.py
-lib/galaxy/webapps/tool_shed/model/migrate/versions/
 lib/pkg_resources.py
 scripts/api/
 scripts/data_libraries/
@@ -76,6 +76,7 @@ tool-data/shared/igv/igv_build_sites.txt
 tool-data/shared/rviewer/rviewer_build_sites.txt
 tool-data/shared/ncbi/builds.txt
 tool-data/shared/ucsc/builds.txt
+tool-data/shared/ucsc/manual_builds.txt
 tool-data/shared/ensembl/builds.txt
 tool-data/shared/ucsc/publicbuilds.txt
 tool-data/shared/ucsc/ucsc_build_sites.txt
@@ -1,34 +1,38 @@
 define(['utils/utils', 'mvc/tools', 'mvc/upload/upload-view', 'mvc/ui/ui-misc',
-        'mvc/history/options-menu', 'mvc/history/history-panel-edit-current'],
-    function( Utils, Tools, Upload, Ui, optionsMenu, HistoryPanel ) {
+        'mvc/history/options-menu', 'mvc/history/history-panel-edit-current', 'mvc/tools/tools-form'],
+    function( Utils, Tools, Upload, Ui, optionsMenu, HistoryPanel, ToolsForm ) {

    /* Builds the center panel */
    var CenterPanel = Backbone.View.extend({
        initialize: function( options ) {
            this.options = Utils.merge( options, {} );
            this.setElement( this._template() );
-            var params = this.options.params;
-            this.$( '#galaxy_main' ).prop( 'src', Galaxy.root + (
-                ( params.tool_id && ( 'tool_runner?' + $.param( params ) ) ) ||
-                ( params.workflow_id && ( 'workflow/run?id=' + params.workflow_id ) ) ||
-                ( params.m_c && ( params.m_c + '/' + params.m_a ) ) ||
-                ( Galaxy.config.require_login && !Galaxy.user.id && 'user/login') ||
-                'root/welcome'
-            ));
            var self = this;
            this.$( '#galaxy_main' ).on( 'load', function() {
-                $( this ).show();
-                self.$( '#center-panel' ).empty().hide();
-                var galaxy_main = this.contentWindow;
-                if ( galaxy_main ) {
+                var location = this.contentWindow && this.contentWindow.location;
+                if ( location && location.host ) {
+                    $( this ).show();
+                    self.$( '#center-panel' ).empty().hide();
                    Galaxy.trigger( 'galaxy_main:load', {
-                        fullpath: galaxy_main.location.pathname + galaxy_main.location.search + galaxy_main.location.hash,
-                        pathname: galaxy_main.location.pathname,
-                        search  : galaxy_main.location.search,
-                        hash    : galaxy_main.location.hash
+                        fullpath: location.pathname + location.search + location.hash,
+                        pathname: location.pathname,
+                        search  : location.search,
+                        hash    : location.hash
                    });
                }
            });
+            var params = $.extend( {}, Galaxy.params );
+            if ( params.tool_id !== 'upload1' && ( params.tool_id || params.job_id ) ) {
+                params.tool_id && ( params.id = params.tool_id );
+                this.display( ( new ToolsForm.View( params ) ).$el );
+            } else {
+                this.$( '#galaxy_main' ).prop( 'src', Galaxy.root + (
+                    ( params.workflow_id && ( 'workflow/run?id=' + params.workflow_id ) ) ||
+                    ( params.m_c && ( params.m_c + '/' + params.m_a ) ) ||
+                    ( Galaxy.config.require_login && !Galaxy.user.id && 'user/login') ||
+                    'root/welcome'
+                ));
+            }
        },
        display: function( $el ) {
            this.$( '#galaxy_main' ).hide();
@@ -206,4 +210,4 @@ define(['utils/utils', 'mvc/tools', 'mvc/upload/upload-view', 'mvc/ui/ui-misc',
        center  : CenterPanel,
        right   : RightPanel
    }
-});
+});
@@ -8,12 +8,17 @@ define(['utils/utils', 'galaxy.masthead', 'galaxy.menu', 'galaxy.frame',
    return Backbone.View.extend({
        initialize: function( options ) {
            this.options = Utils.merge( options, {} );
-            this.params  = this.options.params;
            this.setElement( this._template( options ) );

            // register this view
            Galaxy.app = this;

+            // url request parameters
+            Galaxy.params = this.options.params;
+
+            // shared backbone router
+            Galaxy.router = new Backbone.Router();
+
            // configure body
            $( 'body' ).append( this.$el );
            ensure_dd_helper();
@@ -131,4 +136,4 @@ define(['utils/utils', 'galaxy.masthead', 'galaxy.menu', 'galaxy.frame',
                    '</div>';
        }
    });
-});
+});
@@ -56,6 +56,8 @@ var DCEListItemEdit = DCEListItemView.extend(


 //==============================================================================
+// NOTE: this does not inherit from DatasetDCEListItemView as you would expect
+//TODO: but should - if we can find something simpler than using diamond
 /** @class Editable view for a DatasetCollectionElement that is also an DatasetAssociation
 *      (a dataset contained in a dataset collection).
 */
@@ -70,6 +72,24 @@ var DatasetDCEListItemEdit = DATASET_LI_EDIT.DatasetListItemEdit.extend(
        DATASET_LI_EDIT.DatasetListItemEdit.prototype.initialize.call( this, attributes );
    },

+    // NOTE: this does not inherit from DatasetDCEListItemView - so we duplicate this here
+    //TODO: fix
+    /** In this override, only get details if in the ready state.
+     *  Note: fetch with no 'change' event triggering to prevent automatic rendering.
+     */
+    _fetchModelDetails : function(){
+        var view = this;
+        if( view.model.inReadyState() && !view.model.hasDetails() ){
+            return view.model.fetch({ silent: true });
+        }
+        return jQuery.when();
+    },
+
+    /** Override to remove delete button */
+    _renderDeleteButton : function(){
+        return null;
+    },
+
    // ......................................................................... misc
    /** String representation */
    toString : function(){
@@ -195,6 +195,17 @@ var DatasetDCEListItemView = DATASET_LI.DatasetListItemView.extend(
        DATASET_LI.DatasetListItemView.prototype.initialize.call( this, attributes );
    },

+    /** In this override, only get details if in the ready state.
+     *  Note: fetch with no 'change' event triggering to prevent automatic rendering.
+     */
+    _fetchModelDetails : function(){
+        var view = this;
+        if( view.model.inReadyState() && !view.model.hasDetails() ){
+            return view.model.fetch({ silent: true });
+        }
+        return jQuery.when();
+    },
+
    // ......................................................................... misc
    /** String representation */
    toString : function(){
@@ -126,6 +126,18 @@ var DatasetDCE = DATASET_MODEL.DatasetAssociation.extend( BASE_MVC.mixin( Datase
    /** logger used to record this.log messages, commonly set to console */
    //logger              : console,

+    /** url fn */
+    url : function(){
+        var galaxyRoot = (( window.galaxy_config && galaxy_config.root )?( galaxy_config.root ):( '/' ));
+        // won't always be an hda
+        if( !this.has( 'history_id' ) ){
+            console.warn( 'no endpoint for non-hdas within a collection yet' );
+            // (a little silly since this api endpoint *also* points at hdas)
+            return galaxyRoot + 'api/datasets';
+        }
+        return galaxyRoot + 'api/histories/' + this.get( 'history_id' ) + '/contents/' + this.get( 'id' );
+    },
+
    defaults : _.extend( {},
        DATASET_MODEL.DatasetAssociation.prototype.defaults,
        DatasetCollectionElementMixin.defaults
@@ -133,7 +145,7 @@ var DatasetDCE = DATASET_MODEL.DatasetAssociation.extend( BASE_MVC.mixin( Datase

    // because all objects have constructors (as this hashmap would even if this next line wasn't present)
    //  the constructor in hcontentMixin won't be attached by BASE_MVC.mixin to this model
-    //  - re-apply manually it now
+    //  - re-apply manually for now
    /** call the mixin constructor */
    constructor : function( attributes, options ){
        this.debug( '\t DatasetDCE.constructor:', attributes, options );
@@ -148,6 +160,12 @@ var DatasetDCE = DATASET_MODEL.DatasetAssociation.extend( BASE_MVC.mixin( Datase
        DATASET_MODEL.DatasetAssociation.prototype.initialize.call( this, attributes, options );
    },

+    /** Does this model already contain detailed data (as opposed to just summary level data)? */
+    hasDetails : function(){
+        // dataset collection api does return genome_build but doesn't return annotation
+        return _.has( this.attributes, 'annotation' );
+    },
+
    /** String representation. */
    toString : function(){
        var objStr = this.get( 'element_identifier' );
@@ -241,10 +259,12 @@ var DatasetCollection = Backbone.Model
        return json;
    },

-    /** is the collection done with updates and ready to be used? (finished running, etc.) */
+    /** Is this collection in a 'ready' state no processing (for the collection) is left
+     *  to do on the server.
+     */
    inReadyState : function(){
-//TODO: state currenly unimplemented for collections
-        return true;
+        var populated = this.get( 'populated' );
+        return ( this.isDeletedOrPurged() || populated );
    },

    //TODO:?? the following are the same interface as DatasetAssociation - can we combine?
@@ -159,20 +159,22 @@ var DatasetListItemEdit = _super.extend(
    /** Render icon-button to re-run the job that created this dataset. */
    _renderRerunButton : function(){
        var creating_job = this.model.get('creating_job');
-        return faIconButton({
-            title       : _l( 'Run this job again' ),
-            href        : this.model.urls.rerun,
-            classes     : 'rerun-btn',
-            target      : this.linkTarget,
-            faIcon      : 'fa-refresh',
-            onclick     : function(ev) {
-                ev.preventDefault();
-                var form = new ToolsForm.View({'job_id' : creating_job});
-                form.deferred.execute(function(){
-                    Galaxy.app.display(form.$el);
-                });
-            }
-        });
+        if (this.model.get('rerunnable')){
+            return faIconButton({
+                title       : _l( 'Run this job again' ),
+                href        : this.model.urls.rerun,
+                classes     : 'rerun-btn',
+                target      : this.linkTarget,
+                faIcon      : 'fa-refresh',
+                onclick     : function(ev) {
+                    ev.preventDefault();
+                    var form = new ToolsForm.View({'job_id' : creating_job});
+                    form.deferred.execute(function(){
+                        Galaxy.app.display(form.$el);
+                    });
+                }
+            });
+        }
    },

    /** Render an icon-button or popupmenu of links based on the applicable visualizations */
@@ -37,8 +37,6 @@ var DatasetListItemView = _super.extend(

        /** where should pages from links be displayed? (default to new tab/window) */
        this.linkTarget = attributes.linkTarget || '_blank';
-
-        this._setUpListeners();
    },

    /** event listeners */
@@ -60,7 +58,7 @@ var DatasetListItemView = _super.extend(
    },

    // ......................................................................... expandable
-    /** In this override, only get details if in the ready state.
+    /** In this override, only get details if in the ready state, get rerunnable if in other states.
     *  Note: fetch with no 'change' event triggering to prevent automatic rendering.
     */
    _fetchModelDetails : function(){
@@ -100,14 +100,11 @@ define([], function() {
        /** Main Template
        */
        _template: function(options) {
-            // create table element
-            var tmp =   '<div class="ui-table-form-element">' +
+            var tmp =   '<div class="ui-table-form-element input-name-' + options.name + '">' +
                            '<div class="ui-table-form-error ui-error">' +
                                '<span class="fa fa-arrow-down"/><span class="ui-table-form-error-text"/>' +
                            '</div>' +
                            '<div class="ui-table-form-title">';
-
-            // is optional
            if (options.collapsible) {
                tmp +=          '<div class="ui-table-form-optional">' +
                                    '<i class="icon"/>' + options.label +
@@ -115,12 +112,8 @@ define([], function() {
            } else {
                tmp += options.label;
            }
-
-            // finalize title
            tmp +=          '</div>' +
                            '<div class="ui-table-form-field">';
-
-            // add help
            tmp +=              '<div class="ui-table-form-info">';
            if (options.help) {
                tmp +=              options.help;
@@ -129,13 +122,9 @@ define([], function() {
                }
            }
            tmp +=              '</div>';
-
-            // finalize
            tmp +=          '</div>' +
                        '</div>';
-
-            // return input element
            return tmp;
        }
    });
-});
+});
@@ -316,6 +316,7 @@ define(['utils/utils',

            // create input field wrapper
            var input_element = new InputElement(this.app, {
+                name                : input_def.name,
                label               : input_def.label || input_def.name,
                value               : input_def.value,
                default_value       : input_def.default_value,
@@ -348,4 +349,4 @@ define(['utils/utils',
    return {
        View: View
    };
-});
+});
@@ -91,13 +91,6 @@ define(['utils/utils', 'mvc/ui/ui-portlet', 'mvc/ui/ui-misc',
            }
        },

-        /** Shows the final message (usually upon successful job submission)
-        */
-        reciept: function($el) {
-            this.$el.empty();
-            this.$el.append($el);
-        },
-
        /** Highlight and scroll to input element (currently only used for error notifications)
        */
        highlight: function (input_id, message, silent) {
@@ -206,12 +199,8 @@ define(['utils/utils', 'mvc/ui/ui-portlet', 'mvc/ui/ui-misc',
                inputs : this.options.inputs
            });

-            // switch to classic tool form mako if the form definition is incompatible
-            if (this.incompatible) {
-                this.$el.hide();
-                $('#tool-form-classic').show();
-                return;
-            }
+            // remove tooltips
+            $( '.tooltip' ).remove();

            // create portlet
            this.portlet = new Portlet.View({
@@ -245,4 +234,4 @@ define(['utils/utils', 'mvc/ui/ui-portlet', 'mvc/ui/ui-misc',
            console.debug('tools-form-base::initialize() - Completed.');
        }
    });
-});
+});
@@ -16,13 +16,30 @@ var HDAListItemEdit = _super.extend(

    className   : _super.prototype.className + " history-content",

-    // ......................................................................... events
+    /** In this override, only get details if in the ready state, get rerunnable if in other states.
+     *  Note: fetch with no 'change' event triggering to prevent automatic rendering.
+     */
+    _fetchModelDetails : function(){
+        var view = this;
+        if( view.model.inReadyState() && !view.model.hasDetails() ){
+            return view.model.fetch({ silent: true });
+
+        // special case the need for the rerunnable and creating_job attributes
+        // needed for rendering re-run button on queued, running datasets
+        } else if( !view.model.has( 'rerunnable' ) ){
+            return view.model.fetch({ silent: true, data: {
+                // only fetch rerunnable and creating_job to keep overhead down
+                keys: [ 'rerunnable', 'creating_job' ].join(',')
+            }});
+        }
+        return jQuery.when();
+    },
+
    /** event map */
    events : _.extend( _.clone( _super.prototype.events ), {
        'click .unhide-link' : function( ev ){ this.model.unhide(); return false; }
    }),

-    // ......................................................................... misc
    /** string rep */
    toString : function(){
        var modelString = ( this.model )?( this.model + '' ):( '(no model)' );
@@ -21,11 +21,9 @@ var HDCAListItemView = _super.extend(
    /** event listeners */
    _setUpListeners : function(){
        _super.prototype._setUpListeners.call( this );
-        // re-rendering on deletion
-        this.model.on( 'change', function( model, options ){
-            if( _.isEqual( _.keys( model.changed ), [ 'visible' ] ) ){
-                this.render();
-            }
+
+        this.model.on({
+            'change:populated change:visible' : function( model, options ){ this.render(); },
        }, this );
    },

@@ -46,7 +44,7 @@ var HDCAListItemView = _super.extend(
    _swapNewRender : function( $newRender ){
        _super.prototype._swapNewRender.call( this, $newRender );
 //TODO: model currently has no state
-        var state = this.model.get( 'state' ) || STATES.OK;
+        var state = !this.model.get( 'populated' ) ? STATES.RUNNING : STATES.OK;
        //if( this.model.has( 'state' ) ){
        this.$el.addClass( 'state-' + state );
        //}
@@ -87,14 +87,6 @@ var CurrentHistoryPanel = _super.extend(
        _super.prototype._setUpListeners.call( this );

        var panel = this;
-        // cache the scroll position (only if visible)
-        this.$scrollContainer().on( 'scroll', function( ev ){
-            if( panel.$el.is( ':visible' ) ){
-                panel.preferences.set( 'scrollPosition', $( this ).scrollTop() );
-            }
-        });
-
-        //TODO: doesn't work
        // reset scroll position when there's a new history
        this.on( 'new-model', function(){
            panel.preferences.set( 'scrollPosition', 0 );
@@ -191,6 +183,30 @@ var CurrentHistoryPanel = _super.extend(
    },

    // ------------------------------------------------------------------------ panel rendering
+    /** override to add a handler to capture the scroll position when the parent scrolls */
+    _setUpBehaviors : function( $where ){
+        $where = $where || this.$el;
+        // we need to call this in _setUpBehaviors which is called after render since the $el
+        // may not be attached to $el.parent and $scrollContainer() may not work
+        var panel = this;
+        _super.prototype._setUpBehaviors.call( panel, $where );
+
+        // cache the handler to remove and re-add so we don't pile up the handlers
+        if( !this._debouncedScrollCaptureHandler ){
+            this._debouncedScrollCaptureHandler = _.debounce( function scrollCapture(){
+                // cache the scroll position (only if visible)
+                if( panel.$el.is( ':visible' ) ){
+                    panel.preferences.set( 'scrollPosition', $( this ).scrollTop() );
+                }
+            }, 40 );
+        }
+
+        panel.$scrollContainer()
+            .off( 'scroll', this._debouncedScrollCaptureHandler )
+            .on( 'scroll', this._debouncedScrollCaptureHandler );
+        return panel;
+    },
+
    /** In this override, handle null models and move the search input to the top */
    _buildNewRender : function(){
        if( !this.model ){ return $(); }
@@ -281,7 +297,7 @@ var CurrentHistoryPanel = _super.extend(
        _.delay( function(){
            var pos = panel.preferences.get( 'scrollPosition' );
            if( pos ){
-                panel.scrollTo( pos );
+                panel.scrollTo( pos, 0 );
            }
        }, 10 );
        //TODO: is this enough of a delay on larger histories?
@@ -263,6 +263,11 @@ var FolderListView = Backbone.View.extend({
     *  Currently supports only sorting by name. 
     */
    sortFolder: function(sort_by, order){
+        console.log('sorting');
+        // default to asc sort by name
+        if (sort_by === 'undefined' && order === 'undefined'){
+            return this.collection.sortByNameAsc();
+        }
        if (sort_by === 'name'){
            if (order === 'asc'){
                return this.collection.sortByNameAsc();
@@ -15,7 +15,8 @@ var FolderRowView = Backbone.View.extend({
  lastSelectedHistory: '',

  events: {
-    'click .undelete_dataset_btn'    : 'undelete_dataset'
+    'click .undelete_dataset_btn'    : 'undeleteDataset',
+    'click .undelete_folder_btn'     : 'undeleteFolder'
  },

  options: {
@@ -29,16 +30,23 @@ var FolderRowView = Backbone.View.extend({

  render: function(folder_item){
    var template = null;
-    if (folder_item.get('type') === 'folder'){
+    if (folder_item.get('type') === 'folder' || folder_item.get('model_class') === 'LibraryFolder'){
      this.options.type = 'folder';
-      template = this.templateRowFolder();
-    } else {
+      if (folder_item.get('deleted')){
+        template = this.templateRowDeletedFolder();
+      } else{
+        template = this.templateRowFolder();
+      }
+    } else if (folder_item.get('type') === 'file' || folder_item.get('model_class') === 'LibraryDataset'){
      this.options.type = 'file';
      if (folder_item.get('deleted')){
        template = this.templateRowDeletedFile();
      } else {
        template = this.templateRowFile();
      }
+    } else {
+      console.error('Unknown library item type found.');
+      console.error(folder_item.get('type') || folder_item.get('model_class'));
    }
    this.setElement(template({content_item:folder_item}));
    this.$el.show();
@@ -52,7 +60,7 @@ var FolderRowView = Backbone.View.extend({
  /**
   * Undeletes the dataset on server and renders the row again.
   */
-  undelete_dataset : function(event){
+  undeleteDataset : function(event){
    $(".tooltip").hide();
    var that = this;
    var dataset_id = $(event.target).closest('tr')[0].id;
@@ -63,6 +71,7 @@ var FolderRowView = Backbone.View.extend({
          Galaxy.libraries.folderListView.collection.remove(dataset_id);
          var updated_dataset = new mod_library_model.Item(response);
          Galaxy.libraries.folderListView.collection.add(updated_dataset);
+          Galaxy.libraries.folderListView.collection.sortByNameAsc();
          mod_toastr.success('Dataset undeleted. Click this to see it.', '', {onclick: function() {
            var folder_id = that.model.get('folder_id');
            window.location='#folders/' + folder_id + '/datasets/' + that.id;
@@ -78,6 +87,33 @@ var FolderRowView = Backbone.View.extend({
  });
  },

+  /**
+   * Undeletes the folder on server and renders the row again.
+   */
+  undeleteFolder : function(event){
+    $(".tooltip").hide();
+    var that = this;
+    var folder_id = $(event.target).closest('tr')[0].id;
+    var folder = Galaxy.libraries.folderListView.collection.get(folder_id);
+    folder.url = folder.urlRoot + folder.id + '?undelete=true';
+    folder.destroy({
+        success : function(model, response){
+          Galaxy.libraries.folderListView.collection.remove(folder_id);
+          var updated_folder = new mod_library_model.FolderAsModel(response);
+          Galaxy.libraries.folderListView.collection.add(updated_folder);
+          Galaxy.libraries.folderListView.collection.sortByNameAsc();
+          mod_toastr.success('Folder undeleted.');
+        }, 
+        error : function(model, response){
+          if (typeof response.responseJSON !== "undefined"){
+            mod_toastr.error('Folder was not undeleted. ' + response.responseJSON.err_msg);
+          } else {
+            mod_toastr.error('An error occured! Folder was not undeleted. Please try again.');
+          }
+        }
+  });
+  },
+
  templateRowFolder: function() {
    tmpl_array = [];

@@ -138,6 +174,26 @@ var FolderRowView = Backbone.View.extend({
    tmpl_array.push('  <td><span data-toggle="tooltip" data-placement="top" title="Marked deleted" style="color:grey;" class="fa fa-ban fa-lg"> </span><button data-toggle="tooltip" data-placement="top" title="Undelete <%- content_item.get("name") %>" class="primary-button btn-xs undelete_dataset_btn show_on_hover" type="button" style="display:none; margin-left:1em;"><span class="fa fa-unlock"> Undelete</span></button></td>');
    tmpl_array.push('</tr>');

+    return _.template(tmpl_array.join(''));
+  },
+
+  templateRowDeletedFolder: function(){
+    tmpl_array = [];
+
+    tmpl_array.push('<tr class="active folder_row light library-row" id="<%- content_item.id %>">');
+    tmpl_array.push('  <td>');
+    tmpl_array.push('    <span title="Folder" class="fa fa-folder-o"></span>');
+    tmpl_array.push('  </td>');
+    tmpl_array.push('  <td></td>');
+    tmpl_array.push('  <td style="color:grey;">');
+    tmpl_array.push('    <%- content_item.get("name") %>');
+    tmpl_array.push('  </td>');
+    tmpl_array.push('  <td>folder</td>');
+    tmpl_array.push('  <td></td>');
+    tmpl_array.push('  <td><%= _.escape(content_item.get("update_time")) %></td>'); // time updated
+    tmpl_array.push('  <td><span data-toggle="tooltip" data-placement="top" title="Marked deleted" style="color:grey;" class="fa fa-ban fa-lg"> </span><button data-toggle="tooltip" data-placement="top" title="Undelete <%- content_item.get("name") %>" class="primary-button btn-xs undelete_folder_btn show_on_hover" type="button" style="display:none; margin-left:1em;"><span class="fa fa-unlock"> Undelete</span></button></td>');
+    tmpl_array.push('</tr>');
+
    return _.template(tmpl_array.join(''));
  }
   
@@ -863,37 +863,45 @@ var FolderToolbarView = Backbone.View.extend({
   * call them in chain. Update progress bar in between each.
   * @param  {array} lddas_set array of lddas to delete
   */
-  chainCallDeletingHdas: function( lddas_set ){
+  chainCallDeletingItems: function( items_to_delete ){
  var self = this;
-  this.deleted_lddas = new mod_library_model.Folder();
-  var popped_item = lddas_set.pop();
+  this.deleted_items = new mod_library_model.Folder();
+  var popped_item = items_to_delete.pop();
  if ( typeof popped_item === "undefined" ) {
    if ( this.options.chain_call_control.failed_number === 0 ){
-      mod_toastr.success( 'Selected datasets were deleted.' );
+      mod_toastr.success( 'Selected items were deleted.' );
    } else if ( this.options.chain_call_control.failed_number === this.options.chain_call_control.total_number ){
-      mod_toastr.error( 'There was an error and no datasets were deleted. Please make sure you have sufficient permissions.' );
+      mod_toastr.error( 'There was an error and no items were deleted. Please make sure you have sufficient permissions.' );
    } else if ( this.options.chain_call_control.failed_number < this.options.chain_call_control.total_number ){
-      mod_toastr.warning( 'Some of the datasets could not be deleted. Please make sure you have sufficient permissions.' );
+      mod_toastr.warning( 'Some of the items could not be deleted. Please make sure you have sufficient permissions.' );
    }
    Galaxy.modal.hide();
-    return this.deleted_lddas;
+    return this.deleted_items;
  }
  var promise = $.when( popped_item.destroy() );

-  promise.done( function( dataset ){
+  promise.done( function( item ){
            Galaxy.libraries.folderListView.collection.remove( popped_item.id );
            self.updateProgress();
-            // add the deleted dataset to collection, triggers rendering
+            // add the deleted item to collection, triggers rendering
            if ( Galaxy.libraries.folderListView.options.include_deleted ){
-              var updated_dataset = new mod_library_model.Item( dataset );
-              Galaxy.libraries.folderListView.collection.add( updated_dataset );
+              var updated_item = null;
+              if (item.type === 'folder' || item.model_class === 'LibraryFolder'){
+                updated_item = new mod_library_model.FolderAsModel( item );
+              } else if (item.type === 'file' || item.model_class === 'LibraryDataset'){
+                updated_item = new mod_library_model.Item( item );
+              } else {
+                console.error('Unknown library item type found.');
+                console.error(item.type || item.model_class);
+              }
+              Galaxy.libraries.folderListView.collection.add( updated_item );
            }
-            self.chainCallDeletingHdas( lddas_set );
+            self.chainCallDeletingItems( items_to_delete );
          })
          .fail( function(){
            self.options.chain_call_control.failed_number += 1;
            self.updateProgress();
-            self.chainCallDeletingHdas( lddas_set );
+            self.chainCallDeletingItems( items_to_delete );
          });
  },

@@ -931,25 +939,35 @@ var FolderToolbarView = Backbone.View.extend({
      this.options.chain_call_control.failed_number = 0;

      var dataset_ids = [];
+      var folder_ids = [];
      checkedValues.each(function(){
          if (this.parentElement.parentElement.id !== '') {
-              dataset_ids.push(this.parentElement.parentElement.id);
+              if (this.parentElement.parentElement.id.substring(0,1) == 'F'){
+                folder_ids.push(this.parentElement.parentElement.id);
+              } else {
+                dataset_ids.push(this.parentElement.parentElement.id);
+              }
          }
      });
      // init the progress bar
-      this.progressStep = 100 / dataset_ids.length;
+      var items_total = dataset_ids.length + folder_ids.length
+      this.progressStep = 100 / items_total;
      this.progress = 0;
      
      // prepare the dataset items to be added
-      var lddas_to_delete = [];
+      var items_to_delete = [];
      for (var i = dataset_ids.length - 1; i >= 0; i--) {
          var dataset = new mod_library_model.Item({id:dataset_ids[i]});
-          lddas_to_delete.push(dataset);
+          items_to_delete.push(dataset);
+      }
+      for (var i = folder_ids.length - 1; i >= 0; i--) {
+          var folder = new mod_library_model.FolderAsModel({id:folder_ids[i]});
+          items_to_delete.push(folder);
      }

-      this.options.chain_call_control.total_number = dataset_ids.length;
+      this.options.chain_call_control.total_number = items_total.length;
      // call the recursive function to call ajax one after each other (request FIFO queue)
-      this.chainCallDeletingHdas(lddas_to_delete);
+      this.chainCallDeletingItems(items_to_delete);
    }
  },

@@ -1099,6 +1117,7 @@ var FolderToolbarView = Backbone.View.extend({
    tmpl_array.push('<p><%- library.get("description") %></p>');
    tmpl_array.push('<h3>Library synopsis:</h3>');
    tmpl_array.push('<p><%- library.get("synopsis") %></p>');
+    tmpl_array.push('<p data-toggle="tooltip" data-placement="top" title="<%- library.get("create_time") %>">created <%- library.get("create_time_pretty") %></p>');

    tmpl_array.push('</div>');

@@ -174,6 +174,11 @@ var LibraryListView = Backbone.View.extend({
        window.location = '/user/login';
    },

+    searchLibraries: function(search_term){
+        var results = this.collection.search(search_term);
+        this.render({'models': results});
+    },
+
 // MMMMMMMMMMMMMMMMMM
 // === TEMPLATES ====
 // MMMMMMMMMMMMMMMMMM
@@ -5,7 +5,7 @@ function(mod_toastr,
         mod_library_model) {
 /**
 * This view represents the top part of the library page.
- * It contains the tool bar with buttons.
+ * It contains the tool bar with controls.
 */
 var LibraryToolbarView = Backbone.View.extend({
  el: '#center',
@@ -13,7 +13,8 @@ var LibraryToolbarView = Backbone.View.extend({
  events: {
    'click #create_new_library_btn' : 'showLibraryModal',
    'click #include_deleted_chk'    : 'includeDeletedChecked',
-    'click #page_size_prompt'       : 'showPageSizePrompt'
+    'click #page_size_prompt'       : 'showPageSizePrompt',
+    'keyup .library-search-input'   : 'searchLibraries'
  },

  initialize: function( options ){
@@ -36,93 +37,109 @@ var LibraryToolbarView = Backbone.View.extend({
  },

  /**
-   * Called from LibraryListView when needed.
-   * @param  {object} options common options
+   * Renders the element that shows pages into its div within the toolbar.
   */
  renderPaginator: function( options ){
    this.options = _.extend( this.options, options );
    var paginator_template = this.templatePaginator();
    this.$el.find( '#library_paginator' ).html( paginator_template({ 
-        show_page: parseInt( this.options.show_page ),
-        page_count: parseInt( this.options.page_count ),
-        total_libraries_count: this.options.total_libraries_count,
-        libraries_shown: this.options.libraries_shown
+      show_page: parseInt( this.options.show_page ),
+      page_count: parseInt( this.options.page_count ),
+      total_libraries_count: this.options.total_libraries_count,
+      libraries_shown: this.options.libraries_shown
    }));
  },

+  /**
+   * User clicked on 'New library' button. Show modal to
+   * satisfy the wish.
+   */
  showLibraryModal : function (event){
-      event.preventDefault();
-      event.stopPropagation();
-      var self = this;
-      this.modal = Galaxy.modal;
-      this.modal.show({
-          closing_events  : true,
-          title           : 'Create New Library',
-          body            : this.templateNewLibraryInModal(),
-          buttons         : {
-              'Create'    : function() { self.createNewLibrary(); },
-              'Close'     : function() { self.modal.hide(); }
-          }
-      });
-  },
-
-  createNewLibrary: function(){
-      var libraryDetails = this.serializeNewLibrary();
-      if (this.valdiateNewLibrary(libraryDetails)){
-          var library = new mod_library_model.Library();
-          var self = this;
-          library.save(libraryDetails, {
-            success: function (library) {
-              Galaxy.libraries.libraryListView.collection.add(library);
-              self.modal.hide();
-              self.clearLibraryModal();
-              Galaxy.libraries.libraryListView.render();
-              mod_toastr.success('Library created.');
-            },
-            error: function(model, response){
-              if (typeof response.responseJSON !== "undefined"){
-                mod_toastr.error(response.responseJSON.err_msg);
-              } else {
-                mod_toastr.error('An error occured.');
-              }
-            }
-          });
-      } else {
-          mod_toastr.error('Library\'s name is missing.');
+    event.preventDefault();
+    event.stopPropagation();
+    var self = this;
+    this.modal = Galaxy.modal;
+    this.modal.show({
+      closing_events  : true,
+      title           : 'Create New Library',
+      body            : this.templateNewLibraryInModal(),
+      buttons         : {
+        'Create'      : function() { self.createNewLibrary(); },
+        'Close'       : function() { self.modal.hide(); }
      }
-      return false;
+    });
  },

+  /**
+   * Create the new library using the API asynchronously.
+   */
+  createNewLibrary: function(){
+    var libraryDetails = this.serializeNewLibrary();
+    if (this.validateNewLibrary(libraryDetails)){
+      var library = new mod_library_model.Library();
+      var self = this;
+      library.save(libraryDetails, {
+        success: function (library) {
+          Galaxy.libraries.libraryListView.collection.add(library);
+          self.modal.hide();
+          self.clearLibraryModal();
+          Galaxy.libraries.libraryListView.render();
+          mod_toastr.success('Library created.');
+        },
+        error: function(model, response){
+          if (typeof response.responseJSON !== "undefined"){
+            mod_toastr.error(response.responseJSON.err_msg);
+          } else {
+            mod_toastr.error('An error occured.');
+          }
+        }
+      });
+    } else {
+      mod_toastr.error('Library\'s name is missing.');
+    }
+    return false;
+  },
+
+  /**
+   * Show user the propmpt to change the number of libs shown on page.
+   */
  showPageSizePrompt: function(){
    var library_page_size = prompt( 'How many libraries per page do you want to see?', Galaxy.libraries.preferences.get( 'library_page_size' ) );
    if ( ( library_page_size != null ) && ( library_page_size == parseInt( library_page_size ) ) ) {
-        Galaxy.libraries.preferences.set( { 'library_page_size': parseInt( library_page_size ) } );
-        Galaxy.libraries.libraryListView.render( { show_page: 1 } );
+      Galaxy.libraries.preferences.set( { 'library_page_size': parseInt( library_page_size ) } );
+      Galaxy.libraries.libraryListView.render( { show_page: 1 } );
    }
  },

-  // clear the library modal once saved
+  /**
+   * Clear the library modal once it is saved.
+   */
  clearLibraryModal : function(){
-      $("input[name='Name']").val('');
-      $("input[name='Description']").val('');
-      $("input[name='Synopsis']").val('');
+    $("input[name='Name']").val('');
+    $("input[name='Description']").val('');
+    $("input[name='Synopsis']").val('');
  },

+  /**
+   * Prepare new library variables to be submitted to API.
+   */
  serializeNewLibrary : function(){
-      return {
-          name: $("input[name='Name']").val(),
-          description: $("input[name='Description']").val(),
-          synopsis: $("input[name='Synopsis']").val()
-      };
+    return {
+      name: $("input[name='Name']").val(),
+      description: $("input[name='Description']").val(),
+      synopsis: $("input[name='Synopsis']").val()
+    };
  },

-  valdiateNewLibrary: function( libraryDetails ){
+  /**
+   * Check whether entered values are valid.
+   */
+  validateNewLibrary: function( libraryDetails ){
      return libraryDetails.name !== '';
  },

  /**
   * Include or exclude deleted libraries in the view.
-   * @param  {object} event common event
   */
  includeDeletedChecked: function( event ){
    if (event.target.checked){
@@ -134,89 +151,101 @@ var LibraryToolbarView = Backbone.View.extend({
    }
  },

+  /**
+   * Take the contents of the search field and send it to the list view
+   * to query the collection of libraries.
+   */
+  searchLibraries: function(event){
+    var search_term = $(".library-search-input").val();
+    Galaxy.libraries.libraryListView.searchLibraries(search_term);
+  },
+
  templateToolBar: function(){
-    tmpl_array = [];
-
-
-    tmpl_array.push('<div class="library_style_container">');
-    tmpl_array.push('  <div id="toolbar_form">');
-    tmpl_array.push('      <div id="library_toolbar">');
-    tmpl_array.push('      <form class="form-inline" role="form">');
-    tmpl_array.push('          <span><strong><a href="#" title="Go to first page">DATA LIBRARIES</a></strong></span>');
-    tmpl_array.push('          <% if(admin_user === true) { %>');
-    tmpl_array.push('              <div class="checkbox" style="height: 20px;">');
-    tmpl_array.push('                  <label>');
-    tmpl_array.push('                      <input id="include_deleted_chk" type="checkbox"> include deleted </input>');
-    tmpl_array.push('                   </label>');
-    tmpl_array.push('              </div>');
-    tmpl_array.push('              <span data-toggle="tooltip" data-placement="top" title="Create New Library"><button id="create_new_library_btn" class="primary-button btn-xs" type="button"><span class="fa fa-plus"></span> New Library</button></span>');
-    tmpl_array.push('          <% } %>');
-    tmpl_array.push('          <span class="help-button" data-toggle="tooltip" data-placement="top" title="Visit Libraries Wiki"><a href="https://wiki.galaxyproject.org/DataLibraries/screen/ListOfLibraries" target="_blank"><button class="primary-button" type="button"><span class="fa fa-question-circle"></span> Help</button></a></span>');
-    tmpl_array.push('          <span id="library_paginator" class="library-paginator">');
-    // paginator will append here
-    tmpl_array.push('          </span>');
-    tmpl_array.push('      </form>');
-    tmpl_array.push('      </div>');
-    tmpl_array.push('  </div>');
-    tmpl_array.push('  <div id="libraries_element">');
-    // table with libraries will append here
-    tmpl_array.push('  </div>');
-    tmpl_array.push('</div>');
-
-    return _.template(tmpl_array.join(''));
+    return _.template([
+      '<div class="library_style_container">',
+        '<div id="toolbar_form">',
+          '<div id="library_toolbar">',
+            '<form class="form-inline" role="form">',
+              '<span><strong><a href="#" title="Go to first page">DATA LIBRARIES</a></strong></span>',
+              '<span class="help-button" data-toggle="tooltip" data-placement="top" title="Visit Libraries Wiki">',
+                '<a href="https://wiki.galaxyproject.org/DataLibraries/screen/ListOfLibraries" target="_blank">',
+                  '<button class="primary-button" type="button"><span class="fa fa-question-circle"></span> Help</button>',
+                '</a>',
+                '</span>',
+              '<span id="library_paginator" class="library-paginator">',
+              // paginator will append here
+              '</span>',
+              '<div class="form-group">',
+                '<input type="text" class="form-control library-search-input" placeholder="Search">',
+              '</div>',
+              // only admins see the following
+              '<% if(admin_user === true) { %>',
+                '<div class="checkbox" style="height: 20px;">',
+                  '<label>',
+                    '<input id="include_deleted_chk" type="checkbox"> include deleted </input>',
+                  '</label>',
+                '</div>',
+                '<span data-toggle="tooltip" data-placement="top" title="Create New Library">',
+                  '<button id="create_new_library_btn" class="primary-button btn-xs" type="button"><span class="fa fa-plus"></span> New Library</button>',
+                '</span>',
+              '<% } %>',
+            '</form>',
+          '</div>',
+        '</div>',
+        '<div id="libraries_element">',
+        // table with libraries will append here
+        '</div>',
+      '</div>'
+    ].join(''));
  },

  templatePaginator: function(){
-    tmpl_array = [];
-
-    tmpl_array.push('   <ul class="pagination pagination-sm">');
-    tmpl_array.push('       <% if ( ( show_page - 1 ) > 0 ) { %>');
-    tmpl_array.push('           <% if ( ( show_page - 1 ) > page_count ) { %>'); // we are on higher page than total page count
-    tmpl_array.push('               <li><a href="#page/1"><span class="fa fa-angle-double-left"></span></a></li>');
-    tmpl_array.push('               <li class="disabled"><a href="#page/<% print( show_page ) %>"><% print( show_page - 1 ) %></a></li>');
-    tmpl_array.push('           <% } else { %>');
-    tmpl_array.push('               <li><a href="#page/1"><span class="fa fa-angle-double-left"></span></a></li>');
-    tmpl_array.push('               <li><a href="#page/<% print( show_page - 1 ) %>"><% print( show_page - 1 ) %></a></li>');
-    tmpl_array.push('           <% } %>');
-    tmpl_array.push('       <% } else { %>'); // we are on the first page
-    tmpl_array.push('           <li class="disabled"><a href="#page/1"><span class="fa fa-angle-double-left"></span></a></li>');
-    tmpl_array.push('           <li class="disabled"><a href="#page/<% print( show_page ) %>"><% print( show_page - 1 ) %></a></li>');
-    tmpl_array.push('       <% } %>');
-    tmpl_array.push('       <li class="active">');
-    tmpl_array.push('       <a href="#page/<% print( show_page ) %>"><% print( show_page ) %></a>');
-    tmpl_array.push('       </li>');
-    tmpl_array.push('       <% if ( ( show_page ) < page_count ) { %>');
-    tmpl_array.push('           <li><a href="#page/<% print( show_page + 1 ) %>"><% print( show_page + 1 ) %></a></li>');
-    tmpl_array.push('           <li><a href="#page/<% print( page_count ) %>"><span class="fa fa-angle-double-right"></span></a></li>');
-    tmpl_array.push('       <% } else { %>');
-    tmpl_array.push('           <li class="disabled"><a href="#page/<% print( show_page  ) %>"><% print( show_page + 1 ) %></a></li>');
-    tmpl_array.push('           <li class="disabled"><a href="#page/<% print( page_count ) %>"><span class="fa fa-angle-double-right"></span></a></li>');
-    tmpl_array.push('       <% } %>');
-    tmpl_array.push('   </ul>');
-    tmpl_array.push('   <span>');
-    tmpl_array.push('       showing <a data-toggle="tooltip" data-placement="top" title="Click to change the number of libraries on page" id="page_size_prompt"><%- libraries_shown %></a> of <%- total_libraries_count %> libraries');
-    tmpl_array.push('   </span>');
-
-    return _.template(tmpl_array.join(''));
+    return _.template([
+    '<ul class="pagination pagination-sm">',
+      '<% if ( ( show_page - 1 ) > 0 ) { %>',
+      '<% if ( ( show_page - 1 ) > page_count ) { %>', // we are on higher page than total page count
+        '<li><a href="#page/1"><span class="fa fa-angle-double-left"></span></a></li>',
+        '<li class="disabled"><a href="#page/<% print( show_page ) %>"><% print( show_page - 1 ) %></a></li>',
+      '<% } else { %>',
+        '<li><a href="#page/1"><span class="fa fa-angle-double-left"></span></a></li>',
+        '<li><a href="#page/<% print( show_page - 1 ) %>"><% print( show_page - 1 ) %></a></li>',
+      '<% } %>',
+      '<% } else { %>', // we are on the first page
+        '<li class="disabled"><a href="#page/1"><span class="fa fa-angle-double-left"></span></a></li>',
+        '<li class="disabled"><a href="#page/<% print( show_page ) %>"><% print( show_page - 1 ) %></a></li>',
+      '<% } %>',
+        '<li class="active">',
+          '<a href="#page/<% print( show_page ) %>"><% print( show_page ) %></a>',
+        '</li>',
+      '<% if ( ( show_page ) < page_count ) { %>',
+        '<li><a href="#page/<% print( show_page + 1 ) %>"><% print( show_page + 1 ) %></a></li>',
+        '<li><a href="#page/<% print( page_count ) %>"><span class="fa fa-angle-double-right"></span></a></li>',
+      '<% } else { %>',
+        '<li class="disabled"><a href="#page/<% print( show_page  ) %>"><% print( show_page + 1 ) %></a></li>',
+        '<li class="disabled"><a href="#page/<% print( page_count ) %>"><span class="fa fa-angle-double-right"></span></a></li>',
+      '<% } %>',
+    '</ul>',
+    '<span>',
+      'showing <a data-toggle="tooltip" data-placement="top" title="Click to change the number of libraries on page" id="page_size_prompt"><%- libraries_shown %></a> of <%- total_libraries_count %> libraries',
+    '</span>'
+    ].join(''));
  },

  templateNewLibraryInModal: function(){
-      tmpl_array = [];
-
-      tmpl_array.push('<div id="new_library_modal">');
-      tmpl_array.push('   <form>');
-      tmpl_array.push('       <input type="text" name="Name" value="" placeholder="Name">');
-      tmpl_array.push('       <input type="text" name="Description" value="" placeholder="Description">');
-      tmpl_array.push('       <input type="text" name="Synopsis" value="" placeholder="Synopsis">');
-      tmpl_array.push('   </form>');
-      tmpl_array.push('</div>');
-
-      return tmpl_array.join('');
+    return _.template([
+      '<div id="new_library_modal">',
+        '<form>',
+          '<input type="text" name="Name" value="" placeholder="Name">',
+          '<input type="text" name="Description" value="" placeholder="Description">',
+          '<input type="text" name="Synopsis" value="" placeholder="Synopsis">',
+        '</form>',
+      '</div>'
+    ].join(''));
  }
 });

 return {
-    LibraryToolbarView: LibraryToolbarView
+  LibraryToolbarView: LibraryToolbarView
 };

 });
@@ -31,6 +31,19 @@ define([], function() {
          options = options || {};
      },

+      search : function(search_term){
+        /**
+         * Search the collection and return only the models that have
+         * the search term in their names.
+         * [the term to search]
+         * @type {string}
+         */
+        if (search_term == "") return this;
+        return this.filter(function(data) {
+          return data.get("name").indexOf(search_term) !== -1;
+        });
+      },
+
      /** Get every 'shown' library in this collection based on deleted filter
       *  @param {Boolean} show_deleted are we including deleted libraries?
       *  @returns array of library models
@@ -83,20 +96,19 @@ define([], function() {
 // ============================================================================
 // FOLDER RELATED MODELS

-    var Item = Backbone.Model.extend({
+    var LibraryItem = Backbone.Model.extend({
+    });
+
+    var Ldda = LibraryItem.extend({
      urlRoot : ( window.galaxy_config ? galaxy_config.root : '/' ) + 'api/libraries/datasets/'
    });

-    var Ldda = Backbone.Model.extend({
-      urlRoot : ( window.galaxy_config ? galaxy_config.root : '/' ) + 'api/libraries/datasets/'
-    });
-
-    var FolderAsModel = Backbone.Model.extend({
-      urlRoot: ( window.galaxy_config ? galaxy_config.root : '/' ) + 'api/folders'
+    var FolderAsModel = LibraryItem.extend({
+      urlRoot: ( window.galaxy_config ? galaxy_config.root : '/' ) + 'api/folders/'
    });

    var Folder = Backbone.Collection.extend({
-      model: Item,
+      model: LibraryItem,

      /** Sort collection by item name (ascending) and return the sorted
       *  collection. Folders go before datasets. 
@@ -150,16 +162,28 @@ define([], function() {
    });

    var FolderContainer = Backbone.Model.extend({
-        defaults : {
-            folder : new Folder(),
-            urlRoot : ( window.galaxy_config ? galaxy_config.root : '/' ) + 'api/folders/',
-            id : "unknown"
-        },
-    parse : function(obj) {
-      // response is not a simple array, it contains metadata
-      // this will update the inner collection
-      this.get("folder").reset(obj.folder_contents);
-      return obj;
+      defaults : {
+          folder : new Folder(),
+          urlRoot : ( window.galaxy_config ? galaxy_config.root : '/' ) + 'api/folders/',
+          id : "unknown"
+      },
+      parse : function(obj) {
+        // empty the collection
+        this.get("folder").reset();
+        // response is not a simple array, it contains metadata
+        // this will update the inner collection
+          for (var i = 0; i < obj.folder_contents.length; i++) {
+            if (obj.folder_contents[i].type === 'folder'){
+              var folder_item = new FolderAsModel(obj.folder_contents[i])
+              this.get("folder").add(folder_item);
+            } else if(obj.folder_contents[i].type === 'file'){
+              var file_item = new Ldda(obj.folder_contents[i])
+              this.get("folder").add(file_item);
+            } else{
+              console.error('Unknown folder item type encountered while parsing response.');
+            }
+          };
+        return obj;
      }
    });

@@ -204,10 +228,10 @@ define([], function() {

 return {
    Library: Library,
-    FolderAsModel : FolderAsModel,
    Libraries : Libraries,
-    Item : Item,
+    Item : Ldda,
    Ldda : Ldda,
+    FolderAsModel : FolderAsModel,
    Folder : Folder,
    FolderContainer : FolderContainer,
    HistoryItem : HistoryItem,
@@ -525,7 +525,7 @@ var ToolLinkView = BaseView.extend({
        }

        // regular tools
-        if (this.model.get('model_class') === 'Tool') {
+        if ( this.model.id !== 'upload1' && this.model.get( 'model_class' ) === 'Tool' ) {
            var self = this;
            $link.find('a').on('click', function(e) {
                e.preventDefault();
@@ -44,6 +44,9 @@ define(['utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view',
                }
            }, this.options);

+            // allow option customization
+            this.options.customize && this.options.customize( this.options );
+
            // create form
            this.form = new Form(this.options);

@@ -65,20 +68,19 @@ define(['utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view',
            this.options.id = options.id;
            this.options.version = options.version;

-            if (options.job_id) {
+            if ( options.job_id ) {
                build_url = Galaxy.root + 'api/jobs/' + options.job_id + '/build_for_rerun';
            } else {
-                // construct url
                var build_url = Galaxy.root + 'api/tools/' + options.id + '/build?';
-                if (options.dataset_id) {
-                    build_url += 'dataset_id=' + options.dataset_id;
-                } else {
+                if ( options.version ) {
                    build_url += 'tool_version=' + options.version + '&';
-                    var loc = top.location.href;
-                    var pos = loc.indexOf('?');
-                    if (loc.indexOf('tool_id=') != -1 && pos !== -1) {
-                        build_url += loc.slice(pos + 1);
-                    }
+                }
+                if ( Galaxy.params && Galaxy.params.tool_id == options.id ) {
+                    _.each( Galaxy.params, function ( item, key ) {
+                        if ( [ 'tool_version', 'tool_id' ].indexOf( key ) == -1 ) {
+                            build_url += key + '=' + item + '&';
+                        }
+                    } );
                }
            }

@@ -116,16 +118,25 @@ define(['utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view',
                    console.debug(response);

                    // show error
-                    var error_message = response.error || 'Uncaught error.';
-                    self.form.modal.show({
-                        title   : 'Tool cannot be executed',
-                        body    : error_message,
-                        buttons : {
-                            'Close' : function() {
-                                self.form.modal.hide();
+                    var error_message = ( response && response.err_msg ) || 'Uncaught error.';
+                    if ( self.$el.is(':empty') ) {
+                        self.$el.prepend((new Ui.Message({
+                            message     : error_message,
+                            status      : 'danger',
+                            persistent  : true,
+                            large       : true
+                        })).$el);
+                    } else {
+                        Galaxy.modal.show({
+                            title   : 'Tool request failed',
+                            body    : error_message,
+                            buttons : {
+                                'Close' : function() {
+                                    Galaxy.modal.hide();
+                                }
                            }
-                        }
-                    });
+                        });
+                    }
                }
            });
        },
@@ -347,4 +358,4 @@ define(['utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view',
            }
        }
    });
-});
+});
@@ -2,49 +2,46 @@
    This is the regular tool form.
 */
 define(['utils/utils', 'mvc/ui/ui-misc', 'mvc/tools/tools-form-base', 'mvc/tools/tools-jobs'],
-    function(Utils, Ui, ToolFormBase, ToolJobs) {
-
-    // create form view
+    function( Utils, Ui, ToolFormBase, ToolJobs ) {
    var View = ToolFormBase.extend({
-        // initialize
-        initialize: function(options) {
-            // execute button
+        initialize: function( options ) {
            var self = this;
-            var execute = new Ui.Button({
-                icon     : 'fa-check',
-                tooltip  : 'Execute: ' + options.name,
-                title    : 'Execute',
-                cls      : 'btn btn-primary',
-                floating : 'clear',
-                onclick  : function() {
-                    execute.wait();
-                    self.form.portlet.disable();
-                    ToolJobs.submit(self.form, self.options, function() {
-                        execute.unwait();
-                        self.form.portlet.enable();
-                    });
+            ToolFormBase.prototype.initialize.call( this, Utils.merge({
+                customize       : function( options ) {
+                    // build execute button
+                    options.buttons = {
+                        execute : execute_btn = new Ui.Button({
+                            icon     : 'fa-check',
+                            tooltip  : 'Execute: ' + options.name + ' (' + options.version + ')',
+                            title    : 'Execute',
+                            cls      : 'btn btn-primary',
+                            floating : 'clear',
+                            onclick  : function() {
+                                execute_btn.wait();
+                                self.form.portlet.disable();
+                                ToolJobs.submit( self.form, options, function() {
+                                    execute_btn.unwait();
+                                    self.form.portlet.enable();
+                                } );
+                            }
+                        })
+                    };
+
+                    // remap feature
+                    if ( options.job_id && options.job_remap ) {
+                        options.inputs[ 'rerun_remap_job_id' ] = {
+                            label       : 'Resume dependencies from this job',
+                            name        : 'rerun_remap_job_id',
+                            type        : 'select',
+                            display     : 'radio',
+                            ignore      : '__ignore__',
+                            value       : '__ignore__',
+                            options     : [ [ 'Yes', options.job_id ], [ 'No', '__ignore__' ] ],
+                            help        : 'The previous run of this tool failed and other tools were waiting for it to finish successfully. Use this option to resume those tools using the new output(s) of this tool run.'
+                        }
+                    }
                }
-            });
-            // add remap button
-            if (options.job_id && options.job_remap) {
-                options.inputs['rerun_remap_job_id'] = {
-                    label       : 'Resume dependencies from this job',
-                    name        : 'rerun_remap_job_id',
-                    type        : 'select',
-                    display     : 'radio',
-                    ignore      : '__ignore__',
-                    value       : '__ignore__',
-                    options     : [['Yes', options.job_id], ['No', '__ignore__']],
-                    help        : 'The previous run of this tool failed and other tools were waiting for it to finish successfully. Use this option to resume those tools using the new output(s) of this tool run.'
-                }
-            }
-            // create options dictionary
-            options = Utils.merge(options, {
-                buttons     : {
-                    execute : execute
-                }
-            });
-            ToolFormBase.prototype.initialize.call(this, options);
+            }, options ) );
        }
    });

@@ -34,23 +34,23 @@ return {
            data    : job_def,
            success : function(response) {
                callback && callback();
-                form.reciept(ToolTemplate.success(response));
+                form.$el.replaceWith( ToolTemplate.success( response ) );
+                // begin checking the history for updates
                self._refreshHdas();
            },
-            error   : function(response, response_full) {
+            error   : function(response) {
                callback && callback();
-                if (response && response.message && response.message.data) {
-                    var error_messages = form.data.matchResponse(response.message.data);
+                console.debug(response);
+                if (response && response.err_data) {
+                    var error_messages = form.data.matchResponse(response.err_data);
                    for (var input_id in error_messages) {
                        form.highlight(input_id, error_messages[input_id]);
                        break;
                    }
                } else {
-                    // show error message with details
-                    console.debug(response);
-                    response && form.modal.show({
+                    form.modal.show({
                        title   : 'Job submission failed',
-                        body    : ToolTemplate.error(job_def),
+                        body    : ( response && response.err_msg ) || ToolTemplate.error( job_def ),
                        buttons : {
                            'Close' : function() {
                                form.modal.hide();
@@ -135,11 +135,11 @@ return {

    /** Refreshes the history panel
    */
-    _refreshHdas: function() {
+    _refreshHdas: function(detailIds, options) {
        if (parent.Galaxy && parent.Galaxy.currHistoryPanel) {
-            parent.Galaxy.currHistoryPanel.refreshContents();
+            parent.Galaxy.currHistoryPanel.refreshContents(detailIds, options);
        }
    }
 };

-});
+});
@@ -97,12 +97,11 @@ define(['utils/utils',
        // update
        update: function(options) {
            // get options
-            this.options = Utils.merge(options, this.optionsDefault);
+            this.options = Utils.merge(options, this.options);

            // show message
            if (options.message != '') {
                this.$el.html(this._template(this.options));
-                this.$el.find('.alert').append(options.message);
                this.$el.fadeIn();

                // clear previous timeouts
@@ -128,7 +127,15 @@ define(['utils/utils',

        // template
        _template: function(options) {
-            return '<div class="ui-message alert alert-' + options.status + '"/>';
+            var cls_status = 'ui-message alert alert-' + options.status;
+            if (options.large) {
+                cls_status = ( ( options.status == 'success' && 'done' ) ||
+                               ( options.status == 'danger' && 'error' ) ||
+                                 options.status ) + 'messagelarge';
+            }
+            return  '<div class="' + cls_status + '" >' +
+                        options.message +
+                    '</div>';
        }
    });

@@ -13,7 +13,7 @@ var Base = Backbone.View.extend({
            visible     : true,
            data        : [],
            id          : Utils.uid(),
-            error_text  : 'No data available.',
+            error_text  : 'No options available.',
            wait_text   : 'Please wait...',
            multiple    : false
        });
@@ -7,8 +7,8 @@ var View = Backbone.View.extend({
    optionsDefault: {
        id          : Utils.uid(),
        cls         : 'ui-select',
-        error_text  : 'No data available',
-        empty_text  : 'No selection',
+        error_text  : 'No options available',
+        empty_text  : 'Nothing selected',
        visible     : true,
        wait        : false,
        multiple    : false,
@@ -116,7 +116,7 @@ return Backbone.View.extend({
        //

        // handle text editing event
-        this.$('#text-content').on('keyup', function(e) {
+        this.$('#text-content').on('change input', function(e) {
            self.model.set('url_paste', $(e.target).val());
            self.model.set('file_size', $(e.target).val().length);
            self._refreshReady();
@@ -110,7 +110,7 @@ return Backbone.View.extend({
                            .on('mousedown', function(e) { e.preventDefault(); });

        // handle text editing event
-        this.$('#text-content').on('keyup', function(e) {
+        this.$('#text-content').on('change input', function(e) {
            self.model.set('url_paste', $(e.target).val());
            self.model.set('file_size', $(e.target).val().length);
        });
@@ -7,35 +7,17 @@ module.exports = function( grunt ){
    "use strict";

    var app = grunt.config.get( 'app' ),
-        paths = grunt.config.get( 'paths' );
-
-    grunt.config( 'uglify', {
-        target : {
-            files: [{
-                expand : true,
-                cwd : paths.srcSymlink,
-                src : '**/*.js',
-                dest : paths.dist
-            }],
-        }
-    });
-
-    if (grunt.option('develop')){
-        grunt.config( 'uglify.options', {
+        paths = grunt.config.get( 'paths' ),
+        // uglify settings used when scripts are decompressed/not-minified
+        decompressedSettings = {
+            mangle   : false,
            beautify : true,
            compress : {
                drop_debugger : false
            }
-        });
-    } else {
-        grunt.config( 'uglify.target.options', {
-            sourceMap : true,
-            sourceMapName : function( path ){
-                // rewrite name to have all source maps in 'static/maps'
-                return path.replace( paths.dist, paths.maps ) + '.map';
-            }
-        });
-        grunt.config( 'uglify.options', {
+        },
+        // uglify settings used when scripts are compressed/minified
+        compressedSettings = {
            mangle : {
                screw_ie8 : true
            },
@@ -55,16 +37,39 @@ module.exports = function( grunt ){
                if_return : true,
                join_vars : true,
                cascade : true,
-                drop_console : true
+            }
+        };
+
+    grunt.config( 'uglify', {
+        target : {
+            files: [{
+                expand : true,
+                cwd : paths.srcSymlink,
+                src : '**/*.js',
+                dest : paths.dist
+            }],
+        }
+    });
+
+
+    if (grunt.option( 'develop' )){
+        grunt.config( 'uglify.options', decompressedSettings );
+
+    } else {
+        grunt.config( 'uglify.target.options', {
+            sourceMap : true,
+            sourceMapName : function( path ){
+                // rewrite name to have all source maps in 'static/maps'
+                return path.replace( paths.dist, paths.maps ) + '.map';
            }
        });
+        grunt.config( 'uglify.options', compressedSettings );
    }

-
    // -------------------------------------------------------------------------- decompress for easier debugging
    grunt.registerTask( 'decompress', function(){
        grunt.log.writeln( "decompressing... (don't forget to call 'grunt' again before committing)" );
-        grunt.config( 'uglify.options', { beautify: true });
+        grunt.config( 'uglify.options', decompressedSettings );
        grunt.config( 'uglify.target.options', {});
        grunt.task.run( 'uglify' );
    });
@@ -198,7 +198,7 @@
      <converter file="encodepeak_to_bgzip_converter.xml" target_datatype="bgzip"/>
      <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/>
    </datatype>
-    <datatype extension="pdf" type="galaxy.datatypes.images:Pdf" mimetype="application/pdf"/>
+    <datatype extension="pdf" type="galaxy.datatypes.images:Pdf" mimetype="application/pdf" display_in_upload="true"/>
    <datatype extension="pileup" type="galaxy.datatypes.tabular:Pileup" display_in_upload="true">
      <converter file="interval_to_bgzip_converter.xml" target_datatype="bgzip"/>
      <converter file="interval_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/>
@@ -262,6 +262,10 @@ paste.app_factory = galaxy.web.buildapp:app_factory
 # used for the cache
 #template_cache_path = database/compiled_templates

+# Set to false to disable various checks Galaxy will do to ensure it
+# can run job scripts before attempting to execute or submit them.
+#check_job_script_integrity = True
+
 # Citation related caching.  Tool citations information maybe fetched from
 # external sources such as http://dx.doi.org/ by Galaxy - the following
 # parameters can be used to control the caching used to store this information.
@@ -574,6 +578,12 @@ nglims_config_file = tool-data/nglims.yaml
 # nodejs to wrap connections in SSL).
 #dynamic_proxy_external_proxy=False

+# Additionally, when the dynamic proxy is proxied by an upstream server, you'll
+# want to specify a prefixed URL so both Galaxy and the proxy reside under the
+# same path that your cookies are under. This will result in a url like
+# https://FQDN/galaxy-prefix/gie_proxy for proxying
+#dynamic_proxy_prefix=gie_proxy
+
 # -- Logging and Debugging

 # If True, Galaxy will attempt to configure a simple root logger if a
@@ -626,9 +636,8 @@ nglims_config_file = tool-data/nglims.yaml
 # Debug enables access to various config options useful for development and
 # debugging: use_lint, use_profile, use_printdebug and use_interactive.  It
 # also causes the files used by PBS/SGE (submission script, output, and error)
-# to remain on disk after the job is complete.  Debug mode is disabled if
-# commented, but is uncommented by default in the sample config.
-debug = True
+# to remain on disk after the job is complete.
+#debug = False

 # Check for WSGI compliance.
 #use_lint = False
@@ -648,6 +657,20 @@ use_interactive = True
 # lot of CPU.
 #use_heartbeat = False

+# Log to Sentry
+# Sentry is an open source logging and error aggregation platform.  Setting
+# sentry_dsn will enable the Sentry middleware and errors will be sent to the
+# indicated sentry instance.  This connection string is available in your
+# sentry instance under <project_name> -> Settings -> API Keys.
+#sentry_dsn = None
+
+# Log to statsd
+# Statsd is an external statistics aggregator (https://github.com/etsy/statsd)
+# Enabling the following options will cause galaxy to log request timing and
+# other statistics to the configured statsd instance.
+#statsd_host=
+#statsd_port=8125
+
 # -- Data Libraries

 # These library upload options are described in much more detail in the wiki:
@@ -263,7 +263,7 @@
               options should be reserved for defining cluster
               specific options.
          -->
-          <env id="_JAVA_OPTIONS">-Xmx=6GB</env>
+          <env id="_JAVA_OPTIONS">-Xmx6G</env>
          <env id="ANOTHER_OPTION" raw="true">'5'</env> <!-- raw disables auto quoting -->
          <env file="/mnt/java_cluster/environment_setup.sh" /> <!-- will be sourced -->
          <env exec="module load javastuff/2.10" /> <!-- will be sourced -->
@@ -1,5 +1,4 @@
 [main]
-
 # Following options are ignored if using the Galaxy dynamic proxy but
 # are useful if mapping a range of ports for environment consumption.
 #apache_urls = False
@@ -7,26 +6,39 @@
 #ssl = False

 [docker]
-
-# Command to execute docker. For example `sudo docker` or `docker-lxc`.
-command = docker
+# Command to launch docker container. For example `sudo docker` or `docker-lxc`.
+# If you need to use a command like `sg` you can do that here, just be sure to
+# wrap all of the docker portion in single quotes. E.g. `sg 'docker' 'docker {docker_args}'`
+#
+# It is recommended that you use command_inject if you need to inject
+# additional parameters. This command string is re-used for a `docker inspect`
+# command and will likely cause errors if it is extensively modified, past the
+# usual group/sudo changes.
+#command = docker {docker_args}

 # The docker image name that should be started.
-image = bgruening/docker-ipython-notebook:dev
+image = bgruening/docker-ipython-notebook:15.10

 # Additional arguments that are passed to the `docker run` command.
-command_inject = --sig-proxy=true -e DEBUG=false
+#command_inject = --sig-proxy=true -e DEBUG=false

 # URL to access the Galaxy API with from the spawn Docker containter, if empty
 # this falls back to galaxy.ini's galaxy_infrastructure_url and finally to the
 # Docker host of the spawned container if that is also not set.
-#galaxy_url = 
+#galaxy_url =

 # The Docker hostname. It can be useful to run the Docker daemon on a different
 # host than Galaxy.
 #docker_hostname = localhost

 # Try to set the tempdirectory to world execute - this can fix the issue 
-# where	  'sudo docker' is not able to mount the folder otherwise.
+# where 'sudo docker' is not able to mount the folder otherwise.
 # "finalize namespace chdir to /import permission denied"
-wx_tempdir = False
+#wx_tempdir = False
+
+# Overwride the IE tempdirectory. This can be useful if you regular tempdir is
+# located on an NFS share, which does not work well as Docker volume. In this case
+# you can have a shared sshfs share which you can use as temporary directory to
+# share data between the IE and Galaxy.
+#docker_galaxy_temp_dir = None
+
@@ -33,18 +33,16 @@ if hda.datatype.__class__.__name__ != "Ipynb":
 else:
    shutil.copy( hda.file_name, empty_nb_path )

-
-## General IE specific
-# Access URLs for the notebook from within galaxy.
-notebook_access_url = ie_request.url_template('${PROXY_URL}/ipython/${PORT}/notebooks/ipython_galaxy_notebook.ipynb')
-notebook_login_url = ie_request.url_template('${PROXY_URL}/ipython/${PORT}/login?next=%2Fipython%2F${PORT}%2Ftree')
-
-
 # Add all environment variables collected from Galaxy's IE infrastructure
 ie_request.launch(env_override={
    'notebook_password': PASSWORD,
 })

+## General IE specific
+# Access URLs for the notebook from within galaxy.
+notebook_access_url = ie_request.url_template('${PROXY_URL}/ipython/notebooks/ipython_galaxy_notebook.ipynb')
+notebook_login_url = ie_request.url_template('${PROXY_URL}/ipython/login?next=${PROXY_PREFIX}%2Fipython%2Ftree')
+
 %>
 <html>
 <head>
@@ -1,19 +1,44 @@
 [main]
-# This cannot be changed
+# This CANNOT be changed. Eventually will be deprecated
 password_auth = True
-# Other
-apache_urls = False
-ssl = False
+# Following options are ignored if using the Galaxy dynamic proxy but
+# are useful if mapping a range of ports for environment consumption.
+#apache_urls = False
+#ssl = False

 [docker]
-command = docker
-image = erasche/docker-rstudio-notebook:dev
+# Command to launch docker container. For example `sudo docker` or `docker-lxc`.
+# If you need to use a command like `sg` you can do that here, just be sure to
+# wrap all of the docker portion in single quotes. E.g. `sg 'docker' 'docker {docker_args}'`
+#
+# It is recommended that you use command_inject if you need to inject
+# additional parameters. This command string is re-used for a `docker inspect`
+# command and will likely cause errors if it is extensively modified, past the
+# usual group/sudo changes.
+#command = docker {docker_args}

-# Additional arguments that are passed to the `docker run` command. `-u`
-# settings are completely ignored.
-command_inject = --sig-proxy=true -e DEBUG=false
+# The docker image name that should be started.
+image = erasche/docker-rstudio-notebook:15.10

-# URL to access the Galaxy API with from the spawned Docker container, if empty
+# Additional arguments that are passed to the `docker run` command.
+#command_inject = --sig-proxy=true -e DEBUG=false
+
+# URL to access the Galaxy API with from the spawn Docker containter, if empty
 # this falls back to galaxy.ini's galaxy_infrastructure_url and finally to the
-# Docker host of the spawned container, if that is also not set.
-#galaxy_url=
+# Docker host of the spawned container if that is also not set.
+#galaxy_url =
+
+# The Docker hostname. It can be useful to run the Docker daemon on a different
+# host than Galaxy.
+#docker_hostname = localhost
+
+# Try to set the tempdirectory to world execute - this can fix the issue 
+# where 'sudo docker' is not able to mount the folder otherwise.
+# "finalize namespace chdir to /import permission denied"
+#wx_tempdir = False
+
+# Overwride the IE tempdirectory. This can be useful if you regular tempdir is
+# located on an NFS share, which does not work well as Docker volume. In this case
+# you can have a shared sshfs share which you can use as temporary directory to
+# share data between the IE and Galaxy.
+#docker_galaxy_temp_dir = None
@@ -12,14 +12,6 @@ temp_dir = ie_request.temp_dir
 PASSWORD = ie_request.notebook_pw
 USERNAME = "galaxy"

-## General IE specific
-# Access URLs for the notebook from within galaxy.
-# TODO: Make this work without pointing directly to IE. Currently does not work
-# through proxy.
-notebook_pubkey_url = ie_request.url_template('${PROXY_URL}/rstudio/${PORT}/auth-public-key')
-notebook_access_url = ie_request.url_template('${PROXY_URL}/rstudio/${PORT}/')
-notebook_login_url =  ie_request.url_template('${PROXY_URL}/rstudio/${PORT}/auth-do-sign-in')
-
 # Did the user give us an RData file?
 if hda.datatype.__class__.__name__ == "RData":
    shutil.copy( hda.file_name, os.path.join(temp_dir, '.RData') )
@@ -27,8 +19,16 @@ if hda.datatype.__class__.__name__ == "RData":
 ie_request.launch(env_override={
    'notebook_username': USERNAME,
    'notebook_password': PASSWORD,
-    'cors_origin': ie_request.attr.proxy_url,
 })
+
+## General IE specific
+# Access URLs for the notebook from within galaxy.
+# TODO: Make this work without pointing directly to IE. Currently does not work
+# through proxy.
+notebook_pubkey_url = ie_request.url_template('${PROXY_URL}/rstudio/auth-public-key')
+notebook_access_url = ie_request.url_template('${PROXY_URL}/rstudio/')
+notebook_login_url =  ie_request.url_template('${PROXY_URL}/rstudio/auth-do-sign-in')
+
 %>
 <html>
 <head>
@@ -0,0 +1,71 @@
+# Galaxy Graph Visualization Framework  #
+
+The framework visualizes graphs and tries to provide functionality to easily manipulate them in Galaxy platform. 
+For graph construction and visualization [Cytoscape.js](http://js.cytoscape.org/) library is used.
+
+### Input formats ###
+
+As input files json or txt formats can be used. 
+
+Examples: 
+
+* json file 
+   
+```
+#!json
+
+{
+  "nodes":[
+			{
+				"id": 1,
+			},
+			{
+				"id": 2,
+			}
+          ],
+ "links":[
+			
+			{
+				"source":1,
+				"target":2,
+			}
+         ]
+}
+
+```
+
+* txt file with matrix notation
+
+
+```
+#!python
+
+1	3 4 5
+2	4 6
+3	1 5 6 2
+4	3
+5	
+6
+```
+Line "1	 3 4 5" can be read as: source: node "1" , target: node "3" ("4","5")
+
+
+### General Description ###
+
+The framework consists of 3 parts. On the left and right sides of the screen are 2 sliding panels. By default they are hidden. They can be opened by clicking the button on the upper part of each panel. The left panel is  Tool Panel where user can choose different features to manipulate the graph. On the right panel all available node information (metadata) is displayed. 
+In between these panels is the main screen where the visualization is displayed.
+
+
+### Features ###
+
+| Feature      | Description                    |
+| ------------- | ------------------------------ |
+| Lazy loading      |    If the graph contains more than 50 nodes lazy loading will be performed. At first the framework will load only the root nodes(nodes that have no incoming edges) and their child nodes. The child nodes that can be expanded (contain other nodes) will have bigger size. To load the rest of the graph *Expand* feature must be used on the expandable nodes. It's working only if the graph has at least one root node |
+| Delete   | Deletes selected nodes and edges. Can be performed by using corresponding button or shortcut key 'd'     |
+| Restore   | Restores the deleted nodes and edges form the very last deletion performed     |
+| Collapse   | Collapse feature can be performed on the selected node which has   outgoing edges. It hides all outgoing (child) nodes and edges of the selected node. Useful for compact view. Can be performed by using corresponding button or shortcut key 'c'. To uncollapse the node *Expand* or *Restore Structure* can be used      |
+|Expand| Expand feature can be used to load more nodes in case if lazy loading was used or to uncollapse collapsed nodes.Can be performed by using corresponding button or shortcut key 'e'     |
+|Metadata | Node metadata is always displayed on the right panel  |
+|Child/parent dependencies| To see incoming/outgoing nodes and edges of the selected node enable (check) corresponding feature on the left panel |
+|Labels| Node and edge labels can be shown/hidden by enabling/disabling corresponding option on the left Panel |
+|Export PNG| It is possible to export the visualization of the graph as a png picture. To perform the action click on the corresponding button |
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE visualization SYSTEM "../../visualization.dtd">
+<visualization name="Graph Visualization">
+    <data_sources>
+        <data_source>
+            <model_class>HistoryDatasetAssociation</model_class>
+            <test type="isinstance" test_attr="datatype" result_type="datatype">data.Text</test>
+            <to_param param_attr="id">dataset_id</to_param>
+        </data_source>
+    </data_sources>
+    <params>
+        <param type="dataset" var_name_in_template="hda" required="true">dataset_id</param>
+    </params>
+    <template>graphviz.mako</template>
+</visualization>
@@ -0,0 +1,197 @@
+
+
+#cy {
+  height: 100%;
+  width: 100%;
+  position: absolute; 
+  top: 0;
+}
+p {
+text-align:center;
+}
+
+
+.panel {
+
+	width:245px;
+	float:left;
+	height: 650px;
+	background:#d9dada;
+	position:relative;
+	left:-250px;
+	top: -10px;
+	font-family: Arial, "Helvetica Neue", Helvetica, sans-serif;
+	padding-left:5px;
+	 font-size: 14px;
+	
+}
+.slider-arrow {
+	top: -10px;
+	padding:5px;
+	width:10px;
+	float:left;
+	background:#d9dada;
+	font:700 16px  Arial, "Helvetica Neue", Helvetica, sans-serif;
+	color:#3498db;
+	text-decoration:none;
+	position:relative;
+	left:-250px;
+	
+}
+
+
+.btn  {
+  background: #3498db;
+  background-image: -webkit-linear-gradient(top, #3498db, #2980b9);
+  background-image: -moz-linear-gradient(top, #3498db, #2980b9);
+  background-image: -ms-linear-gradient(top, #3498db, #2980b9);
+  background-image: -o-linear-gradient(top, #3498db, #2980b9);
+  background-image: linear-gradient(to bottom, #3498db, #2980b9);
+  -webkit-border-radius: 28;
+  -moz-border-radius: 28;
+  border-radius: 6px;
+  font-family:  Arial, "Helvetica Neue", Helvetica, sans-serif;
+  color: #ffffff;
+  font-size: 14px;
+  padding: 8px 15px 8px 15px;
+  text-decoration: none;
+  width: 200px;
+}
+
+.btn:hover {
+  background: #3cb0fd;
+  background-image: -webkit-linear-gradient(top, #3cb0fd, #3498db);
+  background-image: -moz-linear-gradient(top, #3cb0fd, #3498db);
+  background-image: -ms-linear-gradient(top, #3cb0fd, #3498db);
+  background-image: -o-linear-gradient(top, #3cb0fd, #3498db);
+  background-image: linear-gradient(to bottom, #3cb0fd, #3498db);
+  text-decoration: none;
+  font-family:  Arial, "Helvetica Neue", Helvetica, sans-serif;
+}
+
+.btn:disabled {
+    background: #A4A4A4;
+}
+
+
+input[type=checkbox].css-checkbox {
+    display:none;
+}
+
+input[type=checkbox].css-checkbox + label.css-label {
+    padding-left:20px;
+    height:15px;
+    display:inline-block;
+    line-height:15px;
+    background-repeat:no-repeat;
+    background-position:0 0;
+    font-size:14px;
+    vertical-align:middle;
+    cursor:pointer;
+	font-family:  Arial, "Helvetica Neue", Helvetica, sans-serif;
+}
+
+input[type=checkbox].css-checkbox:checked + label.css-label {
+    background-position:0 -15px;
+}
+
+.css-label {
+    background-image: url(../img/lite-blue-check.png);
+}
+  
+
+.left {
+float: left;
+}
+
+
+#mainselection select {
+   border: 0;
+   color: #EEE;
+   background: transparent;
+   font-size: 14px;
+   font-family:  Arial, "Helvetica Neue", Helvetica, sans-serif;
+   width: 200px;
+   
+   -moz-appearance: none; /* Removes Default Firefox style*/
+  	-webkit-appearance: none;
+   padding: 5px 5px 5px 5px;
+}
+
+#mainselection {
+   overflow:hidden;
+   width:200px;
+   -moz-border-radius: 28px;
+   -webkit-border-radius: 28px;
+   border-radius: 9px 9px 9px 9px;
+   box-shadow: 1px 1px 1px #330033;
+   background: #3498db url(../img/15xvbd5.png) no-repeat
+	scroll 170px center;
+   
+}
+
+#nodeInfoDiv {
+  	width:250px;
+	float:right;
+	height: 650px;
+	background:#d9dada;
+	position:relative;
+	right:-255px;
+	top: -10px;
+	font-family: Arial, "Helvetica Neue", Helvetica, sans-serif;
+	
+	padding-right:5px;
+	font-size: 14px;
+    
+     
+}
+.slider-arrow-forNode {
+	top: -10px;
+	padding:5px;
+	width:10px;
+	float:right;
+	background:#d9dada;
+	font:700 16px  Arial, "Helvetica Neue", Helvetica, sans-serif;
+	color:#3498db;
+	text-decoration:none;
+	position:relative;
+	right:-255px;
+	
+}
+
+#linkDiv{
+	width:245;
+	margin:8px;
+
+
+}
+
+
+.CSSTableGenerator {
+	margin:8px;
+	padding:0;
+	width:240px;
+	box-shadow: 3px 3px 1px #888888;
+	border:2px solid #000000;
+	table-layout: fixed;
+	
+}.CSSTableGenerator table{
+   
+    border-spacing: 0;
+	width:240px;
+	height:100%;
+	margin:0px;padding:0px;
+}
+.CSSTableGenerator tr:nth-child(odd){ background-color:#aad4ff; }
+.CSSTableGenerator tr:nth-child(even)    { background-color:#ffffff; }.CSSTableGenerator td{
+	vertical-align:middle;	
+	border:1px solid #000000;
+	border-width:0px 1px 1px 0px;
+	text-align:left;
+	padding:3px;
+	font-size:14px;
+	font-family:Arial;
+	font-weight:normal;
+	color:#000000;
+	overflow:hidden;
+}
@@ -0,0 +1,215 @@
+var collapseOrder = 0;   // the order of collapsed nodes
+var collapseNodeCount = 0;  // the number of collapsed nodes 
+var num = 0;
+
+function colNode() { 	//// collapse node
+
+
+	var selectedNode = cy.nodes(':selected');
+	var connectedEdges = selectedNode.connectedEdges(function() {
+		return !this.target().anySame(selectedNode);
+	});
+	var connectedNodes = connectedEdges.targets();
+
+	collapseNodeCount = connectedNodes.length;
+	num = selectedNode.data('colNum', collapseNodeCount);
+	connectedNodes.addClass('collapsedNode' + collapseOrder);
+	connectedEdges.addClass('collapsedNode' + collapseOrder);
+	selectedNode.addClass('superNode');
+
+	addCollapsedEdges(selectedNode, collapseOrder);    
+
+// hide collapsed nodes
+	cy.style()
+		.selector('.collapsedNode' + collapseOrder)
+		.css({
+		'visibility': 'hidden'
+	})
+		.update();
+// change the size of superNode
+	cy.style()
+		.selector('.superNode')
+		.css({
+		'opacity': 0.8,
+		'width': 50,
+		'height': 50
+	})
+		.update();
+
+	cy.fit();
+
+
+	collapseOrder = collapseOrder + 1;
+
+	//}
+	$('.btn.colNode').prop('disabled', true);
+	console.log("shape = " + shape);
+
+}
+
+/// uncollapse(expand) superNode
+
+function unColNode() { 
+
+	var selectedNode = cy.nodes(':selected');
+	var connectedEdges = selectedNode.connectedEdges(function() {
+		return !this.target().anySame(selectedNode);
+	});
+	var connectedNodes = connectedEdges.targets();
+	cy.style()
+		.selector('.superNode')
+		.update();
+
+	selectedNode.removeClass('superNode');
+
+	for (var i = collapseOrder; i >= 0; i--) {
+
+		if (connectedNodes.hasClass('collapsedNode' + i) || connectedEdges.hasClass('collapsedNode' + i)) {
+
+			connectedNodes.removeClass('collapsedNode' + i);
+			connectedEdges.removeClass('collapsedNode' + i);
+			removeCollapsedEdges(selectedNode, i);
+
+			cy.style()
+				.update(); // remove invisibility
+			cy.fit();
+		}
+
+	}
+
+	collapseOrder--;
+	
+	$('.btn.colNode').prop('disabled', true);  // disable collapse button
+
+}
+
+/// creating edges from superNode to childNodes of the collapsed Nodes
+function addCollapsedEdges(selectedNode, collapseOrder) {
+
+	var connectedEdges = selectedNode.connectedEdges(function() {
+		return !this.target().anySame(selectedNode);
+	});
+
+	var connectedNodes = connectedEdges.targets();
+	var newTargetEdges = connectedNodes.connectedEdges(function() {
+		return !this.target().anySame(connectedNodes);
+	});
+	var newSourceEdges = connectedNodes.connectedEdges(function() {
+		return !this.source().anySame(connectedNodes);
+	});
+
+	var newTargetNodes = newTargetEdges.targets();
+	var newSourceNodes = newSourceEdges.sources();
+
+
+	newTargetNodes.each(function(i, ele) {
+
+		if (ele.edgesTo(selectedNode).sources().data('id') == selectedNode.data('id') || ele.data('id') == selectedNode.data('id')) { // preventing duplicate edges 
+
+			ele = ele + 1;
+		} else {
+
+
+			cy.add({
+				group: "edges",
+				data: {
+					source: selectedNode.data('id'),
+					target: ele.data('id')
+				}
+			})
+				.addClass('virtualEdges' + collapseOrder)
+		}
+	});
+
+
+
+
+	newSourceNodes.each(function(i, ele) {
+
+		if (ele.edgesTo(selectedNode).targets().data('id') == selectedNode.data('id') || ele.data('id') == selectedNode.data('id')) {
+
+			ele = ele + 1;
+		} else {
+
+			cy.add({
+				group: "edges",
+				data: {
+					source: ele.data('id'),
+					target: selectedNode.data('id')
+				}
+			})
+				.addClass('virtualEdges' + collapseOrder)
+		}
+	});
+
+
+
+}
+
+// removes the edges created by addCollapsedEdges
+function removeCollapsedEdges(selectedNode, collapseOrder) {
+
+	selectedNode.connectedEdges().each(function(i, ele) {
+
+		if (ele.hasClass('virtualEdges' + collapseOrder)) {
+
+			cy.remove(ele);
+		}
+	});
+
+}
+
+//reseting all superNodes
+function resetCollapse() {
+
+	for (var i = collapseOrder; i >= 0; i--) {
+
+		cy.style()
+			.selector('.collapsedNode' + i)
+			.css({
+			'opacity': 0.8,
+			'background-color': '#888888',
+			'line-color': '#ddd',
+			'target-arrow-color': '#ddd'
+		})
+			.update();
+
+		cy.style()
+			.selector('.superNode')
+			.css({
+			'opacity': 0.8,
+			'background-color': '#888888', //
+			'border-width': 0
+		})
+			.update();
+
+		cy.remove('.virtualEdges' + i);
+
+		cy.nodes().removeClass('superNode');
+		cy.nodes().removeClass('collapsedNode' + i);
+		cy.edges().removeClass('collapsedNode' + i);
+		cy.edges().removeClass('virtualEdges' + i);
+
+	}
+}
+
+
+
+// counts the number of collapsed nodes for the given superNode
+function countCollapse(nd) {
+
+	if (nd.hasClass('superNode')) {
+		Tip('contains ' + nd.data().colNum + ' node(s)', PADDING, 10);
+	} else if (nd.hasClass('toBeExpaned')) {
+
+		var eles = allcy.nodes();
+		var selectedNodeId = nd.id();
+		selectedNodeId = selectedNodeId.replace(/[^0-9\.]+/g, "");
+
+		var nodeCount = eles[selectedNodeId].outdegree();
+		Tip('contains ' + nodeCount + ' node(s)', PADDING, 10);
+
+	} else {
+		Tip('contains 0 node(s)', PADDING, 10);
+	}
+}
@@ -0,0 +1,165 @@
+ var showNodeLabel;
+ var showEdgeLabel;
+ var showOut;
+ var showIn;
+ var demoNodes = [];
+ var demoEdges = [];
+ var shape;
+ var allcy;
+ var cy;
+
+
+
+ $(function() { // on dom ready
+
+
+ 	$("body").css("overflow-x", "hidden"); //// remove horizontal scrollbar
+
+
+ }); // END on dom ready
+
+// shape of the graph
+ $("select option:selected").each(function() {
+ 	shape = $(this).val();
+
+ });
+ 
+ ///////// create graph
+ function createGraph(data) {
+
+ 	allcy = cytoscape({
+ 		headless: true,
+ 	});
+
+ 	cy = cytoscape({
+
+ 		container: document.getElementById('cy'),
+
+ 		layout: {
+ 			name: shape
+ 			//avoidOverlap: true,
+ 			//padding: 10
+ 		},
+ 		hideEdgesOnViewport: true,
+ 		hideLabelsOnViewport: true,
+ 		motionBlur: true,
+ 		textureOnViewport: true,
+
+ 		ready: function() {
+
+ 			window.cy = this;
+ 			cy.nodes().on("click", function(e) {
+
+ 				showNodeInfo(e.cyTarget);
+
+ 			});
+ 		},
+ 		style: cytoscape.stylesheet()
+ 			.selector('node')
+ 			.css({
+ 			'content': showNodeLabel,
+ 			'text-valign': 'center',
+ 			'background-color': '#888888',
+ 			'opacity': 0.8
+
+ 		})
+ 			.selector('edge')
+ 			.css({
+ 			'curve-style': 'unbundled-bezier',
+ 			'target-arrow-shape': 'triangle',
+ 			'width': 4,
+ 			'line-color': '#ddd',
+ 			'target-arrow-color': '#ddd',
+ 			'content': showEdgeLabel
+
+
+ 		})
+ 			.selector(':selected')
+ 			.css({
+ 			'background-color': '#FE2E64',
+ 			'line-color': '#FE2E64',
+ 			'target-arrow-color': '#FE2E64',
+ 			'source-arrow-color': '#FE2E64',
+ 			'opacity': 1
+ 		})
+ 			.selector('core')
+ 			.css({
+ 			'outside-texture-bg-color': 'white'
+
+ 		})
+ 	});
+ 	allcy.load(data);
+
+// if graph contains more than 50 nodes load only root nodes 
+ 	if (allcy.nodes().length > 50 && allcy.nodes().roots().length != 0 ) {
+			
+			var toAdd = allcy.nodes().roots().closedNeighborhood();
+ 			allcy.nodes().roots().addClass("roots");
+ 			
+			showNodesToExpand(toAdd);
+	 		cy.add(toAdd);
+ 			cy.load(cy.elements('*').jsons());
+
+ 		cy.style()
+ 			.selector('.toBeExpaned')
+ 			.css({
+
+ 			'width': 50,
+ 			'height': 50
+ 		})
+ 			.update(); 			
+ 			
+ 	} else {
+
+ 		cy.add(data);
+ 		cy.load(cy.elements('*').jsons());
+ 	}
+ 	cy.boxSelectionEnabled(true);
+
+
+ 	checkBoxes();
+
+
+ } // END create graph	
+
+
+ //// parse function for matrix data
+ function parseAndCreate(demoNodes, demoEdges) {
+ 	var data = {
+ 		nodes: demoNodes,
+ 		edges: demoEdges
+ 	};
+ 	createGraph(data);
+ }
+
+
+ ///// parsing function for json: for link/egde and "" issues
+ function parseJson(dataToParse) {
+
+ 	if (dataToParse.links == null) {
+
+ 		var links = dataToParse.edges;
+ 	} else {
+ 		var links = dataToParse.links;
+ 		dataToParse.edges = links;
+
+ 	}
+
+ 	var nodes = dataToParse.nodes;
+
+ 	for (var i = 0; i < nodes.length; i++) {
+
+ 		dataToParse.nodes[i].data = nodes[i];
+ 		dataToParse.nodes[i].data.id = '"' + nodes[i].id + '"';
+ 	}
+
+
+ 	for (var i = 0; i < links.length; i++) {
+
+ 		dataToParse.edges[i].data = links[i];
+ 		dataToParse.edges[i].data.source = '"' + links[i].source + '"';
+ 		dataToParse.edges[i].data.target = '"' + links[i].target + '"';
+ 	}
+
+ 	createGraph(dataToParse);
+ }
@@ -0,0 +1,104 @@
+/*
+tip_centerwindow.js  v. 1.21
+
+The latest version is available at
+http://www.walterzorn.com
+or http://www.devira.com
+or http://www.walterzorn.de
+
+Initial author: Walter Zorn
+Last modified: 3.6.2008
+
+Extension for the tooltip library wz_tooltip.js.
+Centers a sticky tooltip in the window's visible clientarea,
+optionally even if the window is being scrolled or resized.
+*/
+
+// Make sure that the core file wz_tooltip.js is included first
+if(typeof config == "undefined")
+	alert("Error:\nThe core tooltip script file 'wz_tooltip.js' must be included first, before the plugin files!");
+
+// Here we define new global configuration variable(s) (as members of the
+// predefined "config." class).
+// From each of these config variables, wz_tooltip.js will automatically derive
+// a command which can be passed to Tip() or TagToTip() in order to customize
+// tooltips individually. These command names are just the config variable
+// name(s) translated to uppercase,
+// e.g. from config. CenterWindow a command CENTERWINDOW will automatically be
+// created.
+
+//===================  GLOBAL TOOLTIP CONFIGURATION  =========================//
+config. CenterWindow = false	// true or false - set to true if you want this to be the default behaviour
+config. CenterAlways = false	// true or false - recenter if window is resized or scrolled
+//=======  END OF TOOLTIP CONFIG, DO NOT CHANGE ANYTHING BELOW  ==============//
+
+
+// Create a new tt_Extension object (make sure that the name of that object,
+// here ctrwnd, is unique amongst the extensions available for
+// wz_tooltips.js):
+var ctrwnd = new tt_Extension();
+
+// Implement extension eventhandlers on which our extension should react
+ctrwnd.OnLoadConfig = function()
+{
+	if(tt_aV[CENTERWINDOW])
+	{
+		// Permit CENTERWINDOW only if the tooltip is sticky
+		if(tt_aV[STICKY])
+		{
+			if(tt_aV[CENTERALWAYS])
+			{
+				// IE doesn't support style.position "fixed"
+				if(tt_ie)
+					tt_AddEvtFnc(window, "scroll", Ctrwnd_DoCenter);
+				else
+					tt_aElt[0].style.position = "fixed";
+				tt_AddEvtFnc(window, "resize", Ctrwnd_DoCenter);
+			}
+			return true;
+		}
+		tt_aV[CENTERWINDOW] = false;
+	}
+	return false;
+};
+// We react on the first OnMouseMove event to center the tip on that occasion
+ctrwnd.OnMoveBefore = Ctrwnd_DoCenter;
+ctrwnd.OnKill = function()
+{
+	if(tt_aV[CENTERWINDOW] && tt_aV[CENTERALWAYS])
+	{
+		tt_RemEvtFnc(window, "resize", Ctrwnd_DoCenter);
+		if(tt_ie)
+			tt_RemEvtFnc(window, "scroll", Ctrwnd_DoCenter);
+		else
+			tt_aElt[0].style.position = "absolute";
+	}
+	return false;
+};
+// Helper function
+function Ctrwnd_DoCenter()
+{
+	if(tt_aV[CENTERWINDOW])
+	{
+		var x, y, dx, dy;
+
+		// Here we use some functions and variables (tt_w, tt_h) which the
+		// extension API of wz_tooltip.js provides for us
+		if(tt_ie || !tt_aV[CENTERALWAYS])
+		{
+			dx = tt_GetScrollX();
+			dy = tt_GetScrollY();
+		}
+		else
+		{
+			dx = 0;
+			dy = 0;
+		}
+		// Position the tip, offset from the center by OFFSETX and OFFSETY
+		x = (tt_GetClientW() - tt_w) / 2 + dx + tt_aV[OFFSETX];
+		y = (tt_GetClientH() - tt_h) / 2 + dy + tt_aV[OFFSETY];
+		tt_SetTipPos(x, y);
+		return true;
+	}
+	return false;
+}
@@ -0,0 +1,88 @@
+/*
+tip_followscroll.js	v. 1.11
+
+The latest version is available at
+http://www.walterzorn.com
+or http://www.devira.com
+or http://www.walterzorn.de
+
+Initial author: Walter Zorn
+Last modified: 3.6.2008
+
+Extension for the tooltip library wz_tooltip.js.
+Lets a "sticky" tooltip keep its position inside the clientarea if the window
+is scrolled.
+*/
+
+// Make sure that the core file wz_tooltip.js is included first
+if(typeof config == "undefined")
+	alert("Error:\nThe core tooltip script file 'wz_tooltip.js' must be included first, before the plugin files!");
+
+// Here we define new global configuration variable(s) (as members of the
+// predefined "config." class).
+// From each of these config variables, wz_tooltip.js will automatically derive
+// a command which can be passed to Tip() or TagToTip() in order to customize
+// tooltips individually. These command names are just the config variable
+// name(s) translated to uppercase,
+// e.g. from config. FollowScroll a command FOLLOWSCROLL will automatically be
+// created.
+
+//===================	GLOBAL TOOLTIP CONFIGURATION	======================//
+config. FollowScroll = false		// true or false - set to true if you want this to be the default behaviour
+//=======	END OF TOOLTIP CONFIG, DO NOT CHANGE ANYTHING BELOW	==============//
+
+
+// Create a new tt_Extension object (make sure that the name of that object,
+// here fscrl, is unique amongst the extensions available for
+// wz_tooltips.js):
+var fscrl = new tt_Extension();
+
+// Implement extension eventhandlers on which our extension should react
+fscrl.OnShow = function()
+{
+	if(tt_aV[FOLLOWSCROLL])
+	{
+		// Permit FOLLOWSCROLL only if the tooltip is sticky
+		if(tt_aV[STICKY])
+		{
+			var x = tt_x - tt_GetScrollX(), y = tt_y - tt_GetScrollY();
+
+			if(tt_ie)
+			{
+				fscrl.MoveOnScrl.offX = x;
+				fscrl.MoveOnScrl.offY = y;
+				fscrl.AddRemEvtFncs(tt_AddEvtFnc);
+			}
+			else
+			{
+				tt_SetTipPos(x, y);
+				tt_aElt[0].style.position = "fixed";
+			}
+			return true;
+		}
+		tt_aV[FOLLOWSCROLL] = false;
+	}
+	return false;
+};
+fscrl.OnHide = function()
+{
+	if(tt_aV[FOLLOWSCROLL])
+	{
+		if(tt_ie)
+			fscrl.AddRemEvtFncs(tt_RemEvtFnc);
+		else
+			tt_aElt[0].style.position = "absolute";
+	}
+};
+// Helper functions (encapsulate in the class to avoid conflicts with other
+// extensions)
+fscrl.MoveOnScrl = function()
+{
+	tt_SetTipPos(fscrl.MoveOnScrl.offX + tt_GetScrollX(), fscrl.MoveOnScrl.offY + tt_GetScrollY());
+};
+fscrl.AddRemEvtFncs = function(PAddRem)
+{
+	PAddRem(window, "resize", fscrl.MoveOnScrl);
+	PAddRem(window, "scroll", fscrl.MoveOnScrl);
+};
+
@@ -0,0 +1,631 @@
+$(function() { // on dom ready
+
+
+	///// show/hide Labels
+	$('#nodeLabelCheck').change(function() {
+		if ($(this).is(":checked")) {
+
+			showNodeLabel = 'data(id)'; //.replace(/[^0-9\.]+/g, "")';  // change to label or name if needed
+		} else {
+			showNodeLabel = "";
+		}
+
+		cy.style()
+			.selector('node')
+			.css({
+			'content': showNodeLabel
+		})
+			.update();
+	});
+
+	$('#linkLabelCheck').change(function() {
+		if ($(this).is(":checked")) {
+			showEdgeLabel = "data(id)"; // change to label or name if needed
+		} else {
+			showEdgeLabel = "";
+		}
+
+		cy.style()
+			.selector('edge')
+			.css({
+			'content': showEdgeLabel
+		})
+			.update();
+	});
+
+
+	//////// change the shape of the graph 
+	$('#selectShape').change(function() {
+
+		$("select option:selected").each(function() {
+			shape = $(this).val();
+		});
+		cy.makeLayout({
+			'name': shape
+		})
+			.run();
+
+		$('#selectShape').blur();
+	});
+
+	////////  highlighting outgoing nodes
+
+	$('#showOutNode').change(function() {
+		if ($('#showOutNode').is(":checked")) {
+			showOut = true;
+			cy.nodes().on("tapend", highlightOut);
+			
+		} else {
+			showOut = false;
+			cy.nodes().off("tapend", highlightOut);
+			
+			resetHighlightOut(showIn, showOut);
+		}
+	});
+
+
+	////////  highlighting incoming nodes
+
+	$('#showInNode').change(function() {
+
+		if ($('#showInNode').is(":checked")) {
+			showIn = true;
+			cy.nodes().on("tapend", highlightIn);
+			
+
+		} else {
+			showIn = false;
+			cy.nodes().off("tapend", highlightIn);
+			resetHighlightIn(showIn, showOut);
+			
+			cy.nodes().removeClass('connectedNodeIn');
+		}
+
+	});
+
+	//// show number of contained nodes for a collapsed node
+	$('#collapseCount').change(function() {
+		checkBoxes();
+	});
+
+	////expand(Load) nodes
+	$('#expandNode').click(function() {
+		var selectedNode = cy.nodes(':selected');
+
+		if (selectedNode.hasClass('superNode')) {
+			unColNode();
+		} else {
+			expandNodes(selectedNode);
+			$('.btn.expNode').prop('disabled', true);
+		}
+
+	});
+
+	//// disable/unable buttons 
+	$(document).on('click', function() {
+
+		var selectedNode = cy.nodes(':selected');
+
+		if (selectedNode.outgoers().length == 0) {
+
+			$('.btn.colNode').prop('disabled', true);
+
+
+		} else if (selectedNode.hasClass('superNode')) {
+
+			$('.btn.colNode').prop('disabled', true);
+
+		} else {
+
+			$('.btn.colNode').prop('disabled', false);
+
+		}
+
+		if (selectedNode.hasClass('toBeExpaned') || selectedNode.hasClass('superNode')) {
+
+			$('.btn.expNode').prop('disabled', false);
+
+		} else {
+
+			$('.btn.expNode').prop('disabled', true);
+
+		}
+
+		if (cy.nodes(":selected").length > 0) {
+
+			$('.btn.delNode').prop('disabled', false);
+		} else {
+
+			$('.btn.delNode').prop('disabled', true);
+		}
+
+
+	});
+
+
+	///// keyboard commands
+
+	$(document).keypress(function(e) {
+
+		var selectedNode = cy.nodes(':selected');
+
+		if (e.which == 101 && selectedNode.size() != 0) { // 101 for 'e' = expand
+
+			if (selectedNode.hasClass('toBeExpaned')) {
+
+				expandNodes(selectedNode);
+
+			} else if (selectedNode.hasClass('superNode')) {
+
+				unColNode();
+			}
+
+
+		}
+		if (e.which == 99 && selectedNode.size() != 0 && !selectedNode.hasClass('superNode') && selectedNode.outgoers().length != 0) { // 99 for 'c' = collapse
+
+			colNode();
+
+		}
+
+		if (e.which == 100 && selectedNode.size() != 0) { // 100 for 'd' = delete
+			deleteSelectedNodes();
+		}
+
+
+	});
+
+	/////////////// tool Panel movement 
+
+	$(document).on('click', '.slider-arrow.show', function() {
+		$(".slider-arrow, .panel").animate({
+			left: "+=243"
+		}, 700, function() {
+			// Animation complete.
+		});
+		$(this).html('&laquo;').removeClass('show').addClass('hide');
+
+		document.getElementById('cy').style.left = "243px";
+		//document.getElementById('cy').style.width="80%";
+
+		cy.resize();
+
+
+	});
+
+	$(document).on('click', '.slider-arrow.hide', function() {
+		$(".slider-arrow, .panel").animate({
+			left: "-=243"
+		}, 700, function() {
+			// Animation complete.
+		});
+		$(this).html('&raquo;').removeClass('hide').addClass('show');
+		document.getElementById('cy').style.left = "0px";
+		// document.getElementById('cy').style.width="100%";
+		cy.resize();
+	});
+
+	//////////////////panel for Node info
+
+	$(document).on('click', '.slider-arrow-forNode.show', function() {
+		$(".slider-arrow-forNode, .nodePanel").animate({
+			right: "+=248"
+		}, 700, function() {
+			// Animation complete.
+		});
+		$(this).html('&raquo;').removeClass('show').addClass('hide');
+
+		document.getElementById('cy').style.right = "248px";
+		//document.getElementById('cy').style.width="80%";
+
+		cy.resize();
+
+
+	});
+
+	$(document).on('click', '.slider-arrow-forNode.hide', function() {
+		$(".slider-arrow-forNode, .nodePanel").animate({
+			right: "-=248"
+		}, 700, function() {
+			// Animation complete.
+		});
+		$(this).html('&laquo;').removeClass('hide').addClass('show');
+		document.getElementById('cy').style.right = "0px";
+		// document.getElementById('cy').style.width="100%";
+		cy.resize();
+	});
+
+
+
+
+}) // on dom ready END
+
+///////// finding outgoing Nodes		
+function highlightOut() {
+
+	var selectedNode = this;
+
+	var connectedNodes = selectedNode.outgoers();
+
+	if (selectedNode.hasClass('selectedNodeOut')) {
+
+		connectedNodes.removeClass('connectedNodeOut');
+		selectedNode.removeClass('selectedNodeOut');
+
+		cy.style()
+			.selector('.connectedNodeOut')
+			.css({
+			'opacity': 0.8
+		})
+			.update();
+	} else {
+		
+		cy.nodes().not(this).removeClass('connectedNodeOut');
+		cy.nodes().not(this).removeClass('selectedNodeOut');
+		cy.edges().not(this).removeClass('connectedNodeOut');
+
+		connectedNodes.addClass('connectedNodeOut');
+		selectedNode.addClass('selectedNodeOut');
+		cy.style()
+			.selector('.connectedNodeOut')
+			.css({
+			'background-color': '#FE2E64',
+			'line-color': '#FE2E64',
+			'target-arrow-color': '#FE2E64',
+			'source-arrow-color': '#FE2E64',
+			'opacity': 0.8
+		})
+			.update();
+
+		cy.style()
+			.selector('.selectedNodeOut')
+			.css({
+			'background-color': '#FE2E64',
+			'line-color': '#FE2E64',
+			'target-arrow-color': '#FE2E64',
+			'source-arrow-color': '#FE2E64',
+			'opacity': 0.8
+		})
+			.update();
+	}
+}
+
+/////// reset highlighting of outgoing nodes 		
+function resetHighlightOut(showIn, showOut) {
+
+	cy.style()
+		.selector('.connectedNodeOut')
+		.css({
+		'opacity': 0.8,
+		'background-color': '#888888',
+		'line-color': '#ddd',
+		'target-arrow-color': '#ddd'
+	})
+		.update();
+
+
+	if (!showIn && !showOut) {
+		cy.style()
+			.selector('.selectedNodeOut')
+			.css({
+			'opacity': 0.8,
+			'background-color': '#888888', //
+			'border-width': 0
+		})
+			.update();
+
+	}
+	cy.nodes().removeClass('selectedNodeOut');
+	cy.nodes().removeClass('connectedNodeOut');
+	cy.edges().removeClass('connectedNodeOut');
+}
+
+//////////// find inComing nodes
+
+function highlightIn() {
+
+	var selectedNode = this;
+	var connectedNodes = selectedNode.incomers();
+
+	if (selectedNode.hasClass('selectedNodeIn')) {
+
+		cy.style()
+			.selector('.connectedNodeIn')
+			.css({
+			'opacity': 0.8
+		})
+			.update();
+		connectedNodes.removeClass('connectedNodeIn');
+		selectedNode.removeClass('selectedNodeIn');
+
+	} else {
+
+		cy.nodes().not(this).removeClass('connectedNodeIn');
+		cy.nodes().not(this).removeClass('selectedNodeIn');
+		cy.edges().not(this).removeClass('connectedNodeIn');
+
+		connectedNodes.addClass('connectedNodeIn');
+		selectedNode.addClass('selectedNodeIn');
+		cy.style()
+			.selector('.connectedNodeIn')
+			.css({
+			'background-color': '#FE2E64',
+			'line-color': '#FE2E64',
+			'target-arrow-color': '#FE2E64',
+			'source-arrow-color': '#FE2E64',
+			'opacity': 0.8
+		})
+			.update();
+		cy.style()
+			.selector('.selectedNodeIn')
+			.css({
+			'background-color': '#FE2E64',
+			'line-color': '#FE2E64',
+			'target-arrow-color': '#FE2E64',
+			'source-arrow-color': '#FE2E64',
+			'opacity': 0.8
+		})
+			.update();
+	}
+}
+
+
+/////// reset highlighting of incoming nodes 		
+function resetHighlightIn(showIn, showOut) {
+
+	cy.style()
+		.selector('.connectedNodeIn')
+		.css({
+		'opacity': 0.8,
+		'background-color': '#888888',
+		'line-color': '#ddd',
+		'target-arrow-color': '#ddd'
+	})
+		.update();
+
+	if (!showIn && !showOut) {
+		cy.style()
+			.selector('.selectedNodeIn')
+			.css({
+			'opacity': 0.8,
+			'background-color': '#888888', //
+			'border-width': 0
+		})
+			.update();
+	}
+
+	cy.nodes().removeClass('selectedNodeIn');
+	cy.nodes().removeClass('connectedNodeIn');
+	cy.edges().removeClass('connectedNodeIn');
+
+}
+
+
+////// export PNG 	
+function exportFunction() {
+
+	var pngPic = cy.png();
+	downloadURI(pngPic, "graph"); //// tested on Chrome and Firefox
+}
+
+function downloadURI(uri, fname) {
+
+	var link = document.createElement("a");
+	link.download = fname;
+
+	link.href = uri;
+	document.body.appendChild(link);
+	link.click();
+	// Cleanup the DOM
+	document.body.removeChild(link);
+	delete link;
+}
+
+///// restore Graph structure
+function restorGraphStructure() {
+
+	resetCollapse();
+
+	var shape = "";
+	$("select option:selected").each(function() {
+		shape = $(this).val();
+	});
+	cy.makeLayout({
+		'name': shape
+	})
+		.run();
+
+}
+
+// show node info on the right panel
+function showNodeInfo(node) {
+
+	var str = "";
+	var nodeContent = node.data();
+
+	var fieldName = [];
+	var i = 0;
+	var values = [];
+	var j = 0;
+
+	for (var key in nodeContent) {
+
+		if (key == 'group' || key == 'data') {
+			continue;
+		}
+
+		fieldName[i] = key;
+		i++;
+		values[j] = nodeContent[key];
+		j++;
+
+	}
+
+
+	var childNodes = node.outgoers().nodes();
+	var childNum = childNodes.length;
+
+	var parentNodes = node.incomers().nodes();
+	var parentNum = parentNodes.length;
+	var degree = node.degree();
+
+	var table = createTable(fieldName, values, degree, parentNum, childNum);
+
+	$('#nodeInfoDiv').html("<p><strong>Node Description </p>" + table);
+	for (var i = 0; i < fieldName.length; i++) {
+
+		if (isValidUrl(values[i])) {
+
+			var e = document.createElement("div");
+			e.id = "linkDiv";
+			$('#nodeInfoDiv').append(e);
+			
+			var url = values[i];
+			
+			document.getElementById("linkDiv").innerHTML += "<br>" + fieldName[i] + "<br>";
+			$('<iframe id="iframeId" width = "240"/>').appendTo(document.getElementById("linkDiv")).prop('src', url);
+
+			i++;
+		}
+	}
+}
+
+function isValidUrl(str) {
+	var pattern = new RegExp(/((([A-Za-z]{3,9}:(?:\/\/)?)(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+|(?:www\.|[\-;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)((?:\/[\+~%\/\.\w\-_]*)?\??(?:[\-\+=&;%@\.\w_]*)#?(?:[\.\!\/\\\w]*))?)/); // fragment locater
+	if (!pattern.test(str)) {
+
+		return false;
+	} else {
+		return true;
+	}
+}
+
+function createTable(fieldName, values, degree, parentNum, childNum) {
+
+	var table = "<table class='CSSTableGenerator'>";
+
+	for (var i = 0; i < fieldName.length; i++) {
+
+
+		if (isValidUrl(values[i])) {
+			i++;
+		}
+
+		table += "<tr>" + "<td>" + fieldName[i] + "</td>" + "<td>" + values[i] + "</td>" + "</tr>";
+
+	}
+	table += "<tr>" + "<td>" + "degree" + "</td>" + "<td>" + degree + "</td>" + "</tr>";
+	table += "<tr>" + "<td>" + "number of child nodes" + "</td>" + "<td>" + childNum + "</td>" + "</tr>";
+	table += "<tr>" + "<td>" + "number of parent nodes" + "</td>" + "<td>" + parentNum + "</td>" + "</tr>";
+
+
+	table += "</table>";
+	return table;
+
+}
+
+
+/////// delete selected nodes
+
+var nodesToRemove;
+var edgesToRemove;
+
+function deleteSelectedNodes() {
+
+	nodesToRemove = cy.nodes(':selected');
+	edgesToRemove = nodesToRemove.connectedEdges();
+
+	cy.remove(nodesToRemove);
+	cy.remove(edgesToRemove);
+
+}
+/////// restore Deleted Nodes 
+function restoreDeletedNodes() {
+	nodesToRemove.restore();
+	edgesToRemove.restore();
+}
+
+// expand(load) nodes
+function expandNodes(selectedNode) {
+
+	cy.nodes().unbind("tapend");
+	var selectedNodeId = selectedNode.id();
+	selectedNodeId = selectedNodeId.replace(/[^0-9\.]+/g, ""); // removing "" 
+
+	var eles = allcy.nodes();
+
+	nodesToAdd = eles[selectedNodeId].outgoers();
+
+	showNodesToExpand(nodesToAdd);
+	cy.add(eles[selectedNodeId].outgoers());
+
+	selectedNode.removeClass('toBeExpaned'); 
+
+	$("select option:selected").each(function() {
+		shape = $(this).val();
+	});
+
+	cy.layout({
+		name: shape
+	});
+
+	checkBoxes();
+	cy.nodes().on("tapend", function(e) {
+
+		showNodeInfo(e.cyTarget);
+
+	});
+}
+
+function showNodesToExpand(toAdd) {
+
+	toAdd.nodes().forEach(function(ele) {
+
+		if (ele.outdegree() > 0 && !ele.hasClass('roots')) {
+			ele.addClass('toBeExpaned');
+		} 
+	});
+
+}
+
+
+// checkBox options
+
+function checkBoxes() {
+
+
+	if ($('#showInNode').is(":checked")) {
+		showIn = true;
+		cy.nodes().on("tapend", highlightIn);
+
+	} else {
+		showIn = false;
+		cy.nodes().off("tapend", highlightIn);
+		resetHighlightIn(showIn, showOut);
+		cy.nodes().removeClass('connectedNodeIn');
+	}
+	
+	if ($('#showOutNode').is(":checked")) {
+		showOut = true;
+		cy.nodes().on("tapend", highlightOut);
+	} else {
+		showOut = false;
+		cy.nodes().off("tapend", highlightOut);
+		resetHighlightOut(showIn, showOut);
+	}
+
+	if ($('#collapseCount').is(":checked")) {
+
+		cy.nodes().on("mouseover", function(event) {
+			var nd = event.cyTarget;
+			countCollapse(nd);
+		});
+		cy.nodes().on("mouseout", function(event) {
+			var nd = event.cyTarget;
+			UnTip();
+		});
+	} else {
+		cy.nodes().off("mouseover");
+	}
+
+}
@@ -0,0 +1,171 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
+
+    <title>${hda.name} | ${visualization_name}</title>
+<%
+    root = h.url_for( '/' )
+%>
+<script type="text/javascript" src="/static/scripts/libs/jquery/jquery.js"></script>
+    ${h.stylesheet_link( root + 'plugins/visualizations/graphviz/static/css/style.css' )}
+    
+    ${h.javascript_link( root + 'plugins/visualizations/graphviz/static/js/jquery.qtip.js' )}
+    ${h.javascript_link( root + 'plugins/visualizations/graphviz/static/js/cytoscape.min.js' )}
+    ${h.javascript_link( root + 'plugins/visualizations/graphviz/static/js/collapse.js' )}
+    ${h.javascript_link( root + 'plugins/visualizations/graphviz/static/js/toolPanelFunctions.js' )}
+    ${h.javascript_link( root + 'plugins/visualizations/graphviz/static/js/graphVis.js' )}
+
+</head>
+
+## ----------------------------------------------------------------------------
+<body>
+    ${h.javascript_link( root + 'plugins/visualizations/graphviz/static/js/wz_tooltip.js' )}
+
+    <script>
+        $(document).ready(function() {
+        
+            var hdaId   = '${trans.security.encode_id( hda.id )}',
+            hdaExt  = '${hda.ext}',
+            dataUrl = "${h.url_for( controller='/datasets', action='index')}/" + hdaId + "/display?to_ext=" + hdaExt;
+
+			// var reader = new FileReader();
+			
+             $.ajax({
+                 'async': true,
+                 'global': false,
+                 'url': dataUrl,
+                 'dataType': "json",
+                 'success': function(data) {
+                     parseJson(data);
+                     //  createGraph(data);
+                 },
+                 'error': function(data) {
+                 
+							jQuery.get(dataUrl, function(data) {
+  		 				
+  							var lines = data.split('\n');
+                         var chars;
+                         for (var line = 0; line < lines.length; line++) {
+                             chars = lines[line].split(/,?\s+/); // split by comma or space
+
+                             demoNodes.push({
+                                 data: {
+                                     id: chars[0],
+                                     label: chars[0]
+                                 }
+                             });
+
+                             for (var i = 1; i < chars.length; i++) {
+                                 demoEdges.push({
+                                     data: {
+                                         source: chars[0],
+                                         target: chars[i],
+                                         id: chars[0] + chars[i],
+
+                                     }
+                                 })
+                             }
+                         }
+					  
+  								parseAndCreate(demoNodes, demoEdges);
+  
+							});                     
+                     
+                  
+                 }
+             });
+
+        } );
+
+    </script>
+    <div id="cy"></div>
+    <div class="panel">
+        <br>
+        <div id="mainselection">
+            <select name="select" id="selectShape">
+                <option value="random" selected>Choose the shape</option>
+                <option value="random">random</option>
+                <option value="circle">circle</option>
+                <option value="grid">grid</option>
+                <option value="concentric">concentric</option>
+                <option value="breadthfirst">breadthfirst</option>
+            </select>
+        </div>
+        <br>
+
+        <input type="checkbox" class="css-checkbox" name="nlabelSelection" id="nodeLabelCheck" />
+        <label for="nodeLabelCheck" name="nodeLabelCheck" class="css-label">
+            show node label
+        </label>
+        <br>
+        <br>
+
+        <input type="checkbox" class="css-checkbox" name="elabelSelection" id="linkLabelCheck" />
+        <label for="linkLabelCheck" name="nodeLabelCheck" class="css-label">
+            show edge label
+        </label>
+        <br>
+        <br>
+
+        <input type="checkbox" class="css-checkbox" name="showOutgoing" id="showOutNode">
+        <label for="showOutNode" name="nodeLabelCheck" class="css-label">
+            highlight outgoing nodes
+        </label>
+        <br>
+        <br>
+
+        <input type="checkbox" class="css-checkbox" name="showInComing" id="showInNode">
+        <label for="showInNode" name="nodeLabelCheck" class="css-label">
+            highlight incoming nodes
+        </label>
+        <br>
+        <br>
+
+        <input type="checkbox" class="css-checkbox" name="showCollapsedNodeNum" id="collapseCount">
+        <label for="collapseCount" name="nodeLabelCheck" class="css-label">
+            show the number of collapsed nodes
+        </label>
+        <br>
+        <br>
+
+        <input type="button" class="btn colNode" name="collapseNode" id="collapseNode" value="Collapse Selected Node " onclick="colNode()" disabled="disabled">
+        <br>
+        <br>
+
+        <input type="button" class="btn expNode" name="expandNode" id="expandNode" value="Expand Selected Node " disabled="disabled">
+        <br>
+        <br>
+
+
+        <input type="button" class="btn delNode" name="deleteNodes" id="deleteNodes" value="Delete Selected Nodes " onclick="deleteSelectedNodes()" disabled="disabled">
+        <br>
+        <br>
+
+
+        <input type="button" class="btn" name="restoreNodes" id="restoreNodes" value="Restore Deleted Nodes" onclick="restoreDeletedNodes()">
+        <br>
+        <br>
+        <input type="button" class="btn" name="restore" id="restore" value="Restore The Structure  " onclick="restorGraphStructure()">
+        <br>
+        <br>
+        <input type="button" class="btn" name="export" id="export" value="Export PNG" onclick="exportFunction()">
+        <br>
+        <br>
+        <input type="button" class="btn" name="manual" id="manual" value="More Info" onclick="window.open('https://github.com/eteriSokhoyan/GraphVis')">
+        <br>
+        <br>
+    </div>
+
+    <a href="javascript:void(0);" class="slider-arrow show">&raquo;</a>
+    <div id="nodeInfoDiv" class="nodePanel"> 
+        <p> <strong>Node Description </strong></p>
+        <br><br>
+        <p> Please select a node </p>
+    </div>
+    <a href="javascript:void(0);" class="slider-arrow-forNode show">&laquo;</a>
+</body>
+
+</html>
@@ -0,0 +1,11 @@
+Special Topics in Galaxy Administration & Deployment Documentation
+==================================================================
+
+This documentation intends to function as a version-specific supplement to the
+`wiki <https://wiki.galaxyproject.org/Admin/>`__, not the primary admin
+documentation. These resources should be used together.
+
+.. toctree::
+   :maxdepth: 3
+
+   interactive_environments.rst
@@ -0,0 +1,685 @@
+Galaxy Interactive Environments (GIEs)
+======================================
+
+GIEs were a new feature back in Galaxy 15.05, leading with the release of the
+IPython IE. They were presented at GCC2015, and the RStudio IE was released as
+part of 15.07.
+
+A GIE is a docker container, launched by Galaxy, proxied by Galaxy, with some
+extra sugar inside the container to allow users to interact easily with their
+Galaxy histories.
+
+How GIEs Work
+-------------
+
+A GIE is primarily composed of a Docker container, and the Galaxy visualization
+component. GIE's are essentially visualization plugins which abuse the fact
+that they're rendered with mako templates. The mako templating language used in
+Viz plugins allows running python code, which in turn allows for us to launch
+docker containers. Once this container is launched, we notify a proxy built
+into Galaxy which helps coordinate a 1:1 mapping of users and their docker containers.
+
+Here's a simple diagram recapping the above:
+
+.. image:: interactive_environments.png
+
+Deploying GIEs
+--------------
+
+Deploying GIEs is not a trivial operation. They have complex interactions with
+numerous services, you'll need to be a fairly competent SysAdmin to debug all
+of the possible problems that can occur during deployment. After the initial
+hurdle, most find that GIEs require little to no maintenance.
+
+Setting up the Proxy
+^^^^^^^^^^^^^^^^^^^^
+
+Currently the Galaxy proxy is a NodeJS+Sqlite3 proxy.
+
+- Node has recently upgraded, and our proxy is pinned to an old version of
+  sqlite3. As such you'll currently need to have an older version of Node
+  available (0.10.X - 0.11.X vintage).
+- We're working on solutions in this space to provide a better deployment
+  mechanism here and fewer dependencies.
+- Please note that if you have NodeJS installed under Ubuntu, it often
+  installs to ``/usr/bin/nodejs``, whereas ``npm`` expects it to be
+  ``/usr/bin/node``. You will need to create that symlink yourself.
+
+Once Node and npm are ready to go, you'll need to install the dependencies
+
+.. code-block:: console
+
+    $ cd $GALAXY_ROOT/lib/galaxy/web/proxy/js
+    $ npm install
+
+Running ``node lib/main.js --help`` should produce some useful help text
+
+.. code-block::
+
+    Usage: main [options]
+
+    Options:
+
+    -h, --help             output usage information
+    -V, --version          output the version number
+    --ip <n>               Public-facing IP of the proxy
+    --port <n>             Public-facing port of the proxy
+    --cookie <cookiename>  Cookie proving authentication
+    --sessions <file>      Routes file to monitor
+    --verbose
+
+There are two ways to handle actually running the proxy. The first is to have
+Galaxy automatically launch the proxy as needed, the second is to manage it
+with something like Supervisord. The command for launching the proxy manually
+(or via supervisord) is
+
+.. code-block::  console
+
+    $ node $GALAXY_ROOT/lib/galaxy/web/proxy/js/lib/main.js --ip 0.0.0.0 \
+        --port 8800 --sessions $GALAXY_ROOT/database/session_map.sqlite \
+        --cookie galaxysession --verbose
+
+Configuring the Proxy
+^^^^^^^^^^^^^^^^^^^^^
+
+Configuration is all managed in ``galaxy.ini``
+
+.. code-block::  ini
+
+    dynamic_proxy_manage=True
+    dynamic_proxy_session_map=database/session_map.sqlite
+    dynamic_proxy_bind_port=8800
+    dynamic_proxy_bind_ip=0.0.0.0
+    dynamic_proxy_debug=True
+
+As you can see most of these variables map directly to the command line
+arguments to the NodeJS script. There are a few extra parameters which merit
+individual discussion
+
+.. code-block:: ini
+
+    dynamic_proxy_external_proxy=True
+    dynamic_proxy_prefix=gie_proxy
+
+The first option says that you have Galaxy and the Galaxy NodeJS proxy wrapped
+in an upstream proxy like Apache or NGINX. If you're using an upstream proxy, then
+you'll need to set this to true. This will cause Galaxy to connect users to the
+same port as Galaxy is being served on (so 80/443), rather than directing them
+to ``:8800``.
+
+The second option is closely entertwined with the first option. When Galaxy is
+accessed, it sets a cookie. This cookie generally cannot be sent with requests
+to different domains and different ports, so Galaxy and the dynamic proxy must
+be accessible on the same port and protocol. A further restriction is the
+cookie path; if you're running Galaxy under a URL like
+``https://f.q.d.n/galaxy/``, the cookie is only accessible to URLs that look
+like ``https://f.q.d.n/galaxy/*``. Galaxy and the Galaxy NodeJS proxy will take
+care of most of this for you, but you should be aware of how it functions. If
+your ``galaxysession`` cookie is not available in your request to the proxy
+URL, it will through up an error and you won't be able to connect. Back on
+topic, specifically, the second option sets the URL path that's used to
+differentiate requests that should go through the proxy to those that should go
+to Galaxy. You will need to add special upstream proxy configuration to handle
+this, and you'll need to use the same ``dynamic_proxy_prefix`` in your
+``galaxy.ini`` that you use in your URL routes.
+
+Apache
+
+.. code-block:: apache
+
+    # IPython specific. Other IEs may require their own routes.
+    ProxyPass        /galaxy/gie_proxy/ipython/api/kernels ws://localhost:8800/galaxy/gie_proxy/ipython/api/kernels
+
+    # Global GIE configuration
+    ProxyPass        /galaxy/gie_proxy http://localhost:8800/galaxy/gie_proxy
+    ProxyPassReverse /galaxy/gie_proxy http://localhost:8800/galaxy/gie_proxy
+
+    # Normal Galaxy configuration
+    ProxyPass        /galaxy http://localhost:8000/galaxy
+    ProxyPassReverse /galaxy http://localhost:8000/galaxy
+
+Please note you will need to be using apache2.4 with ``mod_proxy_wstunnel``.
+
+Nginx
+
+.. code-block::
+
+    # TODO, please PR / ping erasche on IRC if you have samples
+
+
+Docker on Another Host
+^^^^^^^^^^^^^^^^^^^^^^
+
+There are many reasons to run Interactive Environments on a separate host and
+not on your webserver, serving Galaxy. This feature has been available since
+15.07 and is used in production at the University of Freiburg.
+
+First you need to configure a second host to be Docker enabled. In the
+following we call this host ``gx-docker`` You need to start the Docker daemon
+and bind it to a TCP port, not to a socket as is the default. For example
+you can start the daemon with
+
+.. code-block:: console
+
+    $ docker -H 0.0.0.0:4243 -d
+
+On your client, the Galaxy webserver, you can now install a Docker client. This
+can also be done on older Systems like Scientific-Linux, CentOS 6, which does
+not have Docker support by default. The client just talks to the Docker daemon
+on host ``gx-docker``, and does not run anything itself, locally. You can test
+your configuration for example by starting busybox from your client on the
+Docker host with
+
+.. code-block:: console
+
+    $ docker -H tcp://gx-docker:4243 run -it busybox sh
+
+So far so good! Now we need to configure Galaxy to use our new Docker host
+to start the Interactive Environments. For that we need to edit the IPython GIE
+configuration, ``ipython.ini`` to use our custom docker host
+
+.. code-block:: ini
+
+    [main]
+
+    [docker]
+    command = docker -H tcp://gx-docker:4243 {docker_args}
+    image = bgruening/docker-ipython-notebook:dev
+    docker_hostname = gx-docker
+
+Please adapt your ``command`` and the ``image`` as needed.
+
+As next step we need to configure a share mount point between the Docker host
+and Galaxy. Unfortunately, this can not be a NFS mount. Docker does not like
+NFS yet. You could for example use a sshfs mount with the following script
+
+.. code-block:: bash
+
+    if mount | grep ^gx-docker:/var/tmp/gx-docker; then
+        echo "/var/tmp/gx-docker already mounted."
+    else
+        sshfs gx-docker:/var/tmp/gx-docker /var/tmp/gx-docker
+        echo 'Mounting ...'
+    fi
+
+This will let Galaxy and the Docker host share temporary files.
+
+Interactive Environments in Detail (and How to Build Your Own)
+--------------------------------------------------------------
+
+Unfortunately building a GIE isn't completely straightforward, and it's
+certainly not as simple as picking out an existing container and plugging it
+in. Here we'll go through build a "Hello, World" GIE which just displays a file
+from a user's history.
+
+Directory Layout
+^^^^^^^^^^^^^^^^
+
+The GIE directory layout looks identical to that of normal visualization
+plugins, for those familiar with developing those
+
+.. code-block:: console
+
+    $ tree $GALAXY_ROOT/config/plugins/interactive_environments/ipython/
+    config/plugins/interactive_environments/ipython/
+    ├── config
+    │   ├── ipython.ini
+    │   ├── ipython.ini.sample
+    │   └── ipython.xml
+    ├── static
+    │   └── js
+    │       └── ipython.js
+    └── templates
+        ├── ipython.mako
+        └── notebook.ipynb
+
+We'll use the variable ``{gie}`` to stand in for the name of your GIE. It
+should match ``[a-z]+``, like ``ipython`` or ``rstudio``. Here you can see the
+``config/`` directory with a ``{gie}.ini.sample`` providing docker and image
+configuration, and then ``{gie}.xml`` which outlines that it is a GIE.
+
+The static directory can hold resources such as javascript and css files. If
+you are actively developing a GIE, you'll need to restart Galaxy after adding
+any resources to that file, before they can be accessed in the browser.
+
+Lastly, and most importantly, there's the templates folder. This normally just
+contains ``{gie}.mako``, however the IPython file needs an extra template file.
+
+First Steps, Configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+We will name our GIE "helloworld", but you are free to name your's differently.
+We'll first need to create the directory structure and set up our
+configuration
+
+.. code-block:: console
+
+    $ mkdir -p $GALAXY_ROOT/config/plugins/interactive_environments/helloworld/{config,static,templates}
+    $ cd $GALAXY_ROOT/config/plugins/interactive_environments/helloworld/
+
+Next, you'll need to create the GIE plugin XML file ``config/helloworld.xml``
+
+.. code-block:: xml
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <!DOCTYPE interactive_environment SYSTEM "../../interactive_environments.dtd">
+    <!-- This is the name which will show up in the User's Browser -->
+    <interactive_environment name="HelloWorld">
+        <data_sources>
+            <data_source>
+                <model_class>HistoryDatasetAssociation</model_class>
+
+                <!-- here you filter out which types of datasets are
+                     appropriate for this GIE -->
+                <test type="isinstance" test_attr="datatype" result_type="datatype">tabular.Tabular</test>
+                <test type="isinstance" test_attr="datatype" result_type="datatype">data.Text</test>
+                <to_param param_attr="id">dataset_id</to_param>
+            </data_source>
+        </data_sources>
+        <params>
+            <param type="dataset" var_name_in_template="hda" required="true">dataset_id</param>
+        </params>
+        <!-- Be sure that your entrypoint name is correct! -->
+        <entry_point entry_point_type="mako">helloworld.mako</entry_point>
+    </interactive_environment>
+
+Once this is done, we can set up our INI file, ``config/helloworld.ini.sample`` which controlls docker interaction
+
+.. code-block:: ini
+
+    [main]
+    # Unused
+
+    [docker]
+    # Command to execute docker. For example `sudo docker` or `docker-lxc`.
+    #command = docker {docker_args}
+
+    # The docker image name that should be started.
+    image = hello-ie
+
+    # Additional arguments that are passed to the `docker run` command.
+    #command_inject = --sig-proxy=true -e DEBUG=false
+
+    # URL to access the Galaxy API with from the spawn Docker container, if empty
+    # this falls back to galaxy.ini's galaxy_infrastructure_url and finally to the
+    # Docker host of the spawned container if that is also not set.
+    #galaxy_url =
+
+    # The Docker hostname. It can be useful to run the Docker daemon on a different
+    # host than Galaxy.
+    #docker_hostname = localhost
+
+We've named our image ``hello-ie``, we'll get to creating that in a minute.
+
+Mako Templates
+^^^^^^^^^^^^^^
+
+Mako templates are very easy to use, and they allow significantly more
+flexibility than many other templating languages. It's because of this
+flexibility (and ability to write plain python code in them) that GIEs were
+possible to develop easily.
+
+In our ``templates/helloworld.mako``, we'll add the following
+
+.. code-block:: html+mako
+
+    <%namespace name="ie" file="ie.mako" />
+
+This line says to inherit from the ``ie.mako`` file that's available in
+``$GALAXY_ROOT/config/plugins/interactive_environments/common/templates/ie.mako``.
+Next we'll add the following
+
+.. code-block:: html+mako
+
+    <%
+    # Sets ID and sets up a lot of other variables
+    ie_request.load_deploy_config()
+
+    # Define a volume that will be mounted into the container.
+    # This is a useful way to provide access to large files in the container,
+    # if the user knows ahead of time that they will need it.
+    user_file = ie_request.volume(
+        hda.file_name, '/import/file.dat', how='ro')
+
+    # Launch the IE. This builds and runs the docker command in the background.
+    ie_request.launch(
+        volumes=[user_file],
+        env_override={
+            'custom': '42'
+        }
+    )
+
+    # Only once the container is launched can we template our URLs. The ie_request
+    # doesn't have all of the information needed until the container is running.
+    url = ie_request.url_template('${PROXY_URL}/helloworld/')
+    %>
+
+That mako snippet loaded the configuration from the INI files, launched the
+docker container, and then built a URL to the correct endpoint, through the
+Galaxy NodeJS proxy. Additionally we've set an environment variable named ``CUSTOM`` with the value ``42`` to be passed to the container, and we've attached the dataset the user selected (available in ``hda``) to the container as a read-only volume.
+
+We'll continue appending to our ``helloworld.mako`` the HTML code that's actually displayed to the user, when this template is rendered
+
+.. code-block:: html+mako
+
+    <html>
+    <head>
+    <!-- Loads some necessary javascript libraries. Specifically jquery,
+         toastr, and requirejs -->
+    ${ ie.load_default_js() }
+    </head>
+    <body>
+
+    <script type="text/javascript">
+    // see $GALAXY_ROOT/config/plugins/interactive_environments/common/templates/ie.mako to learn what this does
+    ${ ie.default_javascript_variables() }
+    var notebook_login_url = 'unused';
+    var notebook_access_url = '${ notebook_access_url }';
+
+    // Load code with require.js
+    ${ ie.plugin_require_config() }
+
+    // Load notebook
+    // This will load code from static/helloworld.js, often used to handle
+    // things like Login. The load_notebook function will eventually append
+    // an IFrame to the <div id="main" /> below.
+    requirejs(['interactive_environments', 'plugin/helloworld'], function(){
+        load_notebook(notebook_access_url);
+    });
+    </script>
+    <div id="main" width="100%" height="100%">
+    </div>
+    </body>
+    </html>
+
+We've glossed over some of the features of this file, but most IEs do a significant amount of "magic" in the top half of the mako template. For instance, the IPython notebook:
+
+- If the user is trying to run the IPython GIE Visualization on an existing notebook in their history, then that gets loaded into the docker container via the temp directory and set as the default notebook
+- Otherwise a default notebook is built for the user.
+
+The RStudio notebook:
+
+- generates a random password and configures the image to use this password
+- Copies in an RData file if the user has loaded one
+- sets some custom environment variables.
+
+
+Connecting the User to the Container via Javascript
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+With the mako template above finished, if you were to load this in your
+browser, not a lot would happen because we haven't built the hello-ie image,
+and we haven't used Javascript to connect the user with the container. In the
+tail end of the template, we set a variable ``notebook_access_url``. These are
+partially a legacy of how things used to be done and you're welcome to clean up
+your code according to your desires. Galaxy's NodeJS proxy handles the
+authentication of users, so you don't have to worry about it, and can just
+assume that only the correct user will have access to a given notebook.
+
+In the ``static/`` directory, we generally create a ``js/`` directory below that, and create a ``{gie}.js`` (so, ``static/js/helloworld.js``) file in there. That file will have a function, ``load_notebook`` which will check if the GIE is available, and when it is, display it to the user.
+
+We start by writing the load notebook function, which is pretty generic
+
+.. code-block:: javascript
+
+    // Load an interactive environment (IE) from a remote URL
+    // @param {String} notebook_access_url: the URL embeded in the page and loaded 
+    function load_notebook(notebook_access_url){
+        // When the page has completely loaded...
+        $( document ).ready(function() {
+            // Test if we can access the GIE, and if so, execute the function
+            // to load the GIE for the user.
+            test_ie_availability(notebook_access_url, function(){
+                _handle_notebook_loading(notebook_access_url);
+            });
+        });
+    }
+
+This function will display a spinner to the user to indicate process, and then make multiple requests to ``notebook_access_url``. That MUST return a 200 OK for the ``_handle_notebook_loading`` function to ever be called. 302s do not count!
+
+With that, we've almost completed the Javascript portion, just need to implement the function to display the GIE to the user in an iframe
+
+.. code-block:: javascript
+
+    function _handle_notebook_loading(notebook_access_url){
+        append_notebook(notebook_access_url);
+    }
+
+
+This function is very short. Historically, the GIE process involved a complex dance of:
+
+- generating a random password in the mako template
+- setting it as a javascript variable
+- passing it to the docker container
+- once the container was available, have the javascript automatically log a
+  user in (something browsers try to prevent since that's otherwise an XSS
+  vulnerability.)
+- hope everything worked
+
+Since the NodeJS proxy takes care of authentication/authorization, we can
+reduce the helloworld ``_handle_notebook_loading`` function to a simple
+``append_notebook`` call. You may wish to look at the IPython and RStudio GIEs
+for examples of the complex things that can be done at every step.
+
+The GIE Container
+^^^^^^^^^^^^^^^^^
+
+We'll build a simple container that just displays the dataset a user has
+selected to them. Remember when we attached a volume to the container? We'll
+make use of that now.
+
+GIE Containers (often) consist of:
+
+- Dockerfile
+- NGINX Proxy configuration
+- A custom startup script/entrypoint
+- A script to monitor traffic and kill unused containers
+
+We have to monitor the container's traffic and kill off unused containers,
+bceause no one is watching them. The user launches the container in Galaxy, and
+Galaxy immediately forgets the container exists. Thus, we say that if a
+container has no connections to TCP connections to itself, then it should
+commit suicide by killing the root process.
+
+Here's an example Dockerfile for our helloworld container
+
+.. code-block:: dockerfile
+
+    FROM ubuntu:14.04
+    # These environment variables are passed from Galaxy to the container
+    # and help you enable connectivity to Galaxy from within the container.
+    # This means your user can import/export data from/to Galaxy.
+    ENV DEBIAN_FRONTEND=noninteractive \
+        API_KEY=none \
+        DEBUG=false \
+        PROXY_PREFIX=none \
+        GALAXY_URL=none \
+        GALAXY_WEB_PORT=10000 \
+        HISTORY_ID=none \
+        REMOTE_HOST=none
+
+    RUN apt-get -qq update && \
+        apt-get install --no-install-recommends -y \
+        wget procps nginx python python-pip net-tools nginx
+
+    # Our very important scripts. Make sure you've run `chmod +x startup.sh
+    # monitor_traffic.sh` outside of the container!
+    ADD ./startup.sh /startup.sh
+    ADD ./monitor_traffic.sh /monitor_traffic.sh
+
+    # /import will be the universal mount-point for IPython
+    # The Galaxy instance can copy in data that needs to be present to the
+    # container
+    RUN mkdir /import
+
+    # Nginx configuration
+    COPY ./proxy.conf /proxy.conf
+
+    VOLUME ["/import"]
+    WORKDIR /import/
+
+    # EXTREMELY IMPORTANT! You must expose a SINGLE port on your container.
+    EXPOSE 80
+    CMD /startup.sh
+
+If you have questions on this, please feel free to contact us on IRC
+(`irc.freenode.net#galaxyproject <https://webchat.freenode.net/?channels=galaxyproject>`__).
+
+The proxy configuration is interesting, here we'll point NGINX to reverse proxy
+a service running on ``:8000`` inside the container. That port will be hosting
+a python process which serves up the directory contents of ``/import``, i.e.
+the file the user selected which was mounted as a volume into
+``/import/file.dat``
+
+.. code-block:: nginx
+
+    server {
+        listen 80;
+        server_name localhost;
+        access_log /var/log/nginx/localhost.access.log;
+
+        # Note the trailing slash used everywhere!
+        location PROXY_PREFIX/helloworld/ {
+            proxy_buffering off;
+            proxy_pass         http://127.0.0.1:8000/;
+            proxy_redirect     http://127.0.0.1:8000/ PROXY_PREFIX/helloworld/;
+        }
+    }
+
+
+And here we'll run that service in our ``startup.sh`` file
+
+.. code-block:: bash
+
+    #!/bin/bash
+    # First, replace the PROXY_PREFIX value in /proxy.conf with the value from
+    # the environment variable.
+    sed -i "s|PROXY_PREFIX|${PROXY_PREFIX}|" /proxy.conf;
+    # Then copy into the default location for ubuntu+nginx
+    cp /proxy.conf /etc/nginx/sites-enabled/default;
+
+    # Here you would normally start whatever service you want to start. In our
+    # example we start a simple directory listing service on port 8000
+    cd /import/ && python -mSimpleHTTPServer &
+
+    # Launch traffic monitor which will automatically kill the container if
+    # traffic stops
+    /monitor_traffic.sh &
+    # And finally launch nginx in foreground mode. This will make debugging
+    # easier as logs will be available from `docker logs ...`
+    nginx -g 'daemon off;'
+
+Lastly, our ``monitor_traffic.sh`` file is often re-used between containers, the only adjustment being the port that is looked at
+
+.. code-block:: bash
+
+    #!/bin/bash
+    while true; do
+        sleep 60
+        if [ `netstat -t | grep -v CLOSE_WAIT | grep ':80' | wc -l` -lt 3 ]
+        then
+            pkill nginx
+        fi
+    done
+
+With those files, ``monitor_traffic.sh``, ``Dockerfile``, ``startup.sh``, and ``proxy.conf``, you should be able to build your ``hello-ie`` container
+
+.. code-block:: bash
+
+    $ cd hello-ie
+    $ docker build -t hello-ie .
+
+Now, if everything went smoothly, you should be able to restart Galaxy and try out your new GIE on a tabular or text file!
+
+Debugging
+^^^^^^^^^
+
+When you launch your new GIE in Galaxy, your Galaxy logs should show something like the following:
+
+.. code-block:: console
+
+    Starting docker container for IE helloworld with command [docker run --sig-proxy=true -e DEBUG=false -e "GALAXY_URL=http://localhost/galaxy/" -e "CORS_ORIGIN=http://localhost" -e "GALAXY_WEB_PORT=8000" -e "HISTORY_ID=f2db41e1fa331b3e" -e "CUSTOM=42" -e "GALAXY_PASTER_PORT=8000" -e "PROXY_PREFIX=/galaxy/gie_proxy" -e "API_KEY=1712364174a0ff79b34e9a78fee3ca1c" -e "REMOTE_HOST=127.0.0.1" -e "USER_EMAIL=hxr@local.host" -d -P -v "/home/hxr/work/galaxy/database/tmp/tmp5HaqZy:/import/" -v "/home/hxr/work/galaxy/database/files/000/dataset_68.dat:/import/file.dat:ro" hello-ie]
+
+Here's the docker command written out in a more readable manner:
+
+.. code-block:: console
+
+    $ docker run --sig-proxy=true \
+        -d -P \
+        -e "API_KEY=1712364174a0ff79b34e9a78fee3ca1c" \
+        -e "CORS_ORIGIN=http://localhost" \
+        -e "CUSTOM=42" \
+        -e "DEBUG=false" \
+        -e "GALAXY_PASTER_PORT=8000" \
+        -e "GALAXY_URL=http://localhost/galaxy/" \
+        -e "GALAXY_WEB_PORT=8000" \
+        -e "HISTORY_ID=f2db41e1fa331b3e" \
+        -e "PROXY_PREFIX=/galaxy/gie_proxy" \
+        -e "REMOTE_HOST=127.0.0.1" \
+        -e "USER_EMAIL=hxr@local.host" \
+        -v "/home/hxr/work/galaxy/database/tmp/tmp5HaqZy:/import/" \
+        -v "/home/hxr/work/galaxy/database/files/000/dataset_68.dat:/import/file.dat:ro" \
+      hello-ie
+
+As you can see, a LOT is going on! We'll break it down further:
+
+- ``-d`` runs the container in daemon mode, it launches and the client
+  submitting the container finished
+- ``-P`` randomly assigns an unused port to the container for each ``EXPOSE``d
+  port from our ``Dockerfile``. This is why you must expose a port, and only
+  one port.
+- A large number of environment variables are set:
+
+    - The user's API key is provided, allowing you to access datasets and
+      submit jobs on their behalf. If you have an environment like
+      IPython/RStudio, it is **highly recommended** that you provide some magic
+      by which the user can use their API key without embedding it in the
+      ntoebook. If you do embed it somehow in a document that gets saved to
+      their history, anyone can impersonate that user if they get a hold of it.
+      In the IPython GIE we have a variable that just runs
+      ``os.environ.get('API_KEY')`` to avoid embedding it in their notebook.
+    - A CORS Origin is provided for very strict servers, but it may be easier
+      to simply void CORS requirements within the nginx proxy in your
+      container.
+    - Custom variables specified in your ``launch()`` command are available
+    - A ``DEBUG`` environment variable should be used to help admins debug
+      existing containers. You should use it to increase logging, not cleanup
+      temporary files, and so on.
+    - ``GALAXY_PASTER_PORT`` (deprecated) and ``GALAXY_WEB_PORT`` are the raw
+      port that Galaxy is listening on. You can use this to help decide how to
+      takl to Galaxy.
+    - ``GALAXY_URL`` is the URL that Galaxy should be accessible at. For
+      various reasons this may not be true. We recommend looking at our
+      implementation of `galaxy.py
+      <https://github.com/bgruening/docker-ipython-notebook/blob/15.07/galaxy.py>`__
+      which is a small utility script to provide API access to Galaxy to get
+      and fetch data, based on those environment variables.
+    - The ``HISTORY_ID`` of the current history the user is on is provided. In
+      the IPython/RStudio containers, we provide a dead simple method for users
+      to download datasets from their current history which will be visible to
+      them on the right hand side of their screen.
+    - A ``PROXY_PREFIX`` is provided which should be used in the nginx conf.
+    - ``REMOTE_HOST`` is another component used to test for a possible Galaxy
+      access path
+    - The user's email is made available, very convenient for webservices like
+      Entrez which require the user's email address. You can pre-fill it out
+      for them, making their life easier.
+    - Two volumes are mounted, one a temporary directory from Galaxy (rw), and one
+      the dataset the user selected (ro).
+
+- and finally the image is specified.
+
+Most of this information is usually required to build friendly, easy-to-use
+GIEs. One of the strong points of GIEs is their magic interaction with Galaxy.
+Here we've mounted a volume read-only, but in real life you may wish to provide
+connectivity like IPython and RStudio provide, allowing the user to load
+datasets on demand for interactive analysis, and then to store analysis
+artefacts (and a log of what was done inside the container, à la IPython's
+"notebooks") back to their current history.
+
+If everything went well, at this point you should see a directory listing show up:
+
+.. image:: interactive_environments_success.png
+
+If you find yourself encountering difficulties, the "Hello, World" IE is
+available in a `GitHub repo <https://github.com/erasche/hello-world-interactive-environment/releases/tag/v15.10>`__, and there are people on the IRC channel who can help debug.
@@ -7,6 +7,6 @@ to provide relatively concrete answers for specific tool development scenarios.
 ... interact with the Galaxy codebase interactively?
 ----------------------------------------------------

-This can be done with either IPython/Jupyter or a plain Python console, depending on your preferences::
+This can be done with either IPython/Jupyter or a plain python console, depending on your preferences::

    python -i scripts/db_shell.py
@@ -38,6 +38,8 @@ Contents

   Developer Documentation <dev/index>

+   Special topics in Administration <admin/index>
+
 Indices and tables
 ==================

@@ -1,132 +0,0 @@
-galaxy.jobs.runners.lwr_client package
-======================================
-
-.. automodule:: galaxy.jobs.runners.lwr_client
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Subpackages
-----------
-
-.. toctree::
-
-    galaxy.jobs.runners.lwr_client.staging
-    galaxy.jobs.runners.lwr_client.transport
-
-Submodules
----------
-
-galaxy.jobs.runners.lwr_client.action_mapper module
---------------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.action_mapper
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.amqp_exchange module
---------------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.amqp_exchange
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.amqp_exchange_factory module
-----------------------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.amqp_exchange_factory
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.client module
--------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.client
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.config_util module
-------------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.config_util
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.decorators module
------------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.decorators
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.destination module
-------------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.destination
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.interface module
-----------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.interface
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.job_directory module
---------------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.job_directory
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.manager module
---------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.manager
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.object_client module
---------------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.object_client
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.path_mapper module
-------------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.path_mapper
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.setup_handler module
---------------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.setup_handler
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.util module
------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.util
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-
@@ -1,28 +0,0 @@
-galaxy.jobs.runners.lwr_client.staging package
-==============================================
-
-.. automodule:: galaxy.jobs.runners.lwr_client.staging
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Submodules
----------
-
-galaxy.jobs.runners.lwr_client.staging.down module
--------------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.staging.down
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.staging.up module
------------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.staging.up
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-
@@ -1,28 +0,0 @@
-galaxy.jobs.runners.lwr_client.transport package
-================================================
-
-.. automodule:: galaxy.jobs.runners.lwr_client.transport
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Submodules
----------
-
-galaxy.jobs.runners.lwr_client.transport.curl module
----------------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.transport.curl
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.jobs.runners.lwr_client.transport.standard module
--------------------------------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr_client.transport.standard
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-
@@ -11,7 +11,6 @@ Subpackages

 .. toctree::

-    galaxy.jobs.runners.lwr_client
    galaxy.jobs.runners.state_handlers
    galaxy.jobs.runners.util

@@ -50,13 +49,6 @@ galaxy.jobs.runners.local module
    :undoc-members:
    :show-inheritance:

-galaxy.jobs.runners.lwr module
------------------------------
-
-.. automodule:: galaxy.jobs.runners.lwr
-    :members:
-    :undoc-members:
-    :show-inheritance:

 galaxy.jobs.runners.pbs module
 ------------------------------
@@ -171,6 +171,7 @@ class Configuration( object ):
        self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) )
        self.output_size_limit = int( kwargs.get( 'output_size_limit', 0 ) )
        self.retry_job_output_collection = int( kwargs.get( 'retry_job_output_collection', 0 ) )
+        self.check_job_script_integrity = string_as_bool( kwargs.get( "check_job_script_integrity", True ) )
        self.job_walltime = kwargs.get( 'job_walltime', None )
        self.job_walltime_delta = None
        if self.job_walltime is not None:
@@ -423,6 +424,9 @@ class Configuration( object ):
        self.new_lib_browse = string_as_bool( kwargs.get( 'new_lib_browse', False ) )
        # Error logging with sentry
        self.sentry_dsn = kwargs.get( 'sentry_dsn', None )
+        # Statistics and profiling with statsd
+        self.statsd_host = kwargs.get( 'statsd_host', '')
+        self.statsd_port = int( kwargs.get( 'statsd_port', 8125 ) )
        # Logging with fluentd
        self.fluent_log = string_as_bool( kwargs.get( 'fluent_log', False ) )
        self.fluent_host = kwargs.get( 'fluent_host', 'localhost' )
@@ -442,6 +446,7 @@ class Configuration( object ):
        self.dynamic_proxy_bind_port = int( kwargs.get( "dynamic_proxy_bind_port", "8800" ) )
        self.dynamic_proxy_bind_ip = kwargs.get( "dynamic_proxy_bind_ip", "0.0.0.0" )
        self.dynamic_proxy_external_proxy = string_as_bool( kwargs.get( "dynamic_proxy_external_proxy", "False" ) )
+        self.dynamic_proxy_prefix = kwargs.get( "dynamic_proxy_prefix", "gie_proxy" )

        # Default chunk size for chunkable datatypes -- 64k
        self.display_chunk_size = int( kwargs.get( 'display_chunk_size', 65536) )
@@ -1,6 +1,6 @@
 from galaxy.util import bunch
 from galaxy import exceptions
-from .structure import get_structure
+from .structure import get_structure, leaf

 CANNOT_MATCH_ERROR_MESSAGE = "Cannot match collection types."

@@ -13,10 +13,11 @@ class CollectionsToMatch( object ):
    def __init__( self ):
        self.collections = {}

-    def add( self, input_name, hdca, subcollection_type=None ):
+    def add( self, input_name, hdca, subcollection_type=None, linked=True ):
        self.collections[ input_name ] = bunch.Bunch(
            hdca=hdca,
            subcollection_type=subcollection_type,
+            linked=linked,
        )

    def has_collections( self ):
@@ -37,21 +38,32 @@ class MatchingCollections( object ):
    """

    def __init__( self ):
-        self.structure = None
+        self.linked_structure = None
+        self.unlinked_structures = []
        self.collections = {}

-    def __attempt_add_to_match( self, input_name, hdca, collection_type_description, subcollection_type ):
+    def __attempt_add_to_linked_match( self, input_name, hdca, collection_type_description, subcollection_type ):
        structure = get_structure( hdca, collection_type_description, leaf_subcollection_type=subcollection_type )
-        if not self.structure:
-            self.structure = structure
+        if not self.linked_structure:
+            self.linked_structure = structure
            self.collections[ input_name ] = hdca
        else:
-            if not self.structure.can_match( structure ):
+            if not self.linked_structure.can_match( structure ):
                raise exceptions.MessageException( CANNOT_MATCH_ERROR_MESSAGE )
            self.collections[ input_name ] = hdca

    def slice_collections( self ):
-        return self.structure.walk_collections( self.collections )
+        return self.linked_structure.walk_collections( self.collections )
+
+    @property
+    def structure( self ):
+        """Yield cross product of all unlinked datasets to linked dataset."""
+        effective_structure = leaf
+        for unlinked_structure in self.unlinked_structures:
+            effective_structure = effective_structure.multiply( unlinked_structure )
+        linked_structure = self.linked_structure or leaf
+        effective_structure = effective_structure.multiply( linked_structure )
+        return None if effective_structure.is_leaf else effective_structure

    @staticmethod
    def for_collections( collections_to_match, collection_type_descriptions ):
@@ -61,8 +73,13 @@ class MatchingCollections( object ):
        matching_collections = MatchingCollections()
        for input_key, to_match in collections_to_match.iteritems():
            hdca = to_match.hdca
-            subcollection_type = to_match.subcollection_type
            collection_type_description = collection_type_descriptions.for_collection_type( hdca.collection.collection_type )
-            matching_collections.__attempt_add_to_match( input_key, hdca, collection_type_description, subcollection_type )
+            subcollection_type = to_match.subcollection_type
+
+            if to_match.linked:
+                matching_collections.__attempt_add_to_linked_match( input_key, hdca, collection_type_description, subcollection_type )
+            else:
+                structure = get_structure( hdca, collection_type_description, leaf_subcollection_type=subcollection_type )
+                matching_collections.unlinked_structures.append( structure )

        return matching_collections
@@ -15,23 +15,34 @@ class Leaf( object ):
    def is_leaf( self ):
        return True

+    def clone( self ):
+        return self
+
+    def multiply( self, other_structure ):
+        return other_structure.clone()
+
+
 leaf = Leaf()


 class Tree( object ):

-    def __init__( self, dataset_collection, collection_type_description ):
+    def __init__( self, children, collection_type_description ):
+        self.children = children
        self.collection_type_description = collection_type_description
+
+    @staticmethod
+    def for_dataset_collection( dataset_collection, collection_type_description ):
        children = []
        for element in dataset_collection.elements:
            if collection_type_description.has_subcollections():
                child_collection = element.child_collection
                subcollection_type_description = collection_type_description.subcollection_type_description()  # Type description of children
-                children.append( ( element.element_identifier, Tree( child_collection, collection_type_description=subcollection_type_description )  ) )
+                tree = Tree.for_dataset_collection( child_collection, collection_type_description=subcollection_type_description )
+                children.append( ( element.element_identifier, tree ) )
            else:
                children.append( ( element.element_identifier, leaf ) )
-
-        self.children = children
+        return Tree( children, collection_type_description )

    def walk_collections( self, hdca_dict ):
        return self._walk_collections( dict_map( lambda hdca: hdca.collection, hdca_dict ) )
@@ -96,6 +107,21 @@ class Tree( object ):
            element_identifiers=element_identifiers,
        )

+    def multiply( self, other_structure ):
+        if other_structure.is_leaf:
+            return self.clone()
+
+        new_collection_type = self.collection_type_description.multiply( other_structure.collection_type_description )
+        new_children = []
+        for (identifier, structure) in self.children:
+            new_children.append( (identifier, structure.multiply( other_structure ) ) )
+
+        return Tree( new_children, new_collection_type )
+
+    def clone( self ):
+        cloned_children = map( lambda (identifier, element): (identifier, element.clone()), self.children )
+        return Tree( cloned_children, self.collection_type_description )
+

 def dict_map( func, input_dict ):
    return dict( [ ( k, func(v) ) for k, v in input_dict.iteritems() ] )
@@ -105,4 +131,4 @@ def get_structure( dataset_collection_instance, collection_type_description, lea
    if leaf_subcollection_type:
        collection_type_description = collection_type_description.effective_collection_type_description( leaf_subcollection_type )

-    return Tree( dataset_collection_instance.collection, collection_type_description )
+    return Tree.for_dataset_collection( dataset_collection_instance.collection, collection_type_description )
@@ -102,6 +102,10 @@ class CollectionTypeDescription( object ):
    def rank_type_plugin( self ):
        return self.collection_type_description_factory.type_registry.get( self.rank_collection_type() )

+    def multiply( self, other_collection_type ):
+        collection_type = map_over_collection_type( self, other_collection_type )
+        return self.collection_type_description_factory.for_collection_type( collection_type )
+
    def __str__( self ):
        return "CollectionTypeDescription[%s]" % self.collection_type

@@ -33,11 +33,11 @@ class Binary( data.Data ):

    @staticmethod
    def register_sniffable_binary_format(data_type, ext, type_class):
-        Binary.sniffable_binary_formats.append({"type": data_type, "ext": ext, "class": type_class})
+        Binary.sniffable_binary_formats.append({"type": data_type, "ext": ext.lower(), "class": type_class})

    @staticmethod
    def register_unsniffable_binary_ext(ext):
-        Binary.unsniffable_binary_formats.append(ext)
+        Binary.unsniffable_binary_formats.append(ext.lower())

    @staticmethod
    def is_sniffable_binary( filename ):
@@ -5,10 +5,9 @@ Uses pysam to bgzip a file

 usage: %prog in_file out_file
 """
-
-import tempfile
 import optparse
 import subprocess
+import tempfile

 from pysam import ctabix

@@ -6,15 +6,15 @@ Convert from GFF file to interval index file.
 usage:
    python gff_to_interval_index_converter.py [input] [output]
 """
-
 from __future__ import division

-import sys
 import fileinput
+import sys

-from galaxy.datatypes.util.gff_util import GFFReaderWrapper, GenomicInterval, convert_gff_coords_to_bed
 from bx.interval_index_file import Indexes

+from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GenomicInterval, GFFReaderWrapper
+

 def main():
    # Arguments
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # Dan Blankenberg
-
 import sys
+
 import bx.intervals.io

 assert sys.version_info[:2] >= ( 2, 4 )
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # Dan Blankenberg
-
 import sys
+
 import bx.intervals.io

 assert sys.version_info[:2] >= ( 2, 4 )
@@ -6,12 +6,13 @@ usage: %prog bed_file out_file
    -1, --cols1=N,N,N,N: Columns for chrom, start, end, strand in interval file
    -2, --cols2=N,N,N,N: Columns for chrom, start, end, strand in coverage file
 """
-from bx.intervals import io
-from bx.cookbook import doc_optparse
 import commands
-from os import environ
 import tempfile
 from bisect import bisect
+from os import environ
+
+from bx.cookbook import doc_optparse
+from bx.intervals import io

 INTERVAL_METADATA = ('chromCol',
                     'startCol',
@@ -14,8 +14,10 @@ where location is formatted as:
 and symbols are sorted in lexigraphical order.
 '''
 import optparse
+
 from bx.tabular.io import Comment
-from galaxy.datatypes.util.gff_util import GFFReaderWrapper, read_unordered_gtf, convert_gff_coords_to_bed
+
+from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GFFReaderWrapper, read_unordered_gtf


 def main():
@@ -6,10 +6,9 @@ Supported presets: bed, gff, vcf

 usage: %prog in_file out_file
 """
-
-import sys
-import os
 import optparse
+import os
+import sys

 from pysam import ctabix

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 # Dan Blankenberg
-
 import sys
+
 import bx.align.maf
+
 from galaxy.tools.util import maf_utilities

 assert sys.version_info[:2] >= ( 2, 4 )
@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 # Dan Blankenberg
-
 import sys
+
 import bx.align.maf
+
 from galaxy.tools.util import maf_utilities

 assert sys.version_info[:2] >= ( 2, 4 )
@@ -3,13 +3,14 @@
 """
 Convert from VCF file to interval index file.
 """
-
 from __future__ import division

 import optparse
-import galaxy_utils.sequence.vcf
+
 from bx.interval_index_file import Indexes

+import galaxy_utils.sequence.vcf
+

 def main():
    # Read options, args.
@@ -7,7 +7,9 @@ Read a wiggle track and print out a series of lines containing
 and fixedStep wiggle lines.
 """
 import sys
+
 import bx.wiggle
+
 from galaxy.tools.exception_handling import UCSCOutWrapper, UCSCLimitException


@@ -163,12 +163,22 @@ class GenomeGraphs( Tabular ):
    def sniff( self, filename ):
        """
        Determines whether the file is in gg format
+
+        >>> from galaxy.datatypes.sniff import get_test_fname
+        >>> fname = get_test_fname( 'test_space.txt' )
+        >>> GenomeGraphs().sniff( fname )
+        False
+        >>> fname = get_test_fname( '1.gg' )
+        >>> GenomeGraphs().sniff( fname )
+        True
        """
        f = open(filename, 'r')
        f.readline()  # header
-        rows = [f.readline().split()[1:] for x in range(3)]  # small sample
-        # headers = get_headers( filename, '\t' )
+        rows = [f.readline().split()[1:] for x in range(3)]  # small sample, trimming first column
        for row in rows:
+            if len(row) < 1:
+                # Must actually have at least one value
+                return False
            try:
                [float(x) for x in row]  # first col has been removed
            except:
@@ -1,22 +1,25 @@
 """
 Interval datatypes
 """
-
+import logging
+import math
 import os
 import sys
-import logging
 import tempfile
-import data
-from galaxy import util
-from galaxy.web import url_for
 import urllib
+
+import numpy
 from bx.intervals.io import GenomicIntervalReader, ParseError
+
+from galaxy import util
 from galaxy.datatypes import metadata
 from galaxy.datatypes.metadata import MetadataElement
+from galaxy.datatypes.sniff import get_headers
 from galaxy.datatypes.tabular import Tabular
 from galaxy.datatypes.util.gff_util import parse_gff_attributes
-from galaxy.datatypes.sniff import get_headers
-import math
+from galaxy.web import url_for
+
+import data
 import dataproviders

 log = logging.getLogger(__name__)
@@ -1245,9 +1248,6 @@ class Wiggle( Tabular, _RemoteCallMixin ):
        """
        Assumes we have a numpy file.
        """
-        # Maybe if we import here people will still be able to use Galaxy when numpy kills it
-        import numpy
-
        range = end - start
        # Determine appropriate resolution to plot ~1000 points
        resolution = ( 10 ** math.ceil( math.log10( range / 1000 ) ) )
@@ -634,32 +634,9 @@ class Registry( object ):
    def load_external_metadata_tool( self, toolbox ):
        """Adds a tool which is used to set external metadata"""
        # We need to be able to add a job to the queue to set metadata. The queue will currently only accept jobs with an associated
-        # tool.  We'll create a special tool to be used for Auto-Detecting metadata; this is less than ideal, but effective
-        # Properly building a tool without relying on parsing an XML file is near impossible...so we'll create a temporary file
-        tool_xml_text = """
-            <tool id="__SET_METADATA__" name="Set External Metadata" version="1.0.1" tool_type="set_metadata">
-              <type class="SetMetadataTool" module="galaxy.tools"/>
-              <requirements>
-                  <requirement type="package">samtools</requirement>
-              </requirements>
-              <action module="galaxy.tools.actions.metadata" class="SetMetadataToolAction"/>
-              <command>python "${set_metadata}" ${__SET_EXTERNAL_METADATA_COMMAND_LINE__}</command>
-              <inputs>
-                <param format="data" name="input1" type="data" label="File to set metadata on."/>
-                <param name="__ORIGINAL_DATASET_STATE__" type="hidden" value=""/>
-                <param name="__SET_EXTERNAL_METADATA_COMMAND_LINE__" type="hidden" value="">
-                  <sanitizer sanitize="False"/>
-                </param>
-              </inputs>
-              <configfiles>
-                <configfile name="set_metadata">from galaxy_ext.metadata.set_metadata import set_metadata; set_metadata()</configfile>
-              </configfiles>
-            </tool>
-            """
-        tmp_name = tempfile.NamedTemporaryFile()
-        tmp_name.write( tool_xml_text )
-        tmp_name.flush()
-        set_meta_tool = toolbox.load_hidden_tool( tmp_name.name )
+        # tool.  We'll load a special tool to be used for Auto-Detecting metadata; this is less than ideal, but effective
+        # Properly building a tool without relying on parsing an XML file is near difficult...so we bundle with Galaxy.
+        set_meta_tool = toolbox.load_hidden_lib_tool( "galaxy/datatypes/set_metadata_tool.xml" )
        self.set_external_metadata_tool = set_meta_tool
        self.log.debug( "Loaded external metadata tool: %s", self.set_external_metadata_tool.id )

@@ -0,0 +1,18 @@
+<tool id="__SET_METADATA__" name="Set External Metadata" version="1.0.1" tool_type="set_metadata">
+  <type class="SetMetadataTool" module="galaxy.tools"/>
+  <requirements>
+      <requirement type="package">samtools</requirement>
+  </requirements>
+  <action module="galaxy.tools.actions.metadata" class="SetMetadataToolAction"/>
+  <command>python "${set_metadata}" ${__SET_EXTERNAL_METADATA_COMMAND_LINE__}</command>
+  <inputs>
+    <param format="data" name="input1" type="data" label="File to set metadata on."/>
+    <param name="__ORIGINAL_DATASET_STATE__" type="hidden" value=""/>
+    <param name="__SET_EXTERNAL_METADATA_COMMAND_LINE__" type="hidden" value="">
+      <sanitizer sanitize="False"/>
+    </param>
+  </inputs>
+  <configfiles>
+    <configfile name="set_metadata">from galaxy_ext.metadata.set_metadata import set_metadata; set_metadata()</configfile>
+  </configfiles>
+</tool>
@@ -1,21 +1,24 @@
 """
 Tabular datatype
-
 """
+import csv
 import gzip
 import logging
 import os
-import csv
+import re
+import tempfile
+import subprocess
+
 from cgi import escape
+
 from galaxy import util
-from galaxy.datatypes import data
-from galaxy.datatypes import metadata
+from galaxy.datatypes import data, metadata
 from galaxy.datatypes.checkers import is_gzip
 from galaxy.datatypes.metadata import MetadataElement
 from galaxy.datatypes.sniff import get_headers
 from galaxy.util.json import dumps
+
 import dataproviders
-import re

 log = logging.getLogger(__name__)

@@ -691,6 +694,19 @@ class Vcf( Tabular ):
            # Found header line, get sample names.
            dataset.metadata.sample_names = line.split()[ 9: ]

+    @staticmethod
+    def merge(split_files, output_file):
+        stderr_f = tempfile.NamedTemporaryFile(prefix="bam_merge_stderr")
+        stderr_name = stderr_f.name
+        command = ["bcftools", "concat"] + split_files + ["-o", output_file]
+        log.info("Merging vcf files with command [%s]" % " ".join(command))
+        exit_code = subprocess.call( args=command, stderr=open( stderr_name, 'wb' ) )
+        with open(stderr_name, "rb") as f:
+            stderr = f.read().strip()
+        # Did merge succeed?
+        if exit_code != 0:
+            raise Exception("Error merging VCF files: %s" % stderr)
+
    # Dataproviders
    @dataproviders.decorators.dataprovider_factory( 'genomic-region',
                                                    dataproviders.dataset.GenomicRegionDataProvider.settings )
@@ -0,0 +1,4 @@
+chrX 100000 1.23
+chrX 100000 1.23
+chrX 100000 1.23
+chrX 100000 1.23
@@ -1,10 +1,11 @@
 """
 Provides utilities for working with GFF files.
 """
-
 import copy
+
 from bx.intervals.io import GenomicInterval, MissingFieldError, NiceReaderWrapper, ParseError, GenomicIntervalReader
 from bx.tabular.io import Header, Comment
+
 from galaxy.util.odict import odict


@@ -289,9 +289,8 @@ class JobConfiguration( object ):
        """
        log.debug('Loading job configuration from %s' % self.app.config.config_file)

-        # Always load local and lwr
-        self.runner_plugins = [dict(id='local', load='local', workers=self.app.config.local_job_queue_workers),
-                               dict(id='lwr', load='lwr', workers=self.app.config.cluster_job_queue_workers)]
+        # Always load local
+        self.runner_plugins = [dict(id='local', load='local', workers=self.app.config.local_job_queue_workers)]
        # Load tasks if configured
        if self.app.config.use_tasked_jobs:
            self.runner_plugins.append(dict(id='tasks', load='tasks', workers=self.app.config.local_task_queue_workers))
@@ -1406,7 +1405,7 @@ class JobWrapper( object ):
        job = has_metrics.get_job()
        per_plugin_properties = self.app.job_metrics.collect_properties( job.destination_id, self.job_id, self.working_directory )
        if per_plugin_properties:
-            log.info( "Collecting job metrics for %s" % has_metrics )
+            log.info( "Collecting metrics for %s %s" % ( type(has_metrics).__name__, getattr( has_metrics, 'id', None ) ) )
        for plugin, properties in per_plugin_properties.iteritems():
            for metric_name, metric_value in properties.iteritems():
                if metric_value is not None:
@@ -73,9 +73,9 @@ def build_command(

 def __externalize_commands(job_wrapper, commands_builder, remote_command_params, script_name="tool_script.sh"):
    local_container_script = join( job_wrapper.working_directory, script_name )
-    fh = file( local_container_script, "w" )
-    fh.write( "#!%s\n%s" % (DEFAULT_SHELL, commands_builder.build()))
-    fh.close()
+    with open( local_container_script, "w" ) as f:
+        script_contents = "#!%s\n%s" % (DEFAULT_SHELL, commands_builder.build())
+        f.write( script_contents )
    chmod( local_container_script, 0755 )

    commands = local_container_script
@@ -188,8 +188,6 @@ class BaseJobRunner( object ):
        raise NotImplementedError()

    def build_command_line( self, job_wrapper, include_metadata=False, include_work_dir_outputs=True ):
-        # TODO: Eliminate extra kwds no longer used (since LWR skips
-        # abstraction and calls build_command directly).
        container = self._find_container( job_wrapper )
        return build_command(
            self,
@@ -248,8 +246,8 @@ class BaseJobRunner( object ):

    def _handle_metadata_externally( self, job_wrapper, resolve_requirements=False ):
        """
-        Set metadata externally. Used by the local and lwr job runners where this
-        shouldn't be attached to command-line to execute.
+        Set metadata externally. Used by the Pulsar job runner where this
+        shouldn't be attached to command line to execute.
        """
        # run the metadata setting script here
        # this is terminate-able when output dataset/job is deleted
@@ -305,6 +303,41 @@ class BaseJobRunner( object ):
        options.update(**kwds)
        return job_script(**options)

+    def write_executable_script( self, path, contents, mode=0o755 ):
+        with open( path, 'w' ) as f:
+            f.write( contents )
+        os.chmod( path, mode )
+        self._handle_script_integrity( path )
+
+    def _handle_script_integrity( self, path ):
+        if not getattr( self.app.config, "check_job_script_integrity", True ):
+            return
+
+        script_integrity_verified = False
+        for i in range(5):
+            try:
+                proc = subprocess.Popen( [path], shell=True, env={"ABC_TEST_JOB_SCRIPT_INTEGRITY_XYZ": "1"} )
+                proc.wait()
+                if proc.returncode == 42:
+                    script_integrity_verified = True
+                    break
+
+                # Else we will sync and wait to see if the script becomes
+                # executable.
+                try:
+                    # sync file system to avoid "Text file busy" problems.
+                    # These have occurred both in Docker containers and on EC2 clusters
+                    # under high load.
+                    subprocess.check_call(["/bin/sync"])
+                except Exception:
+                    pass
+                time.sleep( .1 )
+            except Exception:
+                pass
+
+        if not script_integrity_verified:
+            raise Exception("Failed to write job script, could not verify job script integrity.")
+
    def _complete_terminal_job( self, ajs, **kwargs ):
        if ajs.job_wrapper.get_state() != model.Job.states.DELETED:
            self.work_queue.put( ( self.finish_job, ajs ) )
@@ -72,9 +72,7 @@ class ShellJobRunner( AsynchronousJobRunner ):
        )

        try:
-            fh = file(ajs.job_file, "w")
-            fh.write(script)
-            fh.close()
+            self.write_executable_script( ajs.job_file, script )
        except:
            log.exception("(%s) failure writing job script" % galaxy_id_tag )
            job_wrapper.fail("failure preparing job script", exception=True)
@@ -90,10 +90,7 @@ class CondorJobRunner( AsynchronousJobRunner ):
            slots_statement=galaxy_slots_statement,
        )
        try:
-            fh = file( executable, "w" )
-            fh.write( script )
-            fh.close()
-            os.chmod( executable, 0o750 )
+            self.write_executable_script( executable.job_file, script )
        except:
            job_wrapper.fail( "failure preparing job script", exception=True )
            log.exception( "(%s) failure preparing job script" % galaxy_id_tag )
@@ -148,10 +148,7 @@ class DRMAAJobRunner( AsynchronousJobRunner ):
        # fill in the DRM's job run template
        script = self.get_job_file(job_wrapper, exit_code_path=ajs.exit_code_file)
        try:
-            fh = file( ajs.job_file, "w" )
-            fh.write( script )
-            fh.close()
-            os.chmod( ajs.job_file, 0o755 )
+            self.write_executable_script( ajs.job_file, script )
        except:
            job_wrapper.fail( "failure preparing job script", exception=True )
            log.exception( "(%s) failure writing job script" % galaxy_id_tag )
@@ -68,8 +68,7 @@ class LocalJobRunner( BaseJobRunner ):
            'working_directory': job_wrapper.working_directory,
        }
        job_file_contents = self.get_job_file( job_wrapper, **job_script_props )
-        open( job_file, 'w' ).write( job_file_contents )
-        os.chmod( job_file, 0o755 )
+        self.write_executable_script( job_file, job_file_contents )
        return job_file, exit_code_path

    def queue_job( self, job_wrapper ):
@@ -1,648 +0,0 @@
-import logging
-
-from galaxy import model
-from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner
-from galaxy.jobs import ComputeEnvironment
-from galaxy.jobs import JobDestination
-from galaxy.jobs.command_factory import build_command
-from galaxy.tools.deps import dependencies
-from galaxy.util import string_as_bool_or_none
-from galaxy.util.bunch import Bunch
-from galaxy.util import specs
-
-import errno
-from time import sleep
-import os
-
-from .lwr_client import build_client_manager
-from .lwr_client import url_to_destination_params
-from .lwr_client import finish_job as lwr_finish_job
-from .lwr_client import submit_job as lwr_submit_job
-from .lwr_client import ClientJobDescription
-from .lwr_client import LwrOutputs
-from .lwr_client import ClientOutputs
-from .lwr_client import PathMapper
-
-log = logging.getLogger( __name__ )
-
-__all__ = [ 'LwrJobRunner' ]
-
-NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE = "LWR misconfiguration - LWR client configured to set metadata remotely, but remote LWR isn't properly configured with a galaxy_home directory."
-NO_REMOTE_DATATYPES_CONFIG = "LWR client is configured to use remote datatypes configuration when setting metadata externally, but LWR is not configured with this information. Defaulting to datatypes_conf.xml."
-GENERIC_REMOTE_ERROR = "Failed to communicate with remote job server."
-
-# Is there a good way to infer some default for this? Can only use
-# url_for from web threads. https://gist.github.com/jmchilton/9098762
-DEFAULT_GALAXY_URL = "http://localhost:8080"
-
-LWR_PARAM_SPECS = dict(
-    transport=dict(
-        map=specs.to_str_or_none,
-        valid=specs.is_in("urllib", "curl", None),
-        default=None
-    ),
-    cache=dict(
-        map=specs.to_bool_or_none,
-        default=None,
-    ),
-    url=dict(
-        map=specs.to_str_or_none,
-        default=None,
-    ),
-    galaxy_url=dict(
-        map=specs.to_str_or_none,
-        default=DEFAULT_GALAXY_URL,
-    ),
-    manager=dict(
-        map=specs.to_str_or_none,
-        default=None,
-    ),
-    amqp_consumer_timeout=dict(
-        map=lambda val: None if val == "None" else float(val),
-        default=None,
-    ),
-    amqp_connect_ssl_ca_certs=dict(
-        map=specs.to_str_or_none,
-        default=None,
-    ),
-    amqp_connect_ssl_keyfile=dict(
-        map=specs.to_str_or_none,
-        default=None,
-    ),
-    amqp_connect_ssl_certfile=dict(
-        map=specs.to_str_or_none,
-        default=None,
-    ),
-    amqp_connect_ssl_cert_reqs=dict(
-        map=specs.to_str_or_none,
-        default=None,
-    ),
-    # http://kombu.readthedocs.org/en/latest/reference/kombu.html#kombu.Producer.publish
-    amqp_publish_retry=dict(
-        map=specs.to_bool,
-        default=False,
-    ),
-    amqp_publish_priority=dict(
-        map=int,
-        valid=lambda x: 0 <= x and x <= 9,
-        default=0,
-    ),
-    # http://kombu.readthedocs.org/en/latest/reference/kombu.html#kombu.Exchange.delivery_mode
-    amqp_publish_delivery_mode=dict(
-        map=str,
-        valid=specs.is_in("transient", "persistent"),
-        default="persistent",
-    ),
-    amqp_publish_retry_max_retries=dict(
-        map=int,
-        default=None,
-    ),
-    amqp_publish_retry_interval_start=dict(
-        map=int,
-        default=None,
-    ),
-    amqp_publish_retry_interval_step=dict(
-        map=int,
-        default=None,
-    ),
-    amqp_publish_retry_interval_max=dict(
-        map=int,
-        default=None,
-    ),
-)
-
-
-class LwrJobRunner( AsynchronousJobRunner ):
-    """
-    LWR Job Runner
-    """
-    runner_name = "LWRRunner"
-
-    def __init__( self, app, nworkers, **kwds ):
-        """Start the job runner """
-        super( LwrJobRunner, self ).__init__( app, nworkers, runner_param_specs=LWR_PARAM_SPECS, **kwds )
-        self._init_worker_threads()
-        galaxy_url = self.runner_params.galaxy_url
-        if galaxy_url:
-            galaxy_url = galaxy_url.rstrip("/")
-        self.galaxy_url = galaxy_url
-        self.__init_client_manager()
-        if self.runner_params.url:
-            # This is a message queue driven runner, don't monitor
-            # just setup required callback.
-            self.client_manager.ensure_has_status_update_callback(self.__async_update)
-        else:
-            self._init_monitor_thread()
-
-    def __init_client_manager( self ):
-        client_manager_kwargs = {}
-        for kwd in 'url', 'manager', 'cache', 'transport':
-            client_manager_kwargs[ kwd ] = self.runner_params[ kwd ]
-        for kwd in self.runner_params.keys():
-            if kwd.startswith( 'amqp_' ):
-                client_manager_kwargs[ kwd ] = self.runner_params[ kwd ]
-        self.client_manager = build_client_manager(**client_manager_kwargs)
-
-    def url_to_destination( self, url ):
-        """Convert a legacy URL to a job destination"""
-        return JobDestination( runner="lwr", params=url_to_destination_params( url ) )
-
-    def check_watched_item(self, job_state):
-        try:
-            client = self.get_client_from_state(job_state)
-            status = client.get_status()
-        except Exception:
-            # An orphaned job was put into the queue at app startup, so remote server went down
-            # either way we are done I guess.
-            self.mark_as_finished(job_state)
-            return None
-        job_state = self.__update_job_state_for_lwr_status(job_state, status)
-        return job_state
-
-    def __update_job_state_for_lwr_status(self, job_state, lwr_status):
-        if lwr_status == "complete":
-            self.mark_as_finished(job_state)
-            return None
-        if lwr_status == "failed":
-            self.fail_job(job_state)
-            return None
-        if lwr_status == "running" and not job_state.running:
-            job_state.running = True
-            job_state.job_wrapper.change_state( model.Job.states.RUNNING )
-        return job_state
-
-    def __async_update( self, full_status ):
-        while not hasattr( self.app, 'job_manager' ):
-            # The status update thread can start consuming before app is done initializing
-            log.debug( 'Received a status update message before app is initialized, waiting 5 seconds' )
-            sleep( 5 )
-        job_id = None
-        try:
-            job_id = full_status[ "job_id" ]
-            job, job_wrapper = self.app.job_manager.job_handler.job_queue.job_pair_for_id( job_id )
-            job_state = self.__job_state( job, job_wrapper )
-            self.__update_job_state_for_lwr_status(job_state, full_status["status"])
-        except Exception:
-            log.exception( "Failed to update LWR job status for job_id %s" % job_id )
-            raise
-            # Nothing else to do? - Attempt to fail the job?
-
-    def queue_job(self, job_wrapper):
-        job_destination = job_wrapper.job_destination
-
-        command_line, client, remote_job_config, compute_environment = self.__prepare_job( job_wrapper, job_destination )
-
-        if not command_line:
-            return
-
-        try:
-            dependencies_description = LwrJobRunner.__dependencies_description( client, job_wrapper )
-            rewrite_paths = not LwrJobRunner.__rewrite_parameters( client )
-            unstructured_path_rewrites = {}
-            if compute_environment:
-                unstructured_path_rewrites = compute_environment.unstructured_path_rewrites
-
-            client_job_description = ClientJobDescription(
-                command_line=command_line,
-                input_files=self.get_input_files(job_wrapper),
-                client_outputs=self.__client_outputs(client, job_wrapper),
-                working_directory=job_wrapper.working_directory,
-                tool=job_wrapper.tool,
-                config_files=job_wrapper.extra_filenames,
-                dependencies_description=dependencies_description,
-                env=client.env,
-                rewrite_paths=rewrite_paths,
-                arbitrary_files=unstructured_path_rewrites,
-            )
-            job_id = lwr_submit_job(client, client_job_description, remote_job_config)
-            log.info("lwr job submitted with job_id %s" % job_id)
-            job_wrapper.set_job_destination( job_destination, job_id )
-            job_wrapper.change_state( model.Job.states.QUEUED )
-        except Exception:
-            job_wrapper.fail( "failure running job", exception=True )
-            log.exception("failure running job %d" % job_wrapper.job_id)
-            return
-
-        lwr_job_state = AsynchronousJobState()
-        lwr_job_state.job_wrapper = job_wrapper
-        lwr_job_state.job_id = job_id
-        lwr_job_state.old_state = True
-        lwr_job_state.running = False
-        lwr_job_state.job_destination = job_destination
-        self.monitor_job(lwr_job_state)
-
-    def __prepare_job(self, job_wrapper, job_destination):
-        """ Build command-line and LWR client for this job. """
-        command_line = None
-        client = None
-        remote_job_config = None
-        compute_environment = None
-        try:
-            client = self.get_client_from_wrapper(job_wrapper)
-            tool = job_wrapper.tool
-            remote_job_config = client.setup(tool.id, tool.version)
-            rewrite_parameters = LwrJobRunner.__rewrite_parameters( client )
-            prepare_kwds = {}
-            if rewrite_parameters:
-                compute_environment = LwrComputeEnvironment( client, job_wrapper, remote_job_config )
-                prepare_kwds[ 'compute_environment' ] = compute_environment
-            job_wrapper.prepare( **prepare_kwds )
-            self.__prepare_input_files_locally(job_wrapper)
-            remote_metadata = LwrJobRunner.__remote_metadata( client )
-            dependency_resolution = LwrJobRunner.__dependency_resolution( client )
-            metadata_kwds = self.__build_metadata_configuration(client, job_wrapper, remote_metadata, remote_job_config)
-            remote_command_params = dict(
-                working_directory=remote_job_config['working_directory'],
-                metadata_kwds=metadata_kwds,
-                dependency_resolution=dependency_resolution,
-            )
-            remote_working_directory = remote_job_config['working_directory']
-            # TODO: Following defs work for LWR, always worked for LWR but should be
-            # calculated at some other level.
-            remote_job_directory = os.path.abspath(os.path.join(remote_working_directory, os.path.pardir))
-            remote_tool_directory = os.path.abspath(os.path.join(remote_job_directory, "tool_files"))
-            container = self._find_container(
-                job_wrapper,
-                compute_working_directory=remote_working_directory,
-                compute_tool_directory=remote_tool_directory,
-                compute_job_directory=remote_job_directory,
-            )
-            command_line = build_command(
-                self,
-                job_wrapper=job_wrapper,
-                container=container,
-                include_metadata=remote_metadata,
-                include_work_dir_outputs=False,
-                remote_command_params=remote_command_params,
-            )
-        except Exception:
-            job_wrapper.fail( "failure preparing job", exception=True )
-            log.exception("failure running job %d" % job_wrapper.job_id)
-
-        # If we were able to get a command line, run the job
-        if not command_line:
-            job_wrapper.finish( '', '' )
-
-        return command_line, client, remote_job_config, compute_environment
-
-    def __prepare_input_files_locally(self, job_wrapper):
-        """Run task splitting commands locally."""
-        prepare_input_files_cmds = getattr(job_wrapper, 'prepare_input_files_cmds', None)
-        if prepare_input_files_cmds is not None:
-            for cmd in prepare_input_files_cmds:  # run the commands to stage the input files
-                if 0 != os.system(cmd):
-                    raise Exception('Error running file staging command: %s' % cmd)
-            job_wrapper.prepare_input_files_cmds = None  # prevent them from being used in-line
-
-    def get_output_files(self, job_wrapper):
-        output_paths = job_wrapper.get_output_fnames()
-        return [ str( o ) for o in output_paths ]   # Force job_path from DatasetPath objects.
-
-    def get_input_files(self, job_wrapper):
-        input_paths = job_wrapper.get_input_paths()
-        return [ str( i ) for i in input_paths ]  # Force job_path from DatasetPath objects.
-
-    def get_client_from_wrapper(self, job_wrapper):
-        job_id = job_wrapper.job_id
-        if hasattr(job_wrapper, 'task_id'):
-            job_id = "%s_%s" % (job_id, job_wrapper.task_id)
-        params = job_wrapper.job_destination.params.copy()
-        for key, value in params.iteritems():
-            if value:
-                params[key] = model.User.expand_user_properties( job_wrapper.get_job().user, value )
-        env = getattr( job_wrapper.job_destination, "env", [] )
-        return self.get_client( params, job_id, env )
-
-    def get_client_from_state(self, job_state):
-        job_destination_params = job_state.job_destination.params
-        job_id = job_state.job_id
-        return self.get_client( job_destination_params, job_id )
-
-    def get_client( self, job_destination_params, job_id, env=[] ):
-        # Cannot use url_for outside of web thread.
-        # files_endpoint = url_for( controller="job_files", job_id=encoded_job_id )
-
-        encoded_job_id = self.app.security.encode_id(job_id)
-        job_key = self.app.security.encode_id( job_id, kind="jobs_files" )
-        files_endpoint = "%s/api/jobs/%s/files?job_key=%s" % (
-            self.galaxy_url,
-            encoded_job_id,
-            job_key
-        )
-        get_client_kwds = dict(
-            job_id=str( job_id ),
-            files_endpoint=files_endpoint,
-            env=env
-        )
-        return self.client_manager.get_client( job_destination_params, **get_client_kwds )
-
-    def finish_job( self, job_state ):
-        stderr = stdout = ''
-        job_wrapper = job_state.job_wrapper
-        try:
-            client = self.get_client_from_state(job_state)
-            run_results = client.full_status()
-            remote_working_directory = run_results.get("working_directory", None)
-            stdout = run_results.get('stdout', '')
-            stderr = run_results.get('stderr', '')
-            exit_code = run_results.get('returncode', None)
-            lwr_outputs = LwrOutputs.from_status_response(run_results)
-            # Use LWR client code to transfer/copy files back
-            # and cleanup job if needed.
-            completed_normally = \
-                job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ]
-            cleanup_job = self.app.config.cleanup_job
-            client_outputs = self.__client_outputs(client, job_wrapper)
-            finish_args = dict( client=client,
-                                job_completed_normally=completed_normally,
-                                cleanup_job=cleanup_job,
-                                client_outputs=client_outputs,
-                                lwr_outputs=lwr_outputs )
-            failed = lwr_finish_job( **finish_args )
-
-            if failed:
-                job_wrapper.fail("Failed to find or download one or more job outputs from remote server.", exception=True)
-        except Exception:
-            message = GENERIC_REMOTE_ERROR
-            job_wrapper.fail( message, exception=True )
-            log.exception("failure finishing job %d" % job_wrapper.job_id)
-            return
-        if not LwrJobRunner.__remote_metadata( client ):
-            self._handle_metadata_externally( job_wrapper, resolve_requirements=True )
-        # Finish the job
-        try:
-            job_wrapper.finish(
-                stdout,
-                stderr,
-                exit_code,
-                remote_working_directory=remote_working_directory
-            )
-        except Exception:
-            log.exception("Job wrapper finish method failed")
-            job_wrapper.fail("Unable to finish job", exception=True)
-
-    def fail_job( self, job_state ):
-        """
-        Seperated out so we can use the worker threads for it.
-        """
-        self.stop_job( self.sa_session.query( self.app.model.Job ).get( job_state.job_wrapper.job_id ) )
-        job_state.job_wrapper.fail( getattr( job_state, "fail_message", GENERIC_REMOTE_ERROR ) )
-
-    def check_pid( self, pid ):
-        try:
-            os.kill( pid, 0 )
-            return True
-        except OSError as e:
-            if e.errno == errno.ESRCH:
-                log.debug( "check_pid(): PID %d is dead" % pid )
-            else:
-                log.warning( "check_pid(): Got errno %s when attempting to check PID %d: %s" % ( errno.errorcode[e.errno], pid, e.strerror ) )
-            return False
-
-    def stop_job( self, job ):
-        # if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished
-        job_ext_output_metadata = job.get_external_output_metadata()
-        if job_ext_output_metadata:
-            pid = job_ext_output_metadata[0].job_runner_external_pid  # every JobExternalOutputMetadata has a pid set, we just need to take from one of them
-            if pid in [ None, '' ]:
-                log.warning( "stop_job(): %s: no PID in database for job, unable to stop" % job.id )
-                return
-            pid = int( pid )
-            if not self.check_pid( pid ):
-                log.warning( "stop_job(): %s: PID %d was already dead or can't be signaled" % ( job.id, pid ) )
-                return
-            for sig in [ 15, 9 ]:
-                try:
-                    os.killpg( pid, sig )
-                except OSError as e:
-                    log.warning( "stop_job(): %s: Got errno %s when attempting to signal %d to PID %d: %s" % ( job.id, errno.errorcode[e.errno], sig, pid, e.strerror ) )
-                    return  # give up
-                sleep( 2 )
-                if not self.check_pid( pid ):
-                    log.debug( "stop_job(): %s: PID %d successfully killed with signal %d" % ( job.id, pid, sig ) )
-                    return
-                else:
-                    log.warning( "stop_job(): %s: PID %d refuses to die after signaling TERM/KILL" % ( job.id, pid ) )
-        else:
-            # Remote kill
-            lwr_url = job.job_runner_name
-            job_id = job.job_runner_external_id
-            log.debug("Attempt remote lwr kill of job with url %s and id %s" % (lwr_url, job_id))
-            client = self.get_client(job.destination_params, job_id)
-            client.kill()
-
-    def recover( self, job, job_wrapper ):
-        """Recovers jobs stuck in the queued/running state when Galaxy started"""
-        job_state = self.__job_state( job, job_wrapper )
-        job_wrapper.command_line = job.get_command_line()
-        state = job.get_state()
-        if state in [model.Job.states.RUNNING, model.Job.states.QUEUED]:
-            log.debug( "(LWR/%s) is still in running state, adding to the LWR queue" % ( job.get_id()) )
-            job_state.old_state = True
-            job_state.running = state == model.Job.states.RUNNING
-            self.monitor_queue.put( job_state )
-
-    def shutdown( self ):
-        super( LwrJobRunner, self ).shutdown()
-        self.client_manager.shutdown()
-
-    def __job_state( self, job, job_wrapper ):
-        job_state = AsynchronousJobState()
-        # TODO: Determine why this is set when using normal message queue updates
-        # but not CLI submitted MQ updates...
-        raw_job_id = job.get_job_runner_external_id() or job_wrapper.job_id
-        job_state.job_id = str( raw_job_id )
-        job_state.runner_url = job_wrapper.get_job_runner_url()
-        job_state.job_destination = job_wrapper.job_destination
-        job_state.job_wrapper = job_wrapper
-        return job_state
-
-    def __client_outputs( self, client, job_wrapper ):
-        work_dir_outputs = self.get_work_dir_outputs( job_wrapper )
-        output_files = self.get_output_files( job_wrapper )
-        client_outputs = ClientOutputs(
-            working_directory=job_wrapper.working_directory,
-            work_dir_outputs=work_dir_outputs,
-            output_files=output_files,
-            version_file=job_wrapper.get_version_string_path(),
-        )
-        return client_outputs
-
-    @staticmethod
-    def __dependencies_description( lwr_client, job_wrapper ):
-        dependency_resolution = LwrJobRunner.__dependency_resolution( lwr_client )
-        remote_dependency_resolution = dependency_resolution == "remote"
-        if not remote_dependency_resolution:
-            return None
-        requirements = job_wrapper.tool.requirements or []
-        installed_tool_dependencies = job_wrapper.tool.installed_tool_dependencies or []
-        return dependencies.DependenciesDescription(
-            requirements=requirements,
-            installed_tool_dependencies=installed_tool_dependencies,
-        )
-
-    @staticmethod
-    def __dependency_resolution( lwr_client ):
-        dependency_resolution = lwr_client.destination_params.get( "dependency_resolution", "local" )
-        if dependency_resolution not in ["none", "local", "remote"]:
-            raise Exception("Unknown dependency_resolution value encountered %s" % dependency_resolution)
-        return dependency_resolution
-
-    @staticmethod
-    def __remote_metadata( lwr_client ):
-        remote_metadata = string_as_bool_or_none( lwr_client.destination_params.get( "remote_metadata", False ) )
-        return remote_metadata
-
-    @staticmethod
-    def __use_remote_datatypes_conf( lwr_client ):
-        """ When setting remote metadata, use integrated datatypes from this
-        Galaxy instance or use the datatypes config configured via the remote
-        LWR.
-
-        Both options are broken in different ways for same reason - datatypes
-        may not match. One can push the local datatypes config to the remote
-        server - but there is no guarentee these datatypes will be defined
-        there. Alternatively, one can use the remote datatype config - but
-        there is no guarentee that it will contain all the datatypes available
-        to this Galaxy.
-        """
-        use_remote_datatypes = string_as_bool_or_none( lwr_client.destination_params.get( "use_remote_datatypes", False ) )
-        return use_remote_datatypes
-
-    @staticmethod
-    def __rewrite_parameters( lwr_client ):
-        return string_as_bool_or_none( lwr_client.destination_params.get( "rewrite_parameters", False ) ) or False
-
-    def __build_metadata_configuration(self, client, job_wrapper, remote_metadata, remote_job_config):
-        metadata_kwds = {}
-        if remote_metadata:
-            remote_system_properties = remote_job_config.get("system_properties", {})
-            remote_galaxy_home = remote_system_properties.get("galaxy_home", None)
-            if not remote_galaxy_home:
-                raise Exception(NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE)
-            metadata_kwds['exec_dir'] = remote_galaxy_home
-            outputs_directory = remote_job_config['outputs_directory']
-            configs_directory = remote_job_config['configs_directory']
-            working_directory = remote_job_config['working_directory']
-            # For metadata calculation, we need to build a list of of output
-            # file objects with real path indicating location on Galaxy server
-            # and false path indicating location on compute server. Since the
-            # LWR disables from_work_dir copying as part of the job command
-            # line we need to take the list of output locations on the LWR
-            # server (produced by self.get_output_files(job_wrapper)) and for
-            # each work_dir output substitute the effective path on the LWR
-            # server relative to the remote working directory as the
-            # false_path to send the metadata command generation module.
-            work_dir_outputs = self.get_work_dir_outputs(job_wrapper, job_working_directory=working_directory)
-            outputs = [Bunch(false_path=os.path.join(outputs_directory, os.path.basename(path)), real_path=path) for path in self.get_output_files(job_wrapper)]
-            for output in outputs:
-                for lwr_workdir_path, real_path in work_dir_outputs:
-                    if real_path == output.real_path:
-                        output.false_path = lwr_workdir_path
-            metadata_kwds['output_fnames'] = outputs
-            metadata_kwds['compute_tmp_dir'] = working_directory
-            metadata_kwds['config_root'] = remote_galaxy_home
-            default_config_file = os.path.join(remote_galaxy_home, 'galaxy.ini')
-            metadata_kwds['config_file'] = remote_system_properties.get('galaxy_config_file', default_config_file)
-            metadata_kwds['dataset_files_path'] = remote_system_properties.get('galaxy_dataset_files_path', None)
-            if LwrJobRunner.__use_remote_datatypes_conf( client ):
-                remote_datatypes_config = remote_system_properties.get('galaxy_datatypes_config_file', None)
-                if not remote_datatypes_config:
-                    log.warn(NO_REMOTE_DATATYPES_CONFIG)
-                    remote_datatypes_config = os.path.join(remote_galaxy_home, 'datatypes_conf.xml')
-                metadata_kwds['datatypes_config'] = remote_datatypes_config
-            else:
-                integrates_datatypes_config = self.app.datatypes_registry.integrated_datatypes_configs
-                # Ensure this file gets pushed out to the remote config dir.
-                job_wrapper.extra_filenames.append(integrates_datatypes_config)
-
-                metadata_kwds['datatypes_config'] = os.path.join(configs_directory, os.path.basename(integrates_datatypes_config))
-        return metadata_kwds
-
-
-class LwrComputeEnvironment( ComputeEnvironment ):
-
-    def __init__( self, lwr_client, job_wrapper, remote_job_config ):
-        self.lwr_client = lwr_client
-        self.job_wrapper = job_wrapper
-        self.local_path_config = job_wrapper.default_compute_environment()
-        self.unstructured_path_rewrites = {}
-        # job_wrapper.prepare is going to expunge the job backing the following
-        # computations, so precalculate these paths.
-        self._wrapper_input_paths = self.local_path_config.input_paths()
-        self._wrapper_output_paths = self.local_path_config.output_paths()
-        self.path_mapper = PathMapper(lwr_client, remote_job_config, self.local_path_config.working_directory())
-        self._config_directory = remote_job_config[ "configs_directory" ]
-        self._working_directory = remote_job_config[ "working_directory" ]
-        self._sep = remote_job_config[ "system_properties" ][ "separator" ]
-        self._tool_dir = remote_job_config[ "tools_directory" ]
-        version_path = self.local_path_config.version_path()
-        new_version_path = self.path_mapper.remote_version_path_rewrite(version_path)
-        if new_version_path:
-            version_path = new_version_path
-        self._version_path = version_path
-
-    def output_paths( self ):
-        local_output_paths = self._wrapper_output_paths
-
-        results = []
-        for local_output_path in local_output_paths:
-            wrapper_path = str( local_output_path )
-            remote_path = self.path_mapper.remote_output_path_rewrite( wrapper_path )
-            results.append( self._dataset_path( local_output_path, remote_path ) )
-        return results
-
-    def input_paths( self ):
-        local_input_paths = self._wrapper_input_paths
-
-        results = []
-        for local_input_path in local_input_paths:
-            wrapper_path = str( local_input_path )
-            # This will over-copy in some cases. For instance in the case of task
-            # splitting, this input will be copied even though only the work dir
-            # input will actually be used.
-            remote_path = self.path_mapper.remote_input_path_rewrite( wrapper_path )
-            results.append( self._dataset_path( local_input_path, remote_path ) )
-        return results
-
-    def _dataset_path( self, local_dataset_path, remote_path ):
-        remote_extra_files_path = None
-        if remote_path:
-            remote_extra_files_path = "%s_files" % remote_path[ 0:-len( ".dat" ) ]
-        return local_dataset_path.with_path_for_job( remote_path, remote_extra_files_path )
-
-    def working_directory( self ):
-        return self._working_directory
-
-    def config_directory( self ):
-        return self._config_directory
-
-    def new_file_path( self ):
-        return self.working_directory()  # Problems with doing this?
-
-    def sep( self ):
-        return self._sep
-
-    def version_path( self ):
-        return self._version_path
-
-    def rewriter( self, parameter_value ):
-        unstructured_path_rewrites = self.unstructured_path_rewrites
-        if parameter_value in unstructured_path_rewrites:
-            # Path previously mapped, use previous mapping.
-            return unstructured_path_rewrites[ parameter_value ]
-        if parameter_value in unstructured_path_rewrites.itervalues():
-            # Path is a rewritten remote path (this might never occur,
-            # consider dropping check...)
-            return parameter_value
-
-        rewrite, new_unstructured_path_rewrites = self.path_mapper.check_for_arbitrary_rewrite( parameter_value )
-        if rewrite:
-            unstructured_path_rewrites.update(new_unstructured_path_rewrites)
-            return rewrite
-        else:
-            # Did need to rewrite, use original path or value.
-            return parameter_value
-
-    def unstructured_path_rewriter( self ):
-        return self.rewriter
@@ -1,62 +0,0 @@
-"""
-lwr_client
-==========
-
-This module contains logic for interfacing with an external LWR server.
-
------------------
-Configuring Galaxy
------------------
-
-Galaxy job runners are configured in Galaxy's ``job_conf.xml`` file. See ``job_conf.xml.sample_advanced``
-in your Galaxy code base or on
-`Bitbucket <https://bitbucket.org/galaxy/galaxy-dist/src/tip/job_conf.xml.sample_advanced?at=default>`_
-for information on how to configure Galaxy to interact with the LWR.
-
-Galaxy also supports an older, less rich configuration of job runners directly
-in its main ``galaxy.ini`` file. The following section describes how to
-configure Galaxy to communicate with the LWR in this legacy mode.
-
-Legacy
------
-
-A Galaxy tool can be configured to be executed remotely via LWR by
-adding a line to the ``galaxy.ini`` file under the ``galaxy:tool_runners``
-section with the format::
-
-    <tool_id> = lwr://http://<lwr_host>:<lwr_port>
-
-As an example, if a host named remotehost is running the LWR server
-application on port ``8913``, then the tool with id ``test_tool`` can
-be configured to run remotely on remotehost by adding the following
-line to ``galaxy.ini``::
-
-    test_tool = lwr://http://remotehost:8913
-
-Remember this must be added after the ``[galaxy:tool_runners]`` header
-in the ``universe.ini`` file.
-
-
-"""
-
-from .staging.down import finish_job
-from .staging.up import submit_job
-from .staging import ClientJobDescription
-from .staging import LwrOutputs
-from .staging import ClientOutputs
-from .client import OutputNotFoundException
-from .manager import build_client_manager
-from .destination import url_to_destination_params
-from .path_mapper import PathMapper
-
-__all__ = [
-    build_client_manager,
-    OutputNotFoundException,
-    url_to_destination_params,
-    finish_job,
-    submit_job,
-    ClientJobDescription,
-    LwrOutputs,
-    ClientOutputs,
-    PathMapper,
-]
@@ -1,567 +0,0 @@
-from json import load
-from os import makedirs
-from os.path import exists
-from os.path import abspath
-from os.path import dirname
-from os.path import join
-from os.path import basename
-from os.path import sep
-import fnmatch
-from re import compile
-from re import escape
-import galaxy.util
-from galaxy.util.bunch import Bunch
-from .config_util import read_file
-from .util import directory_files
-from .util import unique_path_prefix
-from .transport import get_file
-from .transport import post_file
-
-
-DEFAULT_MAPPED_ACTION = 'transfer'  # Not really clear to me what this should be, exception?
-DEFAULT_PATH_MAPPER_TYPE = 'prefix'
-
-STAGING_ACTION_REMOTE = "remote"
-STAGING_ACTION_LOCAL = "local"
-STAGING_ACTION_NONE = None
-STAGING_ACTION_DEFAULT = "default"
-
-# Poor man's enum.
-path_type = Bunch(
-    # Galaxy input datasets and extra files.
-    INPUT="input",
-    # Galaxy config and param files.
-    CONFIG="config",
-    # Files from tool's tool_dir (for now just wrapper if available).
-    TOOL="tool",
-    # Input work dir files - e.g. metadata files, task-split input files, etc..
-    WORKDIR="workdir",
-    # Galaxy output datasets in their final home.
-    OUTPUT="output",
-    # Galaxy from_work_dir output paths and other files (e.g. galaxy.json)
-    OUTPUT_WORKDIR="output_workdir",
-    # Other fixed tool parameter paths (likely coming from tool data, but not
-    # nessecarily). Not sure this is the best name...
-    UNSTRUCTURED="unstructured",
-)
-
-
-ACTION_DEFAULT_PATH_TYPES = [
-    path_type.INPUT,
-    path_type.CONFIG,
-    path_type.TOOL,
-    path_type.WORKDIR,
-    path_type.OUTPUT,
-    path_type.OUTPUT_WORKDIR,
-]
-ALL_PATH_TYPES = ACTION_DEFAULT_PATH_TYPES + [path_type.UNSTRUCTURED]
-
-
-class FileActionMapper(object):
-    """
-    Objects of this class define how paths are mapped to actions.
-
-    >>> json_string = r'''{"paths": [ \
-      {"path": "/opt/galaxy", "action": "none"}, \
-      {"path": "/galaxy/data", "action": "transfer"}, \
-      {"path": "/cool/bamfiles/**/*.bam", "action": "copy", "match_type": "glob"}, \
-      {"path": ".*/dataset_\\\\d+.dat", "action": "copy", "match_type": "regex"} \
-    ]}'''
-    >>> from tempfile import NamedTemporaryFile
-    >>> from os import unlink
-    >>> def mapper_for(default_action, config_contents):
-    ...     f = NamedTemporaryFile(delete=False)
-    ...     f.write(config_contents.encode('UTF-8'))
-    ...     f.close()
-    ...     mock_client = Bunch(default_file_action=default_action, action_config_path=f.name, files_endpoint=None)
-    ...     mapper = FileActionMapper(mock_client)
-    ...     mapper = FileActionMapper(config=mapper.to_dict()) # Serialize and deserialize it to make sure still works
-    ...     unlink(f.name)
-    ...     return mapper
-    >>> mapper = mapper_for(default_action='none', config_contents=json_string)
-    >>> # Test first config line above, implicit path prefix mapper
-    >>> action = mapper.action('/opt/galaxy/tools/filters/catWrapper.py', 'input')
-    >>> action.action_type == u'none'
-    True
-    >>> action.staging_needed
-    False
-    >>> # Test another (2nd) mapper, this one with a different action
-    >>> action = mapper.action('/galaxy/data/files/000/dataset_1.dat', 'input')
-    >>> action.action_type == u'transfer'
-    True
-    >>> action.staging_needed
-    True
-    >>> # Always at least copy work_dir outputs.
-    >>> action = mapper.action('/opt/galaxy/database/working_directory/45.sh', 'workdir')
-    >>> action.action_type == u'copy'
-    True
-    >>> action.staging_needed
-    True
-    >>> # Test glob mapper (matching test)
-    >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam', 'input').action_type == u'copy'
-    True
-    >>> # Test glob mapper (non-matching test)
-    >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam.bai', 'input').action_type == u'none'
-    True
-    >>> # Regex mapper test.
-    >>> mapper.action('/old/galaxy/data/dataset_10245.dat', 'input').action_type == u'copy'
-    True
-    >>> # Doesn't map unstructured paths by default
-    >>> mapper.action('/old/galaxy/data/dataset_10245.dat', 'unstructured').action_type == u'none'
-    True
-    >>> input_only_mapper = mapper_for(default_action="none", config_contents=r'''{"paths": [ \
-      {"path": "/", "action": "transfer", "path_types": "input"} \
-    ] }''')
-    >>> input_only_mapper.action('/dataset_1.dat', 'input').action_type == u'transfer'
-    True
-    >>> input_only_mapper.action('/dataset_1.dat', 'output').action_type == u'none'
-    True
-    >>> unstructured_mapper = mapper_for(default_action="none", config_contents=r'''{"paths": [ \
-      {"path": "/", "action": "transfer", "path_types": "*any*"} \
-    ] }''')
-    >>> unstructured_mapper.action('/old/galaxy/data/dataset_10245.dat', 'unstructured').action_type == u'transfer'
-    True
-    """
-
-    def __init__(self, client=None, config=None):
-        if config is None and client is None:
-            message = "FileActionMapper must be constructed from either a client or a config dictionary."
-            raise Exception(message)
-        if config is None:
-            config = self.__client_to_config(client)
-        self.default_action = config.get("default_action", "transfer")
-        self.mappers = mappers_from_dicts(config.get("paths", []))
-        self.files_endpoint = config.get("files_endpoint", None)
-
-    def action(self, path, type, mapper=None):
-        mapper = self.__find_mapper(path, type, mapper)
-        action_class = self.__action_class(path, type, mapper)
-        file_lister = DEFAULT_FILE_LISTER
-        action_kwds = {}
-        if mapper:
-            file_lister = mapper.file_lister
-            action_kwds = mapper.action_kwds
-        action = action_class(path, file_lister=file_lister, **action_kwds)
-        self.__process_action(action, type)
-        return action
-
-    def unstructured_mappers(self):
-        """ Return mappers that will map 'unstructured' files (i.e. go beyond
-        mapping inputs, outputs, and config files).
-        """
-        return filter(lambda m: path_type.UNSTRUCTURED in m.path_types, self.mappers)
-
-    def to_dict(self):
-        return dict(
-            default_action=self.default_action,
-            files_endpoint=self.files_endpoint,
-            paths=map(lambda m: m.to_dict(), self.mappers)
-        )
-
-    def __client_to_config(self, client):
-        action_config_path = client.action_config_path
-        if action_config_path:
-            config = read_file(action_config_path)
-        else:
-            config = dict()
-        config["default_action"] = client.default_file_action
-        config["files_endpoint"] = client.files_endpoint
-        return config
-
-    def __load_action_config(self, path):
-        config = load(open(path, 'rb'))
-        self.mappers = mappers_from_dicts(config.get('paths', []))
-
-    def __find_mapper(self, path, type, mapper=None):
-        if not mapper:
-            normalized_path = abspath(path)
-            for query_mapper in self.mappers:
-                if query_mapper.matches(normalized_path, type):
-                    mapper = query_mapper
-                    break
-        return mapper
-
-    def __action_class(self, path, type, mapper):
-        action_type = self.default_action if type in ACTION_DEFAULT_PATH_TYPES else "none"
-        if mapper:
-            action_type = mapper.action_type
-        if type in ["workdir", "output_workdir"] and action_type == "none":
-            # We are changing the working_directory relative to what
-            # Galaxy would use, these need to be copied over.
-            action_type = "copy"
-        action_class = actions.get(action_type, None)
-        if action_class is None:
-            message_template = "Unknown action_type encountered %s while trying to map path %s"
-            message_args = (action_type, path)
-            raise Exception(message_template % message_args)
-        return action_class
-
-    def __process_action(self, action, file_type):
-        """ Extension point to populate extra action information after an
-        action has been created.
-        """
-        if action.action_type == "remote_transfer":
-            url_base = self.files_endpoint
-            if not url_base:
-                raise Exception("Attempted to use remote_transfer action with defining a files_endpoint")
-            if "?" not in url_base:
-                url_base = "%s?" % url_base
-            # TODO: URL encode path.
-            url = "%s&path=%s&file_type=%s" % (url_base, action.path, file_type)
-            action.url = url
-
-REQUIRED_ACTION_KWD = object()
-
-
-class BaseAction(object):
-    action_spec = {}
-
-    def __init__(self, path, file_lister=None):
-        self.path = path
-        self.file_lister = file_lister or DEFAULT_FILE_LISTER
-
-    def unstructured_map(self, path_helper):
-        unstructured_map = self.file_lister.unstructured_map(self.path)
-        if self.staging_needed:
-            # To ensure uniqueness, prepend unique prefix to each name
-            prefix = unique_path_prefix(self.path)
-            for path, name in unstructured_map.iteritems():
-                unstructured_map[path] = join(prefix, name)
-        else:
-            path_rewrites = {}
-            for path in unstructured_map:
-                rewrite = self.path_rewrite(path_helper, path)
-                if rewrite:
-                    path_rewrites[path] = rewrite
-            unstructured_map = path_rewrites
-        return unstructured_map
-
-    @property
-    def staging_needed(self):
-        return self.staging != STAGING_ACTION_NONE
-
-    @property
-    def staging_action_local(self):
-        return self.staging == STAGING_ACTION_LOCAL
-
-
-class NoneAction(BaseAction):
-    """ This action indicates the corresponding path does not require any
-    additional action. This should indicate paths that are available both on
-    the LWR client (i.e. Galaxy server) and remote LWR server with the same
-    paths. """
-    action_type = "none"
-    staging = STAGING_ACTION_NONE
-
-    def to_dict(self):
-        return dict(path=self.path, action_type=self.action_type)
-
-    @classmethod
-    def from_dict(cls, action_dict):
-        return NoneAction(path=action_dict["path"])
-
-    def path_rewrite(self, path_helper, path=None):
-        return None
-
-
-class RewriteAction(BaseAction):
-    """ This actin indicates the LWR server should simply rewrite the path
-    to the specified file.
-    """
-    action_spec = dict(
-        source_directory=REQUIRED_ACTION_KWD,
-        destination_directory=REQUIRED_ACTION_KWD
-    )
-    action_type = "rewrite"
-    staging = STAGING_ACTION_NONE
-
-    def __init__(self, path, file_lister=None, source_directory=None, destination_directory=None):
-        self.path = path
-        self.file_lister = file_lister or DEFAULT_FILE_LISTER
-        self.source_directory = source_directory
-        self.destination_directory = destination_directory
-
-    def to_dict(self):
-        return dict(
-            path=self.path,
-            action_type=self.action_type,
-            source_directory=self.source_directory,
-            destination_directory=self.destination_directory,
-        )
-
-    @classmethod
-    def from_dict(cls, action_dict):
-        return RewriteAction(
-            path=action_dict["path"],
-            source_directory=action_dict["source_directory"],
-            destination_directory=action_dict["destination_directory"],
-        )
-
-    def path_rewrite(self, path_helper, path=None):
-        if not path:
-            path = self.path
-        new_path = path_helper.from_posix_with_new_base(self.path, self.source_directory, self.destination_directory)
-        return None if new_path == self.path else new_path
-
-
-class TransferAction(BaseAction):
-    """ This actions indicates that the LWR client should initiate an HTTP
-    transfer of the corresponding path to the remote LWR server before
-    launching the job. """
-    action_type = "transfer"
-    staging = STAGING_ACTION_LOCAL
-
-
-class CopyAction(BaseAction):
-    """ This action indicates that the LWR client should execute a file system
-    copy of the corresponding path to the LWR staging directory prior to
-    launching the corresponding job. """
-    action_type = "copy"
-    staging = STAGING_ACTION_LOCAL
-
-
-class RemoteCopyAction(BaseAction):
-    """ This action indicates the LWR server should copy the file before
-    execution via direct file system copy. This is like a CopyAction, but
-    it indicates the action should occur on the LWR server instead of on
-    the client.
-    """
-    action_type = "remote_copy"
-    staging = STAGING_ACTION_REMOTE
-
-    def to_dict(self):
-        return dict(path=self.path, action_type=self.action_type)
-
-    @classmethod
-    def from_dict(cls, action_dict):
-        return RemoteCopyAction(path=action_dict["path"])
-
-    def write_to_path(self, path):
-        galaxy.util.copy_to_path(open(self.path, "rb"), path)
-
-    def write_from_path(self, lwr_path):
-        destination = self.path
-        parent_directory = dirname(destination)
-        if not exists(parent_directory):
-            makedirs(parent_directory)
-        with open(lwr_path, "rb") as f:
-            galaxy.util.copy_to_path(f, destination)
-
-
-class RemoteTransferAction(BaseAction):
-    """ This action indicates the LWR server should copy the file before
-    execution via direct file system copy. This is like a CopyAction, but
-    it indicates the action should occur on the LWR server instead of on
-    the client.
-    """
-    action_type = "remote_transfer"
-    staging = STAGING_ACTION_REMOTE
-
-    def __init__(self, path, file_lister=None, url=None):
-        super(RemoteTransferAction, self).__init__(path, file_lister=file_lister)
-        self.url = url
-
-    def to_dict(self):
-        return dict(path=self.path, action_type=self.action_type, url=self.url)
-
-    @classmethod
-    def from_dict(cls, action_dict):
-        return RemoteTransferAction(path=action_dict["path"], url=action_dict["url"])
-
-    def write_to_path(self, path):
-        get_file(self.url, path)
-
-    def write_from_path(self, lwr_path):
-        post_file(self.url, lwr_path)
-
-
-class MessageAction(object):
-    """ Sort of pseudo action describing "files" store in memory and
-    transferred via message (HTTP, Python-call, MQ, etc...)
-    """
-    action_type = "message"
-    staging = STAGING_ACTION_DEFAULT
-
-    def __init__(self, contents, client=None):
-        self.contents = contents
-        self.client = client
-
-    @property
-    def staging_needed(self):
-        return True
-
-    @property
-    def staging_action_local(self):
-        # Ekkk, cannot be called if created through from_dict.
-        # Shouldn't be a problem the way it is used - but is an
-        # object design problem.
-        return self.client.prefer_local_staging
-
-    def to_dict(self):
-        return dict(contents=self.contents, action_type=MessageAction.action_type)
-
-    @classmethod
-    def from_dict(cls, action_dict):
-        return MessageAction(contents=action_dict["contents"])
-
-    def write_to_path(self, path):
-        open(path, "w").write(self.contents)
-
-
-DICTIFIABLE_ACTION_CLASSES = [RemoteCopyAction, RemoteTransferAction, MessageAction]
-
-
-def from_dict(action_dict):
-    action_type = action_dict.get("action_type", None)
-    target_class = None
-    for action_class in DICTIFIABLE_ACTION_CLASSES:
-        if action_type == action_class.action_type:
-            target_class = action_class
-    if not target_class:
-        message = "Failed to recover action from dictionary - invalid action type specified %s." % action_type
-        raise Exception(message)
-    return target_class.from_dict(action_dict)
-
-
-class BasePathMapper(object):
-
-    def __init__(self, config):
-        action_type = config.get('action', DEFAULT_MAPPED_ACTION)
-        action_class = actions.get(action_type, None)
-        action_kwds = action_class.action_spec.copy()
-        for key, value in action_kwds.items():
-            if key in config:
-                action_kwds[key] = config[key]
-            elif value is REQUIRED_ACTION_KWD:
-                message_template = "action_type %s requires key word argument %s"
-                message = message_template % (action_type, key)
-                raise Exception(message)
-        self.action_type = action_type
-        self.action_kwds = action_kwds
-        path_types_str = config.get('path_types', "*defaults*")
-        path_types_str = path_types_str.replace("*defaults*", ",".join(ACTION_DEFAULT_PATH_TYPES))
-        path_types_str = path_types_str.replace("*any*", ",".join(ALL_PATH_TYPES))
-        self.path_types = path_types_str.split(",")
-        self.file_lister = FileLister(config)
-
-    def matches(self, path, path_type):
-        path_type_matches = path_type in self.path_types
-        return path_type_matches and self._path_matches(path)
-
-    def _extend_base_dict(self, **kwds):
-        base_dict = dict(
-            action=self.action_type,
-            path_types=",".join(self.path_types),
-            match_type=self.match_type
-        )
-        base_dict.update(self.file_lister.to_dict())
-        base_dict.update(self.action_kwds)
-        base_dict.update(**kwds)
-        return base_dict
-
-
-class PrefixPathMapper(BasePathMapper):
-    match_type = 'prefix'
-
-    def __init__(self, config):
-        super(PrefixPathMapper, self).__init__(config)
-        self.prefix_path = abspath(config['path'])
-
-    def _path_matches(self, path):
-        return path.startswith(self.prefix_path)
-
-    def to_pattern(self):
-        pattern_str = "(%s%s[^\s,\"\']+)" % (escape(self.prefix_path), escape(sep))
-        return compile(pattern_str)
-
-    def to_dict(self):
-        return self._extend_base_dict(path=self.prefix_path)
-
-
-class GlobPathMapper(BasePathMapper):
-    match_type = 'glob'
-
-    def __init__(self, config):
-        super(GlobPathMapper, self).__init__(config)
-        self.glob_path = config['path']
-
-    def _path_matches(self, path):
-        return fnmatch.fnmatch(path, self.glob_path)
-
-    def to_pattern(self):
-        return compile(fnmatch.translate(self.glob_path))
-
-    def to_dict(self):
-        return self._extend_base_dict(path=self.glob_path)
-
-
-class RegexPathMapper(BasePathMapper):
-    match_type = 'regex'
-
-    def __init__(self, config):
-        super(RegexPathMapper, self).__init__(config)
-        self.pattern_raw = config['path']
-        self.pattern = compile(self.pattern_raw)
-
-    def _path_matches(self, path):
-        return self.pattern.match(path) is not None
-
-    def to_pattern(self):
-        return self.pattern
-
-    def to_dict(self):
-        return self._extend_base_dict(path=self.pattern_raw)
-
-MAPPER_CLASSES = [PrefixPathMapper, GlobPathMapper, RegexPathMapper]
-MAPPER_CLASS_DICT = dict(map(lambda c: (c.match_type, c), MAPPER_CLASSES))
-
-
-def mappers_from_dicts(mapper_def_list):
-    return map(lambda m: __mappper_from_dict(m), mapper_def_list)
-
-
-def __mappper_from_dict(mapper_dict):
-    map_type = mapper_dict.get('match_type', DEFAULT_PATH_MAPPER_TYPE)
-    return MAPPER_CLASS_DICT[map_type](mapper_dict)
-
-
-class FileLister(object):
-
-    def __init__(self, config):
-        self.depth = int(config.get("depth", "0"))
-
-    def to_dict(self):
-        return dict(
-            depth=self.depth
-        )
-
-    def unstructured_map(self, path):
-        depth = self.depth
-        if self.depth == 0:
-            return {path: basename(path)}
-        else:
-            while depth > 0:
-                path = dirname(path)
-                depth -= 1
-            return dict([(join(path, f), f) for f in directory_files(path)])
-
-DEFAULT_FILE_LISTER = FileLister(dict(depth=0))
-
-ACTION_CLASSES = [
-    NoneAction,
-    RewriteAction,
-    TransferAction,
-    CopyAction,
-    RemoteCopyAction,
-    RemoteTransferAction,
-]
-actions = dict([(clazz.action_type, clazz) for clazz in ACTION_CLASSES])
-
-
-__all__ = [
-    FileActionMapper,
-    path_type,
-    from_dict,
-    MessageAction,
-    RemoteTransferAction,  # For testing
-]
@@ -1,139 +0,0 @@
-try:
-    import kombu
-    from kombu import pools
-except ImportError:
-    kombu = None
-
-import socket
-import logging
-import threading
-from time import sleep
-log = logging.getLogger(__name__)
-
-
-KOMBU_UNAVAILABLE = "Attempting to bind to AMQP message queue, but kombu dependency unavailable"
-
-DEFAULT_EXCHANGE_NAME = "lwr"
-DEFAULT_EXCHANGE_TYPE = "direct"
-# Set timeout to periodically give up looking and check if polling should end.
-DEFAULT_TIMEOUT = 0.2
-DEFAULT_HEARTBEAT = 580
-
-DEFAULT_RECONNECT_CONSUMER_WAIT = 1
-DEFAULT_HEARTBEAT_WAIT = 1
-
-
-class LwrExchange(object):
-    """ Utility for publishing and consuming structured LWR queues using kombu.
-    This is shared between the server and client - an exchange should be setup
-    for each manager (or in the case of the client, each manager one wished to
-    communicate with.)
-
-    Each LWR manager is defined solely by name in the scheme, so only one LWR
-    should target each AMQP endpoint or care should be taken that unique
-    manager names are used across LWR servers targetting same AMQP endpoint -
-    and in particular only one such LWR should define an default manager with
-    name _default_.
-    """
-
-    def __init__(
-        self,
-        url,
-        manager_name,
-        connect_ssl=None,
-        timeout=DEFAULT_TIMEOUT,
-        publish_kwds={},
-    ):
-        """
-        """
-        if not kombu:
-            raise Exception(KOMBU_UNAVAILABLE)
-        self.__url = url
-        self.__manager_name = manager_name
-        self.__connect_ssl = connect_ssl
-        self.__exchange = kombu.Exchange(DEFAULT_EXCHANGE_NAME, DEFAULT_EXCHANGE_TYPE)
-        self.__timeout = timeout
-        # Be sure to log message publishing failures.
-        if publish_kwds.get("retry", False):
-            if "retry_policy" not in publish_kwds:
-                publish_kwds["retry_policy"] = {}
-            if "errback" not in publish_kwds["retry_policy"]:
-                publish_kwds["retry_policy"]["errback"] = self.__publish_errback
-        self.__publish_kwds = publish_kwds
-
-    @property
-    def url(self):
-        return self.__url
-
-    def consume(self, queue_name, callback, check=True, connection_kwargs={}):
-        queue = self.__queue(queue_name)
-        log.debug("Consuming queue '%s'", queue)
-        while check:
-            heartbeat_thread = None
-            try:
-                with self.connection(self.__url, heartbeat=DEFAULT_HEARTBEAT, **connection_kwargs) as connection:
-                    with kombu.Consumer(connection, queues=[queue], callbacks=[callback], accept=['json']):
-                        heartbeat_thread = self.__start_heartbeat(queue_name, connection)
-                        while check and connection.connected:
-                            try:
-                                connection.drain_events(timeout=self.__timeout)
-                            except socket.timeout:
-                                pass
-            except (IOError, socket.error) as exc:
-                # In testing, errno is None
-                log.warning('Got %s, will retry: %s', exc.__class__.__name__, exc)
-                if heartbeat_thread:
-                    heartbeat_thread.join()
-                sleep(DEFAULT_RECONNECT_CONSUMER_WAIT)
-
-    def heartbeat(self, connection):
-        log.debug('AMQP heartbeat thread alive')
-        while connection.connected:
-            connection.heartbeat_check()
-            sleep(DEFAULT_HEARTBEAT_WAIT)
-        log.debug('AMQP heartbeat thread exiting')
-
-    def publish(self, name, payload):
-        with self.connection(self.__url) as connection:
-            with pools.producers[connection].acquire() as producer:
-                key = self.__queue_name(name)
-                producer.publish(
-                    payload,
-                    serializer='json',
-                    exchange=self.__exchange,
-                    declare=[self.__exchange],
-                    routing_key=key,
-                    **self.__publish_kwds
-                )
-
-    def __publish_errback(self, exc, interval):
-        log.error("Connection error while publishing: %r", exc, exc_info=1)
-        log.info("Retrying in %s seconds", interval)
-
-    def connection(self, connection_string, **kwargs):
-        if "ssl" not in kwargs:
-            kwargs["ssl"] = self.__connect_ssl
-        return kombu.Connection(connection_string, **kwargs)
-
-    def __queue(self, name):
-        queue_name = self.__queue_name(name)
-        queue = kombu.Queue(queue_name, self.__exchange, routing_key=queue_name)
-        return queue
-
-    def __queue_name(self, name):
-        key_prefix = self.__key_prefix()
-        queue_name = '%s_%s' % (key_prefix, name)
-        return queue_name
-
-    def __key_prefix(self):
-        if self.__manager_name == "_default_":
-            key_prefix = "lwr_"
-        else:
-            key_prefix = "lwr_%s_" % self.__manager_name
-        return key_prefix
-
-    def __start_heartbeat(self, queue_name, connection):
-        thread_name = "consume-heartbeat-%s" % (self.__queue_name(queue_name))
-        thread = threading.Thread(name=thread_name, target=self.heartbeat, args=(connection,))
-        thread.start()
-        return thread
@@ -1,41 +0,0 @@
-from .amqp_exchange import LwrExchange
-from .util import filter_destination_params
-
-
-def get_exchange(url, manager_name, params):
-    connect_ssl = parse_amqp_connect_ssl_params(params)
-    exchange_kwds = dict(
-        manager_name=manager_name,
-        connect_ssl=connect_ssl,
-        publish_kwds=parse_amqp_publish_kwds(params)
-    )
-    timeout = params.get('amqp_consumer_timeout', False)
-    if timeout is not False:
-        exchange_kwds['timeout'] = timeout
-    exchange = LwrExchange(url, **exchange_kwds)
-    return exchange
-
-
-def parse_amqp_connect_ssl_params(params):
-    ssl_params = filter_destination_params(params, "amqp_connect_ssl_")
-    if not ssl_params:
-        return
-
-    ssl = __import__('ssl')
-    if 'cert_reqs' in ssl_params:
-        value = ssl_params['cert_reqs']
-        ssl_params['cert_reqs'] = getattr(ssl, value.upper())
-    return ssl_params
-
-
-def parse_amqp_publish_kwds(params):
-    all_publish_params = filter_destination_params(params, "amqp_publish_")
-    retry_policy_params = {}
-    for key in all_publish_params.keys():
-        if key.startswith("retry_"):
-            value = all_publish_params[key]
-            retry_policy_params[key[len("retry_"):]] = value
-            del all_publish_params[key]
-    if retry_policy_params:
-        all_publish_params["retry_policy"] = retry_policy_params
-    return all_publish_params
@@ -1,428 +0,0 @@
-import os
-from json import dumps
-
-from .destination import submit_params
-from .setup_handler import build as build_setup_handler
-from .job_directory import RemoteJobDirectory
-from .decorators import parseJson
-from .decorators import retry
-from .util import copy
-from .util import ensure_directory
-from .util import to_base64_json
-
-
-import logging
-log = logging.getLogger(__name__)
-
-CACHE_WAIT_SECONDS = 3
-
-
-class OutputNotFoundException(Exception):
-
-    def __init__(self, path):
-        self.path = path
-
-    def __str__(self):
-        return "No remote output found for path %s" % self.path
-
-
-class BaseJobClient(object):
-
-    def __init__(self, destination_params, job_id):
-        self.destination_params = destination_params
-        self.job_id = job_id
-        if "jobs_directory" in (destination_params or {}):
-            staging_directory = destination_params["jobs_directory"]
-            sep = destination_params.get("remote_sep", os.sep)
-            job_directory = RemoteJobDirectory(
-                remote_staging_directory=staging_directory,
-                remote_id=job_id,
-                remote_sep=sep,
-            )
-        else:
-            job_directory = None
-        self.env = destination_params.get("env", [])
-        self.files_endpoint = destination_params.get("files_endpoint", None)
-        self.job_directory = job_directory
-
-        self.default_file_action = self.destination_params.get("default_file_action", "transfer")
-        self.action_config_path = self.destination_params.get("file_action_config", None)
-
-        self.setup_handler = build_setup_handler(self, destination_params)
-
-    def setup(self, tool_id=None, tool_version=None):
-        """
-        Setup remote LWR server to run this job.
-        """
-        setup_args = {"job_id": self.job_id}
-        if tool_id:
-            setup_args["tool_id"] = tool_id
-        if tool_version:
-            setup_args["tool_version"] = tool_version
-        return self.setup_handler.setup(**setup_args)
-
-    @property
-    def prefer_local_staging(self):
-        # If doing a job directory is defined, calculate paths here and stage
-        # remotely.
-        return self.job_directory is None
-
-
-class JobClient(BaseJobClient):
-    """
-    Objects of this client class perform low-level communication with a remote LWR server.
-
-    **Parameters**
-
-    destination_params : dict or str
-        connection parameters, either url with dict containing url (and optionally `private_token`).
-    job_id : str
-        Galaxy job/task id.
-    """
-
-    def __init__(self, destination_params, job_id, job_manager_interface):
-        super(JobClient, self).__init__(destination_params, job_id)
-        self.job_manager_interface = job_manager_interface
-
-    def launch(self, command_line, dependencies_description=None, env=[], remote_staging=[], job_config=None):
-        """
-        Queue up the execution of the supplied `command_line` on the remote
-        server. Called launch for historical reasons, should be renamed to
-        enqueue or something like that.
-
-        **Parameters**
-
-        command_line : str
-            Command to execute.
-        """
-        launch_params = dict(command_line=command_line, job_id=self.job_id)
-        submit_params_dict = submit_params(self.destination_params)
-        if submit_params_dict:
-            launch_params['params'] = dumps(submit_params_dict)
-        if dependencies_description:
-            launch_params['dependencies_description'] = dumps(dependencies_description.to_dict())
-        if env:
-            launch_params['env'] = dumps(env)
-        if remote_staging:
-            launch_params['remote_staging'] = dumps(remote_staging)
-        if job_config and self.setup_handler.local:
-            # Setup not yet called, job properties were inferred from
-            # destination arguments. Hence, must have LWR setup job
-            # before queueing.
-            setup_params = _setup_params_from_job_config(job_config)
-            launch_params["setup_params"] = dumps(setup_params)
-        return self._raw_execute("launch", launch_params)
-
-    def full_status(self):
-        """ Return a dictionary summarizing final state of job.
-        """
-        return self.raw_check_complete()
-
-    def kill(self):
-        """
-        Cancel remote job, either removing from the queue or killing it.
-        """
-        return self._raw_execute("kill", {"job_id": self.job_id})
-
-    @retry()
-    @parseJson()
-    def raw_check_complete(self):
-        """
-        Get check_complete response from the remote server.
-        """
-        check_complete_response = self._raw_execute("check_complete", {"job_id": self.job_id})
-        return check_complete_response
-
-    def get_status(self):
-        check_complete_response = self.raw_check_complete()
-        # Older LWR instances won't set status so use 'complete', at some
-        # point drop backward compatibility.
-        status = check_complete_response.get("status", None)
-        if status in ["status", None]:
-            # LEGACY: Bug in certains older LWR instances returned literal
-            # "status".
-            complete = check_complete_response["complete"] == "true"
-            old_status = "complete" if complete else "running"
-            status = old_status
-        return status
-
-    def clean(self):
-        """
-        Cleanup the remote job.
-        """
-        self._raw_execute("clean", {"job_id": self.job_id})
-
-    @parseJson()
-    def remote_setup(self, **setup_args):
-        """
-        Setup remote LWR server to run this job.
-        """
-        return self._raw_execute("setup", setup_args)
-
-    def put_file(self, path, input_type, name=None, contents=None, action_type='transfer'):
-        if not name:
-            name = os.path.basename(path)
-        args = {"job_id": self.job_id, "name": name, "input_type": input_type}
-        input_path = path
-        if contents:
-            input_path = None
-        if action_type == 'transfer':
-            return self._upload_file(args, contents, input_path)
-        elif action_type == 'copy':
-            lwr_path = self._raw_execute('input_path', args)
-            copy(path, lwr_path)
-            return {'path': lwr_path}
-
-    def fetch_output(self, path, name, working_directory, action_type, output_type):
-        """
-        Fetch (transfer, copy, etc...) an output from the remote LWR server.
-
-        **Parameters**
-
-        path : str
-            Local path of the dataset.
-        name : str
-            Remote name of file (i.e. path relative to remote staging output
-            or working directory).
-        working_directory : str
-            Local working_directory for the job.
-        action_type : str
-            Where to find file on LWR (output_workdir or output). legacy is also
-            an option in this case LWR is asked for location - this will only be
-            used if targetting an older LWR server that didn't return statuses
-            allowing this to be inferred.
-        """
-        if output_type == 'legacy':
-            self._fetch_output_legacy(path, working_directory, action_type=action_type)
-        elif output_type == 'output_workdir':
-            self._fetch_work_dir_output(name, working_directory, path, action_type=action_type)
-        elif output_type == 'output':
-            self._fetch_output(path=path, name=name, action_type=action_type)
-        else:
-            raise Exception("Unknown output_type %s" % output_type)
-
-    def _raw_execute(self, command, args={}, data=None, input_path=None, output_path=None):
-        return self.job_manager_interface.execute(command, args, data, input_path, output_path)
-
-    # Deprecated
-    def _fetch_output_legacy(self, path, working_directory, action_type='transfer'):
-        # Needs to determine if output is task/working directory or standard.
-        name = os.path.basename(path)
-
-        output_type = self._get_output_type(name)
-        if output_type == "none":
-            # Just make sure the file was created.
-            if not os.path.exists(path):
-                raise OutputNotFoundException(path)
-            return
-        elif output_type in ["task"]:
-            path = os.path.join(working_directory, name)
-
-        self.__populate_output_path(name, path, output_type, action_type)
-
-    def _fetch_output(self, path, name=None, check_exists_remotely=False, action_type='transfer'):
-        if not name:
-            # Extra files will send in the path.
-            name = os.path.basename(path)
-
-        output_type = "direct"  # Task/from_work_dir outputs now handled with fetch_work_dir_output
-        self.__populate_output_path(name, path, output_type, action_type)
-
-    def _fetch_work_dir_output(self, name, working_directory, output_path, action_type='transfer'):
-        ensure_directory(output_path)
-        if action_type == 'transfer':
-            self.__raw_download_output(name, self.job_id, "work_dir", output_path)
-        else:  # Even if action is none - LWR has a different work_dir so this needs to be copied.
-            lwr_path = self._output_path(name, self.job_id, 'work_dir')['path']
-            copy(lwr_path, output_path)
-
-    def __populate_output_path(self, name, output_path, output_type, action_type):
-        ensure_directory(output_path)
-        if action_type == 'transfer':
-            self.__raw_download_output(name, self.job_id, output_type, output_path)
-        elif action_type == 'copy':
-            lwr_path = self._output_path(name, self.job_id, output_type)['path']
-            copy(lwr_path, output_path)
-
-    @parseJson()
-    def _upload_file(self, args, contents, input_path):
-        return self._raw_execute(self._upload_file_action(args), args, contents, input_path)
-
-    def _upload_file_action(self, args):
-        # Hack for backward compatibility, instead of using new upload_file
-        # path. Use old paths.
-        input_type = args['input_type']
-        action = {
-            # For backward compatibility just target upload_input_extra for all
-            # inputs, it allows nested inputs. Want to do away with distinction
-            # inputs and extra inputs.
-            'input': 'upload_extra_input',
-            'config': 'upload_config_file',
-            'workdir': 'upload_working_directory_file',
-            'tool': 'upload_tool_file',
-            'unstructured': 'upload_unstructured_file',
-        }[input_type]
-        del args['input_type']
-        return action
-
-    @parseJson()
-    def _get_output_type(self, name):
-        return self._raw_execute("get_output_type", {"name": name,
-                                                     "job_id": self.job_id})
-
-    @parseJson()
-    def _output_path(self, name, job_id, output_type):
-        return self._raw_execute("output_path",
-                                 {"name": name,
-                                  "job_id": self.job_id,
-                                  "output_type": output_type})
-
-    @retry()
-    def __raw_download_output(self, name, job_id, output_type, output_path):
-        output_params = {
-            "name": name,
-            "job_id": self.job_id,
-            "output_type": output_type
-        }
-        self._raw_execute("download_output", output_params, output_path=output_path)
-
-
-class BaseMessageJobClient(BaseJobClient):
-
-    def __init__(self, destination_params, job_id, client_manager):
-        super(BaseMessageJobClient, self).__init__(destination_params, job_id)
-        if not self.job_directory:
-            error_message = "Message-queue based LWR client requires destination define a remote job_directory to stage files into."
-            raise Exception(error_message)
-        self.client_manager = client_manager
-
-    def clean(self):
-        del self.client_manager.status_cache[self.job_id]
-
-    def full_status(self):
-        full_status = self.client_manager.status_cache.get(self.job_id, None)
-        if full_status is None:
-            raise Exception("full_status() called before a final status was properly cached with cilent manager.")
-        return full_status
-
-    def _build_setup_message(self, command_line, dependencies_description, env, remote_staging, job_config):
-        """
-        """
-        launch_params = dict(command_line=command_line, job_id=self.job_id)
-        submit_params_dict = submit_params(self.destination_params)
-        if submit_params_dict:
-            launch_params['submit_params'] = submit_params_dict
-        if dependencies_description:
-            launch_params['dependencies_description'] = dependencies_description.to_dict()
-        if env:
-            launch_params['env'] = env
-        if remote_staging:
-            launch_params['remote_staging'] = remote_staging
-        if job_config and self.setup_handler.local:
-            # Setup not yet called, job properties were inferred from
-            # destination arguments. Hence, must have LWR setup job
-            # before queueing.
-            setup_params = _setup_params_from_job_config(job_config)
-            launch_params["setup_params"] = setup_params
-        return launch_params
-
-
-class MessageJobClient(BaseMessageJobClient):
-
-    def launch(self, command_line, dependencies_description=None, env=[], remote_staging=[], job_config=None):
-        """
-        """
-        launch_params = self._build_setup_message(
-            command_line,
-            dependencies_description=dependencies_description,
-            env=env,
-            remote_staging=remote_staging,
-            job_config=job_config
-        )
-        response = self.client_manager.exchange.publish("setup", launch_params)
-        log.info("Job published to setup message queue.")
-        return response
-
-    def kill(self):
-        self.client_manager.exchange.publish("kill", dict(job_id=self.job_id))
-
-
-class MessageCLIJobClient(BaseMessageJobClient):
-
-    def __init__(self, destination_params, job_id, client_manager, shell):
-        super(MessageCLIJobClient, self).__init__(destination_params, job_id, client_manager)
-        self.remote_lwr_path = destination_params["remote_lwr_path"]
-        self.shell = shell
-
-    def launch(self, command_line, dependencies_description=None, env=[], remote_staging=[], job_config=None):
-        """
-        """
-        launch_params = self._build_setup_message(
-            command_line,
-            dependencies_description=dependencies_description,
-            env=env,
-            remote_staging=remote_staging,
-            job_config=job_config
-        )
-        base64_message = to_base64_json(launch_params)
-        submit_command = os.path.join(self.remote_lwr_path, "scripts", "submit.bash")
-        # TODO: Allow configuration of manager, app, and ini path...
-        self.shell.execute("nohup %s --base64 %s &" % (submit_command, base64_message))
-
-    def kill(self):
-        # TODO
-        pass
-
-
-class InputCachingJobClient(JobClient):
-    """
-    Beta client that cache's staged files to prevent duplication.
-    """
-
-    def __init__(self, destination_params, job_id, job_manager_interface, client_cacher):
-        super(InputCachingJobClient, self).__init__(destination_params, job_id, job_manager_interface)
-        self.client_cacher = client_cacher
-
-    @parseJson()
-    def _upload_file(self, args, contents, input_path):
-        action = self._upload_file_action(args)
-        if contents:
-            input_path = None
-            return self._raw_execute(action, args, contents, input_path)
-        else:
-            event_holder = self.client_cacher.acquire_event(input_path)
-            cache_required = self.cache_required(input_path)
-            if cache_required:
-                self.client_cacher.queue_transfer(self, input_path)
-            while not event_holder.failed:
-                available = self.file_available(input_path)
-                if available['ready']:
-                    token = available['token']
-                    args["cache_token"] = token
-                    return self._raw_execute(action, args)
-                event_holder.event.wait(30)
-            if event_holder.failed:
-                raise Exception("Failed to transfer file %s" % input_path)
-
-    @parseJson()
-    def cache_required(self, path):
-        return self._raw_execute("cache_required", {"path": path})
-
-    @parseJson()
-    def cache_insert(self, path):
-        return self._raw_execute("cache_insert", {"path": path}, None, path)
-
-    @parseJson()
-    def file_available(self, path):
-        return self._raw_execute("file_available", {"path": path})
-
-
-def _setup_params_from_job_config(job_config):
-    job_id = job_config.get("job_id", None)
-    tool_id = job_config.get("tool_id", None)
-    tool_version = job_config.get("tool_version", None)
-    return dict(
-        job_id=job_id,
-        tool_id=tool_id,
-        tool_version=tool_version
-    )
@@ -1,72 +0,0 @@
-""" Generic interface for reading YAML/INI/JSON config files into nested dictionaries.
-"""
-
-import yaml
-
-try:
-    from ConfigParser import ConfigParser
-except ImportError:
-    from configparser import ConfigParser
-import json
-
-
-CONFIG_TYPE_JSON = "json"
-CONFIG_TYPE_YAML = "yaml"
-CONFIG_TYPE_INI = "ini"
-
-DEFAULT_CONFIG_TYPE = CONFIG_TYPE_YAML
-
-JSON_EXTS = [".json"]
-YAML_EXTS = [".yaml", ".yml"]
-INI_EXTS = [".ini"]
-
-EXT_MAP = {
-    CONFIG_TYPE_JSON: JSON_EXTS,
-    CONFIG_TYPE_YAML: YAML_EXTS,
-    CONFIG_TYPE_INI: INI_EXTS,
-}
-
-
-def read_file(path, type=None, default_type=DEFAULT_CONFIG_TYPE):
-    if path is None:
-        raise ValueError("Undefined path supplied.")
-
-    config_type = __find_type(path, type, default_type)
-    return EXT_READERS[config_type](path)
-
-
-def __find_type(path, explicit_type, default_type):
-    if explicit_type:
-        return explicit_type
-
-    for config_type, config_exts in EXT_MAP.items():
-        for ext in config_exts:
-            if path.endswith(ext):
-                return config_type
-
-    return default_type
-
-
-def __read_yaml(path):
-    if yaml is None:
-        raise ImportError("Attempting to read YAML configuration file - but PyYAML dependency unavailable.")
-
-    with open(path, "rb") as f:
-        return yaml.load(f)
-
-
-def __read_ini(path):
-    config = ConfigParser()
-    config.read(path)
-    return config._sections
-
-
-def __read_json(path):
-    with open(path, "rb") as f:
-        return json.load(f)
-
-EXT_READERS = {
-    CONFIG_TYPE_JSON: __read_json,
-    CONFIG_TYPE_YAML: __read_yaml,
-    CONFIG_TYPE_INI: __read_ini,
-}
--- a/Show More
+++ b/Show More