Merge branch 'dev' into admin_grid_formbuilder

2026-09-24 16:30:27 +08:00 · 2017-09-13 12:41:09 -04:00
parent 87f2ad191f 11d4380d2b
commit 809cabdfd4
205 changed files with 4392 additions and 2945 deletions
@@ -101,7 +101,7 @@ lib/galaxy/webapps/galaxy/api/genomes.py
 lib/galaxy/webapps/galaxy/api/histories.py
 lib/galaxy/webapps/galaxy/api/__init__.py
 lib/galaxy/webapps/galaxy/api/jobs.py
-lib/galaxy/webapps/galaxy/api/lda_datasets.py
+lib/galaxy/webapps/galaxy/api/library_datasets.py
 lib/galaxy/webapps/galaxy/api/requests.py
 lib/galaxy/webapps/galaxy/api/roles.py
 lib/galaxy/webapps/galaxy/api/samples.py
@@ -12,7 +12,7 @@ services:
    ports:
      - "${GALAXY_PORT}:8080"
  selenium:
-    image: selenium/standalone-chrome:3.0.1-aluminum
+    image: selenium/standalone-chrome:3.5.2
    ports:
      - "${SELENIUM_PORT}:4444"
    links:
@@ -7,6 +7,7 @@ var jQuery = require( 'jquery' ),
    Ui = require( 'mvc/ui/ui-misc' ),
    QueryStringParsing = require( 'utils/query-string-parsing' ),
    Router = require( 'layout/router' ),
+    Utils = require( 'utils/utils' ),
    Page = require( 'layout/page' );

 window.app = function app( options, bootstrapped ){
@@ -127,9 +128,10 @@ window.app = function app( options, bootstrapped ){

    $(function() {
        _.extend( options.config, { active_view : 'admin' } );
+        Utils.setWindowTitle("Administration");
        Galaxy.page = new Page.View( _.extend( options, {
            Left    : AdminPanel,
            Router  : AdminRouter
        } ) );
    });
-};
+};
@@ -1,4 +1,4 @@
-define( [ 'layout/masthead', 'layout/panel', 'mvc/ui/ui-modal' ], function( Masthead, Panel, Modal ) {
+define( [ 'layout/masthead', 'layout/panel', 'mvc/ui/ui-modal', 'utils/utils' ], function( Masthead, Panel, Modal, Utils) {
    var View = Backbone.View.extend({
        el : 'body',
        className : 'full-content',
@@ -16,7 +16,16 @@ define( [ 'layout/masthead', 'layout/panel', 'mvc/ui/ui-modal' ], function( Mast

            // attach global objects, build mastheads
            Galaxy.modal = this.modal = new Modal.View();
-            Galaxy.display = this.display = function( view ) { self.center.display( view ) };
+            Galaxy.display = this.display = function( view ) {
+                if ( view.title ){
+                    Utils.setWindowTitle( view.title );
+                    view.allow_title_display = false;
+                } else {
+                    Utils.setWindowTitle();
+                    view.allow_title_display = true;
+                }
+                self.center.display( view );
+            };
            Galaxy.router = this.router = options.Router && new options.Router( self, options );
            this.masthead = new Masthead.View( this.config );
            this.center = new Panel.CenterPanel();
@@ -155,7 +164,7 @@ define( [ 'layout/masthead', 'layout/panel', 'mvc/ui/ui-modal' ], function( Mast
                            }
                        }
                })
-                .error( function( data ) { 
+                .error( function( data ) {
                    // hide the communication icon if the communication server is not available
                    $chat_icon_element.css( "visibility", "hidden" ); 
                });
@@ -166,4 +175,4 @@ define( [ 'layout/masthead', 'layout/panel', 'mvc/ui/ui-modal' ], function( Mast
    });

    return { View: View }
-});
+});
@@ -405,7 +405,7 @@ var PairedCollectionCreator = Backbone.View.extend( baseMVC.LoggableMixin ).exte

    /** autopair by exact match */
    autopairSimple : autoPairFnBuilder({
-        scoreThreshold: function(){ return 1.0; },
+        scoreThreshold: function(){ return 0.6; },
        match : function _match( params ){
            params = params || {};
            if( params.matchTo === params.possible ){
@@ -6,6 +6,7 @@ define( [ 'mvc/grid/grid-view' ], function( GridView ) {
            this.setElement( $( '<div/>' ) );
            this.model = new Backbone.Model( options );
            this.item = this.model.get( 'item' );
+            this.title = this.model.get('plural');
            $.ajax({
                url     : Galaxy.root + this.item + '/' + this.model.get( 'action_id' ),
                success : function( response ) {
@@ -55,4 +56,4 @@ define( [ 'mvc/grid/grid-view' ], function( GridView ) {
    return {
        View: View
    }
-});
+});
@@ -20,6 +20,7 @@ return Backbone.View.extend({
    initialize: function(grid_config) {
        this.grid = new GridModel();
        this.dict_format = grid_config.dict_format;
+        this.title = grid_config.title;
        var self = this;
        window.add_tag_to_grid_filter = function( tag_name, tag_value ){
            // Put tag name and value together.
@@ -83,6 +84,9 @@ return Backbone.View.extend({
        // get options
        var options = this.grid.attributes;

+        if (this.allow_title_display && options.title){
+            Utils.setWindowTitle(options.title);
+        }
        // handle refresh requests
        this.handle_refresh(options.refresh_frames);

@@ -2,6 +2,7 @@
 define( [ 'utils/utils', 'mvc/grid/grid-view', 'mvc/history/history-model', 'mvc/history/copy-dialog' ], function( Utils, GridView, HistoryModel, historyCopyDialog ) {

    var View = Backbone.View.extend({
+        title: "Histories",
        initialize: function( options ) {
            var self = this;
            this.setElement( $( '<div/>' ) );
@@ -46,4 +47,4 @@ define( [ 'utils/utils', 'mvc/grid/grid-view', 'mvc/history/history-model', 'mvc
    return {
        View: View
    }
-});
+});
@@ -156,6 +156,7 @@ var HistoryViewEdit = _super.extend(
    renderItems : function( $whereTo ){
        var views = _super.prototype.renderItems.call( this, $whereTo );
        if( !this.searchFor ){ this._renderCounts( $whereTo ); }
+        else{ this._renderSearchFindings( $whereTo ); }
        return views;
    },

@@ -427,10 +428,10 @@ var HistoryViewEdit = _super.extend(
    },

    /** override to display number found in subtitle */
-    _renderSearchFindings : function(){
-        this.$( '> .controls .subtitle' ).html([
-            _l( 'Found' ), this.views.length
-        ].join(' '));
+    _renderSearchFindings : function( $whereTo ){
+        $whereTo = $whereTo instanceof jQuery? $whereTo : this.$el;
+        var html = this.templates.found( this.model.toJSON(), this );
+        $whereTo.find( '> .controls .subtitle' ).html( html );
        return this;
    },

@@ -598,8 +599,37 @@ HistoryViewEdit.prototype.templates = (function(){
        '<% } %>',
    ], 'history' );

+    var foundTemplate = BASE_MVC.wrapTemplate([
+        _l( 'Found' ), ' <%- view.views.length %>, ',
+
+        '<% if( history.contents_active.deleted ){ %>',
+            '<% if( view.model.contents.includeDeleted ){ %>',
+                '<a class="toggle-deleted-link" href="javascript:void(0);">',
+                    _l( 'hide deleted' ),
+                '</a>, ',
+            '<% } else { %>',
+                '<a class="toggle-deleted-link" href="javascript:void(0);">',
+                    _l( 'show deleted' ),
+                '</a>, ',
+            '<% } %>',
+        '<% } %>',
+
+        '<% if( history.contents_active.hidden ){ %>',
+            '<% if( view.model.contents.includeHidden ){ %>',
+                '<a class="toggle-hidden-link" href="javascript:void(0);">',
+                    _l( 'hide hidden' ),
+                '</a>',
+            '<% } else { %>',
+                '<a class="toggle-hidden-link" href="javascript:void(0);">',
+                    _l( 'show hidden' ),
+                '</a>',
+            '<% } %>',
+        '<% } %>',
+    ], 'history' );
+
    return _.extend( _.clone( _super.prototype.templates ), {
-        counts : countsTemplate
+        counts : countsTemplate,
+        found : foundTemplate 
    });
 }());

@@ -65,7 +65,7 @@ var menu = [
        anon    : true,
        func    : function() {
            if( Galaxy && Galaxy.currHistoryPanel && confirm( _l( 'Really delete the current history?' ) ) ){
-                galaxy_main.window.location.href = 'history/delete?id=' + Galaxy.currHistoryPanel.model.id;
+                Galaxy.currHistoryPanel.model._delete().done(function(){Galaxy.currHistoryPanel.loadCurrentHistory();});
            }
        },
    },
@@ -76,7 +76,7 @@ var menu = [
        func    : function() {
            if( Galaxy && Galaxy.currHistoryPanel
            &&  confirm( _l( 'Really delete the current history permanently? This cannot be undone.' ) ) ){
-                galaxy_main.window.location.href = 'history/delete?purge=True&id=' + Galaxy.currHistoryPanel.model.id;
+                Galaxy.currHistoryPanel.model.purge().done(function(){Galaxy.currHistoryPanel.loadCurrentHistory();});
            }
        },
    },
@@ -16,8 +16,10 @@ var LibraryDatasetView = Backbone.View.extend({

  model: null,

-  options: {
+  options: {},

+  defaults: {
+    edit_mode: false
  },

  events: {
@@ -25,13 +27,11 @@ var LibraryDatasetView = Backbone.View.extend({
    "click .toolbtn_cancel_modifications" :   "render",
    "click .toolbtn-download-dataset"     :   "downloadDataset",
    "click .toolbtn-import-dataset"       :   "importIntoHistory",
-    "click .toolbtn-share-dataset"        :   "shareDataset",
    "click .btn-copy-link-to-clipboard"   :   "copyToClipboard",
    "click .btn-make-private"             :   "makeDatasetPrivate",
    "click .btn-remove-restrictions"      :   "removeDatasetRestrictions",
    "click .toolbtn_save_permissions"     :   "savePermissions",
-    "click .toolbtn_save_modifications"   :   "comingSoon",
-
+    "click .toolbtn_save_modifications"   :   "saveModifications"
  },

  // genome select
@@ -134,7 +134,10 @@ var LibraryDatasetView = Backbone.View.extend({
    $(".tooltip").remove();
    var template = this.templateModifyDataset();
    this.$el.html(template({item: this.model}));
-    this.renderSelectBoxes({genome_build: this.model.get('genome_build'), file_ext: this.model.get('file_ext') });
+    this.renderSelectBoxes({
+      genome_build: this.model.get('genome_build'),
+      file_ext: this.model.get('file_ext')
+    });
    $(".peek").html(this.model.get("peek"));
    $("#center [data-toggle]").tooltip();
  },
@@ -245,10 +248,6 @@ var LibraryDatasetView = Backbone.View.extend({
      });
  },

-  shareDataset: function(){
-    mod_toastr.info('Feature coming soon.');
-  },
-
  goBack: function(){
    Galaxy.libraries.library_router.back();
  },
@@ -457,8 +456,57 @@ var LibraryDatasetView = Backbone.View.extend({
    }
  },

-  comingSoon: function(){
-    mod_toastr.warning('Feature coming soon.');
+  /**
+   * Save the changes made to the library dataset.
+   */
+  saveModifications: function(options){
+    var is_changed = false;
+    var ld = this.model;
+    var new_name = this.$el.find('.input_dataset_name').val();
+    if (typeof new_name !== 'undefined' && new_name !== ld.get('name') ){
+      if (new_name.length > 0){
+        ld.set("name", new_name);
+        is_changed = true;
+      } else{
+        mod_toastr.warning('Library dataset name has to be at least 1 character long.');
+        return;
+      }
+    }
+    var new_info = this.$el.find('.input_dataset_misc_info').val();
+    if (typeof new_info !== 'undefined' && new_info !== ld.get('misc_info') ){
+        ld.set("misc_info", new_info);
+        is_changed = true;
+    }
+    var new_genome_build = this.select_genome.$el.select2('data').id;
+    if (typeof new_genome_build !== 'undefined' && new_genome_build !== ld.get('genome_build') ){
+        ld.set("genome_build", new_genome_build);
+        is_changed = true;
+    }
+    var new_ext = this.select_extension.$el.select2('data').id;
+    if (typeof new_ext !== 'undefined' && new_ext !== ld.get('file_ext') ){
+        ld.set("file_ext", new_ext);
+        is_changed = true;
+    }
+    var dataset_view = this;
+    if (is_changed){
+      ld.save(null, {
+        patch: true,
+        success: function(ld) {
+          dataset_view.render()
+          mod_toastr.success('Changes to library dataset saved.');
+        },
+        error: function(model, response){
+          if (typeof response.responseJSON !== "undefined"){
+            mod_toastr.error(response.responseJSON.err_msg);
+          } else {
+            mod_toastr.error('An error occured while attempting to update the library dataset.');
+          }
+        }
+      });
+    } else {
+      dataset_view.render()
+      mod_toastr.info('Nothing has changed.');
+    }
  },

  copyToClipboard: function(){
@@ -523,42 +571,46 @@ var LibraryDatasetView = Backbone.View.extend({
  },

  /**
-   * Request all extensions and genomes from Galaxy
-   * and save them sorted in arrays.
+   * If needed request all extensions and/or genomes from Galaxy
+   * and save them in sorted arrays.
   */
  fetchExtAndGenomes: function(){
    var that = this;
-    mod_utils.get({
-        url      :  Galaxy.root + "api/datatypes?extension_only=False",
-        success  :  function( datatypes ) {
-                        for (var key in datatypes) {
-                            that.list_extensions.push({
-                                id              : datatypes[key].extension,
-                                text            : datatypes[key].extension,
-                                description     : datatypes[key].description,
-                                description_url : datatypes[key].description_url
-                            });
-                        }
-                        that.list_extensions.sort(function(a, b) {
-                            return a.id > b.id ? 1 : a.id < b.id ? -1 : 0;
-                        });
-                        that.list_extensions.unshift(that.auto);
-                    }
+    if (this.list_genomes.length == 0){
+      mod_utils.get({
+        url      : Galaxy.root + "api/datatypes?extension_only=False",
+        success  : function( datatypes ) {
+          for (var key in datatypes) {
+              that.list_extensions.push({
+                  id              : datatypes[key].extension,
+                  text            : datatypes[key].extension,
+                  description     : datatypes[key].description,
+                  description_url : datatypes[key].description_url
+              });
+          }
+          that.list_extensions.sort(function(a, b) {
+              return a.id > b.id ? 1 : a.id < b.id ? -1 : 0;
+          });
+          that.list_extensions.unshift(that.auto);
+        }
      });
-    mod_utils.get({
-        url     :    Galaxy.root + "api/genomes",
+    }
+    if (this.list_extensions.length == 0){
+      mod_utils.get({
+        url     : Galaxy.root + "api/genomes",
        success : function( genomes ) {
-                    for (var key in genomes ) {
-                        that.list_genomes.push({
-                            id      : genomes[key][1],
-                            text    : genomes[key][0]
-                        });
-                    }
-                    that.list_genomes.sort(function(a, b) {
-                        return a.id > b.id ? 1 : a.id < b.id ? -1 : 0;
-                    });
-                }
-    });
+          for (var key in genomes ) {
+              that.list_genomes.push({
+                  id      : genomes[key][1],
+                  text    : genomes[key][0]
+              });
+          }
+          that.list_genomes.sort(function(a, b) {
+              return a.id > b.id ? 1 : a.id < b.id ? -1 : 0;
+          });
+        }
+      });
+    }
  },

  renderSelectBoxes: function(options){
@@ -566,6 +618,7 @@ var LibraryDatasetView = Backbone.View.extend({
    // See this.fetchExtAndGenomes()
    // TODO switch to common resources:
    // https://trello.com/c/dIUE9YPl/1933-ui-common-resources-and-data-into-galaxy-object
+    var that = this;
    var current_genome = '?';
    var current_ext = 'auto';
    if (typeof options !== 'undefined'){
@@ -576,17 +629,16 @@ var LibraryDatasetView = Backbone.View.extend({
        current_ext = options.file_ext;
      }
    }
-    var that = this;
    this.select_genome = new mod_select.View( {
        css: 'dataset-genome-select',
        data: that.list_genomes,
-        container: that.$el.find( '#dataset_genome_select' ),
+        container: that.$el.find('#dataset_genome_select'),
        value: current_genome
    } );
    this.select_extension = new mod_select.View({
      css: 'dataset-extension-select',
      data: that.list_extensions,
-      container: that.$el.find( '#dataset_extension_select' ),
+      container: that.$el.find('#dataset_extension_select'),
      value: current_ext
    });
  },
@@ -711,13 +763,13 @@ var LibraryDatasetView = Backbone.View.extend({
        '<% } %>',
        '<% if (item.get("misc_blurb")) { %>',
          '<tr>',
-            '<th scope="row">Miscellaneous blurb</th>',
+            '<th scope="row">Misc. blurb</th>',
            '<td scope="row"><%= _.escape(item.get("misc_blurb")) %></td>',
          '</tr>',
        '<% } %>',
        '<% if (item.get("misc_info")) { %>',
          '<tr>',
-            '<th scope="row">Miscellaneous information</th>',
+            '<th scope="row">Misc. info</th>',
            '<td scope="row"><%= _.escape(item.get("misc_info")) %></td>',
          '</tr>',
        '<% } %>',
@@ -902,7 +954,6 @@ var LibraryDatasetView = Backbone.View.extend({
      '</ol>',

      '<div class="dataset_table">',
-        '<p>For full editing options please import the dataset to history and use "Edit attributes" on it.</p>',
        '<table class="grid table table-striped table-condensed">',
          '<tr>',
            '<th class="dataset-first-column" scope="row" id="id_row" data-id="<%= _.escape(item.get("ldda_id")) %>">Name</th>',
@@ -956,12 +1007,12 @@ var LibraryDatasetView = Backbone.View.extend({
            '<td scope="row"><%= _.escape(item.get("message")) %></td>',
          '</tr>',
          '<tr>',
-            '<th scope="row">Miscellaneous information</th>',
-            '<td scope="row"><%= _.escape(item.get("misc_info")) %></td>',
+            '<th scope="row">Misc. blurb</th>',
+            '<td scope="row"><%= _.escape(item.get("misc_blurb")) %></td>',
          '</tr>',
          '<tr>',
-            '<th scope="row">Miscellaneous blurb</th>',
-            '<td scope="row"><%= _.escape(item.get("misc_blurb")) %></td>',
+            '<th scope="row">Misc. information</th>',
+            '<td><input class="input_dataset_misc_info form-control" type="text" placeholder="info" value="<%= _.escape(item.get("misc_info")) %>"></td>',
          '</tr>',
          //TODO: add functionality to modify tags here
          '<% if (item.get("tags")) { %>',
@@ -37,8 +37,6 @@ var FolderView = Backbone.View.extend({
      success: function() {
        if (that.options.show_permissions){
            that.showPermissions();
-        } else {
-            that.render();
        }
      },
      error: function(model, response){
@@ -51,19 +49,6 @@ var FolderView = Backbone.View.extend({
    });
  },

-  render: function(options){
-    $(".tooltip").remove();
-    this.options = _.extend(this.options, options);
-    var template = this.templateFolder();
-    this.$el.html(template({item: this.model}));
-    $(".peek").html(this.model.get("peek"));
-    $("#center [data-toggle]").tooltip();
-  },
-
-  shareFolder: function(){
-    mod_toastr.info('Feature coming soon.');
-  },
-
  goBack: function(){
    Galaxy.libraries.library_router.back();
  },
@@ -165,18 +150,6 @@ var FolderView = Backbone.View.extend({
    return select_options;
  },

-  comingSoon: function(){
-    mod_toastr.warning('Feature coming soon.');
-  },
-
-  copyToClipboard: function(){
-    var href = Backbone.history.location.href;
-    if (href.lastIndexOf('/permissions') !== -1){
-      href = href.substr(0, href.lastIndexOf('/permissions'));
-    }
-    window.prompt("Copy to clipboard: Ctrl+C, Enter", href);
-  },
-
  /**
   * Extract the role ids from Select2 elements's 'data'
   */
@@ -206,57 +179,6 @@ var FolderView = Backbone.View.extend({
    })
  },

-  templateFolder : function(){
-    return _.template([
-    '<div class="library_style_container">',
-      '<div id="library_toolbar">',
-        '<button data-toggle="tooltip" data-placement="top" title="Modify library item" class="btn btn-default toolbtn_modify_dataset primary-button" type="button">',
-          '<span class="fa fa-pencil"/>',
-          '&nbsp;Modify',
-        '</button>',
-        '<a href="#folders/<%- item.get("folder_id") %>/datasets/<%- item.id %>/permissions">',
-          '<button data-toggle="tooltip" data-placement="top" title="Manage permissions" class="btn btn-default toolbtn_change_permissions primary-button" type="button">',
-            '<span class="fa fa-group"/>',
-            '&nbsp;Permissions',
-          '</button>',
-        '</a>',
-        '<button data-toggle="tooltip" data-placement="top" title="Share dataset" class="btn btn-default toolbtn-share-dataset primary-button" type="button">',
-          '<span class="fa fa-share"/>',
-          '&nbsp;Share',
-          '</span>',
-        '</button>',
-      '</div>',
-      '<p>',
-        'This dataset is unrestricted so everybody can access it. Just share the URL of this page. ',
-        '<button data-toggle="tooltip" data-placement="top" title="Copy to clipboard" class="btn btn-default btn-copy-link-to-clipboard primary-button" type="button">',
-          '<span class="fa fa-clipboard"/>',
-          '&nbsp;To Clipboard',
-        '</button> ',
-      '</p>',
-      '<div class="dataset_table">',
-        '<table class="grid table table-striped table-condensed">',
-          '<tr>',
-            '<th scope="row" id="id_row" data-id="<%= _.escape(item.get("ldda_id")) %>">',
-              'Name',
-            '</th>',
-            '<td>',
-              '<%= _.escape(item.get("name")) %>',
-            '</td>',
-          '</tr>',
-          '<% if (item.get("file_ext")) { %>',
-            '<tr>',
-              '<th scope="row">Data type</th>',
-              '<td>',
-                '<%= _.escape(item.get("file_ext")) %>',
-              '</td>',
-            '</tr>',
-          '<% } %>',
-        '</table>',
-      '</div>',
-    '</div>'
-    ].join(''));
-  },
-
  templateFolderPermissions : function(){
    return _.template([
    '<div class="library_style_container">',
@@ -277,7 +277,7 @@ var FolderListView = Backbone.View.extend({
         // Iterate each checkbox
         $(':checkbox', '#folder_list_body').each(function() {
            this.checked = selected;
-            var $row = $(this.parentElement.parentElement);
+            var $row = $(this).closest('tr');
            // Change color of selected/unselected
            if (selected) {
              that.makeDarkRow($row);
@@ -295,12 +295,11 @@ var FolderListView = Backbone.View.extend({
        var checkbox = '';
        var $row;
        var source;
+        $row = $(event.target).closest('tr');
        if (event.target.localName === 'input'){
            checkbox = event.target;
-            $row = $(event.target.parentElement.parentElement);
            source = 'input';
        } else if (event.target.localName === 'td') {
-            $row = $(event.target.parentElement);
            checkbox = $row.find(':checkbox')[0];
            source = 'td';
        }
@@ -205,46 +205,38 @@ var FolderToolbarView = Backbone.View.extend({
      return folderDetails.name !== '';
  },

-
  // show bulk import modal
  modalBulkImport : function(){
      var checkedValues = $('#folder_table').find(':checked');
      if(checkedValues.length === 0){
          mod_toastr.info('You must select some datasets first.');
      } else {
-          this.refreshUserHistoriesList(function(that){
-            var template = that.templateBulkImportInModal();
-            that.modal = Galaxy.modal;
-            that.modal.show({
-                closing_events  : true,
-                title           : 'Import into History',
-                body            : template({histories : that.histories.models}),
-                buttons         : {
-                    'Import'    : function() {that.importAllIntoHistory();},
-                    'Close'     : function() {Galaxy.modal.hide();}
-                }
-            });
+        var that = this;
+        this.histories = new mod_library_model.GalaxyHistories();
+        this.histories.fetch()
+        .done(function(){
+          var template = that.templateBulkImportInModal();
+          that.modal = Galaxy.modal;
+          that.modal.show({
+              closing_events  : true,
+              title           : 'Import into History',
+              body            : template({histories : that.histories.models}),
+              buttons         : {
+                  'Import'    : function() {that.importAllIntoHistory();},
+                  'Close'     : function() {Galaxy.modal.hide();}
+              }
          });
+        })
+        .fail(function(model, response){
+          if (typeof response.responseJSON !== "undefined"){
+            mod_toastr.error(response.responseJSON.err_msg);
+          } else {
+            mod_toastr.error('An error ocurred.');
+          }
+        });
      }
  },

-  refreshUserHistoriesList: function(callback){
-    var that = this;
-    this.histories = new mod_library_model.GalaxyHistories();
-    this.histories.fetch({
-      success: function (){
-        callback(that);
-      },
-      error: function(model, response){
-        if (typeof response.responseJSON !== "undefined"){
-          mod_toastr.error(response.responseJSON.err_msg);
-        } else {
-          mod_toastr.error('An error ocurred.');
-        }
-      }
-    });
-  },
-
  /**
   * Import all selected datasets into history.
   */
@@ -253,7 +245,7 @@ var FolderToolbarView = Backbone.View.extend({
    var new_history_name = this.modal.$('input[name=history_name]').val();
    var that = this;
    if (new_history_name !== ''){
-      $.post( Galaxy.root + 'api/histories', {name: new_history_name})
+      $.post(Galaxy.root + 'api/histories', {name: new_history_name})
        .done(function( new_history ) {
          that.options.last_used_history_id = new_history.id;
          that.processImportToHistory(new_history.id, new_history.name);
@@ -277,10 +269,11 @@ var FolderToolbarView = Backbone.View.extend({
    var dataset_ids = [];
    var folder_ids = [];
    $('#folder_table').find(':checked').each(function(){
-        if ($(this.parentElement.parentElement).data('id') !== '' && this.parentElement.parentElement.classList.contains('dataset_row') ) {
-            dataset_ids.push($(this.parentElement.parentElement).data('id'));
-        } else if ($(this.parentElement.parentElement).data('id') !== '' && this.parentElement.parentElement.classList.contains('folder_row') ) {
-            folder_ids.push($(this.parentElement.parentElement).data('id'));
+        var row_id = $(this).closest('tr').data('id');
+        if (row_id.substring(0,1) == 'F'){
+            folder_ids.push(row_id);
+        } else {
+            dataset_ids.push(row_id);
        }
    });
    // prepare the dataset objects to be imported
@@ -331,10 +324,11 @@ var FolderToolbarView = Backbone.View.extend({
    var dataset_ids = [];
    var folder_ids = [];
        $( '#folder_table' ).find( ':checked' ).each( function(){
-            if ( $(this.parentElement.parentElement).data('id') !== '' && this.parentElement.parentElement.classList.contains('dataset_row') ) {
-                dataset_ids.push( $(this.parentElement.parentElement).data('id') );
-            } else if ( $(this.parentElement.parentElement).data('id') !== '' && this.parentElement.parentElement.classList.contains('folder_row') ) {
-                folder_ids.push( $(this.parentElement.parentElement).data('id') );
+            var row_id = $(this).closest('tr').data('id');
+            if (row_id.substring(0,1) == 'F'){
+                folder_ids.push(row_id);
+            } else {
+                dataset_ids.push(row_id);
            }
        } );
    var url = Galaxy.root + 'api/libraries/datasets/download/' + format;
@@ -369,33 +363,36 @@ var FolderToolbarView = Backbone.View.extend({
  },

  addFilesFromHistoryModal: function(){
-    this.refreshUserHistoriesList( function( self ){
-      self.modal = Galaxy.modal;
-      var template_modal = self.templateAddFilesFromHistory();
-      var folder_name = self.options.full_path[self.options.full_path.length - 1][1]
-      self.modal.show({
-          closing_events  : true,
-          title           : 'Adding datasets from your history to folder ' + folder_name,
-          body            : template_modal({histories: self.histories.models}),
-          buttons         : {
-              'Add'       : function() {self.addAllDatasetsFromHistory();},
-              'Close'     : function() {Galaxy.modal.hide();}
-          },
-          closing_callback: function(){
-            Galaxy.libraries.library_router.back();
-          }
-      });
-
-      // user should always have a history, even anonymous user
-      if (self.histories.models.length > 0){
+    this.histories = new mod_library_model.GalaxyHistories();
+    var self = this;
+    this.histories.fetch()
+      .done(function(){
+        self.modal = Galaxy.modal;
+        var template_modal = self.templateAddFilesFromHistory();
+        self.modal.show({
+            closing_events  : true,
+            title           : 'Adding datasets from your history',
+            body            : template_modal({histories: self.histories.models}),
+            buttons         : {
+                'Add'       : function() {self.addAllDatasetsFromHistory();},
+                'Close'     : function() {Galaxy.modal.hide();}
+            },
+            closing_callback: function(){
+              Galaxy.libraries.library_router.navigate('folders/' + self.id, {trigger: true});
+            }
+        });
        self.fetchAndDisplayHistoryContents(self.histories.models[0].id);
        $( "#dataset_add_bulk" ).change(function(event) {
          self.fetchAndDisplayHistoryContents(event.target.value);
        });
-      } else {
-        mod_toastr.error( 'An error ocurred.' );
-      }
-    });
+      })
+      .fail(function(model, response){
+        if (typeof response.responseJSON !== "undefined"){
+          mod_toastr.error(response.responseJSON.err_msg);
+        } else {
+          mod_toastr.error('An error ocurred.');
+        }
+      });
  },

  /**
@@ -528,12 +525,10 @@ var FolderToolbarView = Backbone.View.extend({
          that.renderJstree( options );
          $('.jstree-folders-message').hide();
          $('.jstree-preserve-structure').hide();
-          $('.jstree-link-files').hide();
          $('.jstree-files-message').show();
        } else if ( event.target.value ==='jstree-disable-files' ){
          $('.jstree-files-message').hide();
          $('.jstree-folders-message').show();
-          $('.jstree-link-files').show();
          $('.jstree-preserve-structure').show();
          options.disabled_jstree_element = 'files';
          that.renderJstree( options );
@@ -606,6 +601,9 @@ var FolderToolbarView = Backbone.View.extend({
  importFromPathsClicked: function(){
    var preserve_dirs = this.modal.$el.find('.preserve-checkbox').is(':checked');
    var link_data = this.modal.$el.find('.link-checkbox').is(':checked');
+    var space_to_tab = this.modal.$el.find('.spacetab-checkbox').is(':checked');
+    var to_posix_lines = this.modal.$el.find('.posix-checkbox').is(':checked');
+    var tag_using_filenames = this.modal.$el.find('.tag-files').is(':checked');
    var file_type = this.select_extension.value();
    var dbkey = this.select_genome.value();
    var paths = $('textarea#import_paths').val();
@@ -625,8 +623,11 @@ var FolderToolbarView = Backbone.View.extend({
      this.chainCallImportingFolders( { paths: valid_paths,
                                        preserve_dirs: preserve_dirs,
                                        link_data: link_data,
+                                        space_to_tab: space_to_tab,
+                                        to_posix_lines: to_posix_lines,
                                        source: 'admin_path',
                                        file_type: file_type,
+                                        tag_using_filenames: tag_using_filenames,
                                        dbkey: dbkey } );
    }
  },
@@ -678,6 +679,8 @@ var FolderToolbarView = Backbone.View.extend({
    var selected_nodes = _.filter(all_nodes, function(node){ return node.state.disabled == false; })
    var preserve_dirs = this.modal.$el.find( '.preserve-checkbox' ).is( ':checked' );
    var link_data = this.modal.$el.find( '.link-checkbox' ).is( ':checked' );
+    var space_to_tab = this.modal.$el.find('.spacetab-checkbox').is(':checked');
+    var to_posix_lines = this.modal.$el.find('.posix-checkbox').is(':checked');
    var file_type = this.select_extension.value();
    var dbkey = this.select_genome.value();
    var tag_using_filenames = this.modal.$el.find( '.tag-files' ).is( ':checked' );
@@ -698,6 +701,8 @@ var FolderToolbarView = Backbone.View.extend({
        this.chainCallImportingFolders( { paths: paths,
                                          preserve_dirs: preserve_dirs,
                                          link_data: link_data,
+                                          space_to_tab: space_to_tab,
+                                          to_posix_lines: to_posix_lines,
                                          source: full_source,
                                          file_type: file_type,
                                          dbkey: dbkey,
@@ -707,6 +712,9 @@ var FolderToolbarView = Backbone.View.extend({
        this.chainCallImportingUserdirFiles( { paths : paths,
                                               file_type: file_type,
                                               dbkey: dbkey,
+                                               link_data: link_data,
+                                               space_to_tab: space_to_tab,
+                                               to_posix_lines: to_posix_lines,
                                               source: full_source,
                                               tag_using_filenames: tag_using_filenames } );
      }
@@ -721,6 +729,12 @@ var FolderToolbarView = Backbone.View.extend({
        var history_contents_template = self.templateHistoryContents();
        self.histories.get(history_id).set({'contents' : history_contents});
        self.modal.$el.find('#selected_history_content').html(history_contents_template({history_contents: history_contents.models.reverse()}));
+        self.modal.$el.find('.history-import-select-all').bind("click", function(){
+          $('#selected_history_content [type=checkbox]').prop('checked', true);
+        });
+        self.modal.$el.find('.history-import-unselect-all').bind("click", function(){
+          $('#selected_history_content [type=checkbox]').prop('checked', false);
+        });
      },
      error: function(model, response){
        if (typeof response.responseJSON !== "undefined"){
@@ -745,9 +759,9 @@ var FolderToolbarView = Backbone.View.extend({
    } else {
      this.modal.disableButton( 'Add' );
      checked_hdas.each(function(){
-        var hid = $( this.parentElement ).data( 'id' );
+        var hid = $(this).closest('li').data( 'id' );
        if ( hid ) {
-          var item_type = $( this.parentElement ).data( 'name' );
+          var item_type = $(this).closest('li').data( 'name' );
          history_item_ids.push( hid );
          history_item_types.push( item_type );
        }
@@ -808,7 +822,6 @@ var FolderToolbarView = Backbone.View.extend({
   * @param  {boolean} tag_using_filenames    add tags to datasets using names of files
   */
  chainCallImportingUserdirFiles: function( options ){
-
    var that = this;
    var popped_item = options.paths.pop();
    if ( typeof popped_item === "undefined" ) {
@@ -824,6 +837,9 @@ var FolderToolbarView = Backbone.View.extend({
                                                       '&source=' + options.source +
                                                       '&path=' + popped_item +
                                                       '&file_type=' + options.file_type +
+                                                       '&link_data=' + options.link_data +
+                                                       '&space_to_tab=' + options.space_to_tab +
+                                                       '&to_posix_lines=' + options.to_posix_lines +
                                                       '&dbkey=' + options.dbkey +
                                                       '&tag_using_filenames=' + options.tag_using_filenames ) )
    promise.done( function( response ){
@@ -838,11 +854,13 @@ var FolderToolbarView = Backbone.View.extend({
  },

  /**
-   * Take the array of paths and createa request for each of them
-   * calling them in chain. Update the progress bar in between each.
+   * Take the array of paths and create a request for each of them
+   * calling them in series. Update the progress bar in between each.
   * @param  {array} paths                    paths relative to Galaxy root folder
   * @param  {boolean} preserve_dirs          indicates whether to preserve folder structure
   * @param  {boolean} link_data              copy files to Galaxy or link instead
+   * @param  {boolean} to_posix_lines         convert line endings to POSIX standard
+   * @param  {boolean} space_to_tab           convert spaces to tabs
   * @param  {str} source                     string representing what type of folder
   *                                          is the source of import
   * @param  {boolean} tag_using_filenames    add tags to datasets using names of files
@@ -866,6 +884,8 @@ var FolderToolbarView = Backbone.View.extend({
                                                          '&path=' + popped_item +
                                                          '&preserve_dirs=' + options.preserve_dirs +
                                                          '&link_data=' + options.link_data +
+                                                          '&to_posix_lines=' + options.to_posix_lines +
+                                                          '&space_to_tab=' + options.space_to_tab +
                                                          '&file_type=' + options.file_type +
                                                          '&dbkey=' + options.dbkey +
                                                          '&tag_using_filenames=' + options.tag_using_filenames ) )
@@ -997,11 +1017,12 @@ var FolderToolbarView = Backbone.View.extend({
      var dataset_ids = [];
      var folder_ids = [];
      checkedValues.each(function(){
-          if ($(this.parentElement.parentElement).data('id') !== undefined) {
-              if ($(this.parentElement.parentElement).data('id').substring(0,1) == 'F'){
-                folder_ids.push($(this.parentElement.parentElement).data('id'));
+          var row_id = $(this).closest('tr').data('id');
+          if (row_id !== undefined) {
+              if (row_id.substring(0,1) == 'F'){
+                folder_ids.push(row_id);
              } else {
-                dataset_ids.push($(this.parentElement.parentElement).data('id'));
+                dataset_ids.push(row_id);
              }
          }
      });
@@ -1162,7 +1183,7 @@ var FolderToolbarView = Backbone.View.extend({
              '<span class="fa fa-info-circle"></span>',
              '&nbsp;Details',
            '</button>',
-            '<span class="help-button" data-toggle="tooltip" data-placement="top" title="Visit Libraries Wiki">',
+            '<span class="help-button" data-toggle="tooltip" data-placement="top" title="See this screen annotated">',
              '<a href="https://galaxyproject.org/data-libraries/screen/folder-contents/" target="_blank">',
                '<button class="primary-button" type="button">',
                  '<span class="fa fa-question-circle"></span>',
@@ -1333,16 +1354,24 @@ var FolderToolbarView = Backbone.View.extend({
          '<input class="preserve-checkbox" type="checkbox" value="preserve_directory_structure">',
          'Preserve directory structure',
        '</label>',
-        '<label class="checkbox-inline jstree-link-files" style="display:none;">',
+        '<label class="checkbox-inline">',
          '<input class="link-checkbox" type="checkbox" value="link_files">',
          'Link files instead of copying',
        '</label>',
+        '<label class="checkbox-inline">',
+          '<input class="posix-checkbox" type="checkbox" value="to_posix_lines" checked="checked">',
+          'Convert line endings to POSIX',
+        '</label>',
+        '<label class="checkbox-inline">',
+          '<input class="spacetab-checkbox" type="checkbox" value="space_to_tab">',
+          'Convert spaces to tabs',
+        '</label>',
      '</div>',
      '<button title="Select all files" type="button" class="button primary-button libimport-select-all">',
        'Select all',
      '</button>',
      '<button title="Select no files" type="button" class="button primary-button libimport-select-none">',
-        'Select none',
+        'Unselect all',
      '</button>',
      '<hr />',
      // append jstree object here
@@ -1354,6 +1383,47 @@ var FolderToolbarView = Backbone.View.extend({
        'Type: <span id="library_extension_select" class="library-extension-select" />',
        'Genome: <span id="library_genome_select" class="library-genome-select" />',
      '</div>',
+      '<br>',
+      '<div>',
+         '<label class="checkbox-inline tag-files">',
+            'Tag datasets based on file names.',
+            '<input class="tag-files" type="checkbox" value="tag_using_filenames" checked="checked">',
+         '</label>',
+      '</div>',
+    '</div>'
+    ].join(''));
+  },
+
+  templateImportPathModal: function(){
+    return _.template([
+    '<div id="file_browser_modal">',
+      '<div class="alert alert-info jstree-folders-message">All files within the given folders and their subfolders will be imported into the current folder.</div>',
+      '<div style="margin-bottom: 0.5em;">',
+        '<label class="checkbox-inline">',
+          '<input class="preserve-checkbox" type="checkbox" value="preserve_directory_structure">',
+          'Preserve directory structure',
+        '</label>',
+        '<label class="checkbox-inline">',
+          '<input class="link-checkbox" type="checkbox" value="link_files">',
+          'Link files instead of copying',
+        '</label>',
+        '<br>',
+        '<label class="checkbox-inline">',
+          '<input class="posix-checkbox" type="checkbox" value="to_posix_lines" checked="checked">',
+          'Convert line endings to POSIX',
+        '</label>',
+        '<label class="checkbox-inline">',
+          '<input class="spacetab-checkbox" type="checkbox" value="space_to_tab">',
+          'Convert spaces to tabs',
+        '</label>',
+      '</div>',
+      '<textarea id="import_paths" class="form-control" rows="5" placeholder="Absolute paths (or paths relative to Galaxy root) separated by newline" autofocus></textarea>',
+      '<hr />',
+      '<p>You can set extension type and genome for all imported datasets at once:</p>',
+      '<div>',
+        'Type: <span id="library_extension_select" class="library-extension-select" />',
+        'Genome: <span id="library_genome_select" class="library-genome-select" />',
+      '</div>',
      '<div>',
         '<label class="checkbox-inline tag-files">',
            'Tag datasets based on file names.',
@@ -1364,36 +1434,11 @@ var FolderToolbarView = Backbone.View.extend({
    ].join(''));
  },

-  templateImportPathModal: function(){
-    return _.template([
-    '<div id="file_browser_modal">',
-      '<div class="alert alert-info jstree-folders-message">All files within the given folders and their subfolders will be imported into the current folder.</div>',
-      '<div style="margin-bottom: 0.5em;">',
-        '<label class="checkbox-inline jstree-preserve-structure">',
-          '<input class="preserve-checkbox" type="checkbox" value="preserve_directory_structure">',
-          'Preserve directory structure',
-        '</label>',
-        '<label class="checkbox-inline jstree-link-files">',
-          '<input class="link-checkbox" type="checkbox" value="link_files">',
-          'Link files instead of copying',
-        '</label>',
-      '</div>',
-      '<textarea id="import_paths" class="form-control" rows="5" placeholder="Absolute paths (or paths relative to Galaxy root) separated by newline" autofocus></textarea>',
-      '<hr />',
-      '<p>You can set extension type and genome for all imported datasets at once:</p>',
-      '<div>',
-        'Type: <span id="library_extension_select" class="library-extension-select" />',
-        'Genome: <span id="library_genome_select" class="library-genome-select" />',
-      '</div>',
-    '</div>'
-    ].join(''));
-  },
-
  templateAddFilesFromHistory: function (){
    return _.template([
    '<div id="add_files_modal">',
      '<div>',
-        'Select history:  ',
+        '1.&nbsp;Select history:&nbsp;',
        '<select id="dataset_add_bulk" name="dataset_add_bulk" style="width:66%; "> ',
          '<% _.each(histories, function(history) { %>', //history select box
            '<option value="<%= _.escape(history.get("id")) %>"><%= _.escape(history.get("name")) %></option>',
@@ -1409,31 +1454,49 @@ var FolderToolbarView = Backbone.View.extend({

  templateHistoryContents: function (){
    return _.template([
-    '<strong>Choose the datasets to import:</strong>',
+    '<p>2.&nbsp;Choose the datasets to import:</p>',
+    '<div>',
+    '<button title="Select all datasets" type="button" class="button primary-button history-import-select-all">',
+      'Select all',
+    '</button>',
+    '<button title="Select all datasets" type="button" class="button primary-button history-import-unselect-all">',
+      'Unselect all',
+    '</button>',
+    '</div>',
+    '<br>',
    '<ul>',
      '<% _.each(history_contents, function(history_item) { %>',
        '<% if (history_item.get("deleted") != true ) { %>',
+          '<% var item_name = history_item.get("name") %>',
          '<% if (history_item.get("type") === "collection") { %>',
              '<% var collection_type = history_item.get("collection_type") %>',
              '<% if (collection_type === "list") { %>',
                '<li data-id="<%= _.escape(history_item.get("id")) %>" data-name="<%= _.escape(history_item.get("type")) %>">',
-                  '<input style="margin: 0;" type="checkbox"> <%= _.escape(history_item.get("hid")) %>: <%= _.escape(history_item.get("name")) %> (Dataset Collection)',
+                  '<label>',
+                '<label title="<%= _.escape(item_name) %>">',
+                    '<input style="margin: 0;" type="checkbox"> <%= _.escape(history_item.get("hid")) %>: ',
+                    '<%= item_name.length > 75 ? _.escape("...".concat(item_name.substr(-75))) : _.escape(item_name) %> (Dataset Collection)',
+                  '</label>',
                '</li>',
               '<% } else { %>',
                 '<li><input style="margin: 0;" type="checkbox" onclick="return false;" disabled="disabled">',
                    '<span title="You can convert this collection into a collection of type list using the Collection Tools">',
-                      ' <%= _.escape(history_item.get("hid")) %>: <%= _.escape(history_item.get("name")) %> (Dataset Collection of type <%= _.escape(collection_type) %> not supported.)',
+                      '<%= _.escape(history_item.get("hid")) %>: ',
+                      '<%= item_name.length > 75 ? _.escape("...".concat(item_name.substr(-75))) : _.escape(item_name) %> (Dataset Collection of type <%= _.escape(collection_type) %> not supported.)',
                    '</span>',
                  '</li>',
                '<% } %>',
          '<% } else if (history_item.get("visible") === true && history_item.get("state") === "ok") { %>',
              '<li data-id="<%= _.escape(history_item.get("id")) %>" data-name="<%= _.escape(history_item.get("type")) %>">',
-                '<input style="margin: 0;" type="checkbox"> <%= _.escape(history_item.get("hid")) %>: <%= _.escape(history_item.get("name")) %>',
+                '<label title="<%= _.escape(item_name) %>">',
+                  '<input style="margin: 0;" type="checkbox"> <%= _.escape(history_item.get("hid")) %>: ',
+                  '<%= item_name.length > 75 ? _.escape("...".concat(item_name.substr(-75))) : _.escape(item_name) %>',
+                '</label>',
              '</li>',
          '<% } %>',
        '<% } %>',
      '<% }); %>',
-    '</ul>'
+    '</ul>',
    ].join(''));
  },

@@ -37,8 +37,6 @@ var LibraryView = Backbone.View.extend({
      success: function() {
        if (that.options.show_permissions){
            that.showPermissions();
-        } else {
-            that.render();
        }
      },
      error: function(model, response){
@@ -51,18 +49,6 @@ var LibraryView = Backbone.View.extend({
    });
  },

-  render: function(options){
-    $(".tooltip").remove();
-    this.options = _.extend(this.options, options);
-    var template = this.templateLibrary();
-    this.$el.html(template({item: this.model}));
-    $("#center [data-toggle]").tooltip();
-  },
-
-  shareDataset: function(){
-    mod_toastr.info('Feature coming soon.');
-  },
-
  goBack: function(){
    Galaxy.libraries.library_router.back();
  },
@@ -175,18 +161,6 @@ var LibraryView = Backbone.View.extend({
    return select_options;
  },

-  comingSoon: function(){
-    mod_toastr.warning('Feature coming soon.');
-  },
-
-  copyToClipboard: function(){
-    var href = Backbone.history.location.href;
-    if (href.lastIndexOf('/permissions') !== -1){
-      href = href.substr(0, href.lastIndexOf('/permissions'));
-    }
-    window.prompt("Copy to clipboard: Ctrl+C, Enter", href);
-  },
-
  makeDatasetPrivate: function(){
    var self = this;
    $.post( Galaxy.root + "api/libraries/datasets/" + self.id + "/permissions?action=make_private").done(function(fetched_permissions) {
@@ -237,56 +211,6 @@ var LibraryView = Backbone.View.extend({
    })
  },

-  templateLibrary : function(){
-    return _.template([
-    '<div class="library_style_container">',
-      '<div id="library_toolbar">',
-        '<button data-toggle="tooltip" data-placement="top" title="Modify library item" class="btn btn-default toolbtn_modify_dataset primary-button" type="button">',
-          '<span class="fa fa-pencil"/>',
-          '&nbsp;Modify',
-        '</button>',
-        '<a href="#folders/<%- item.get("folder_id") %>/datasets/<%- item.id %>/permissions">',
-          '<button data-toggle="tooltip" data-placement="top" title="Manage permissions" class="btn btn-default toolbtn_change_permissions primary-button" type="button">',
-            '<span class="fa fa-group"/>',
-            '&nbsp;Permissions',
-          '</button>',
-        '</a>',
-        '<button data-toggle="tooltip" data-placement="top" title="Share dataset" class="btn btn-default toolbtn-share-dataset primary-button" type="button">',
-          '<span class="fa fa-share"/>',
-          '&nbsp;Share',
-        '</button>',
-      '</div>',
-      '<p>',
-        'This dataset is unrestricted so everybody can access it. Just share the URL of this page. ',
-        '<button data-toggle="tooltip" data-placement="top" title="Copy to clipboard" class="btn btn-default btn-copy-link-to-clipboard primary-button" type="button">',
-          '<span class="fa fa-clipboard"/>',
-          '&nbsp;To Clipboard',
-        '</button> ',
-      '</p>',
-      '<div class="dataset_table">',
-        '<table class="grid table table-striped table-condensed">',
-          '<tr>',
-            '<th scope="row" id="id_row" data-id="<%= _.escape(item.get("ldda_id")) %>">',
-              'Name',
-            '</th>',
-            '<td>',
-              '<%= _.escape(item.get("name")) %>',
-            '</td>',
-          '</tr>',
-          '<% if (item.get("file_ext")) { %>',
-            '<tr>',
-              '<th scope="row">Data type</th>',
-              '<td>',
-                '<%= _.escape(item.get("file_ext")) %>',
-              '</td>',
-            '</tr>',
-          '<% } %>',
-        '</table>',
-      '</div>',
-    '</div>',
-    ].join(''));
-  },
-
  templateLibraryPermissions : function(){
    return _.template([
    '<div class="library_style_container">',
@@ -210,7 +210,7 @@ var LibraryToolbarView = Backbone.View.extend({
                    '<button id="create_new_library_btn" class="primary-button btn-xs" type="button"><span class="fa fa-plus"></span> New Library</button>',
                '</span>',
              '<% } %>',
-              '<span class="help-button" data-toggle="tooltip" data-placement="top" title="Visit Libraries Wiki">',
+              '<span class="help-button" data-toggle="tooltip" data-placement="top" title="See this screen annotated">',
                '<a href="https://galaxyproject.org/data-libraries/screen/list-of-libraries/" target="_blank">',
                  '<button class="primary-button" type="button"><span class="fa fa-question-circle"></span> Help</button>',
                '</a>',
@@ -11,29 +11,40 @@ var TagsEditor = Backbone.View
        .extend( baseMVC.LoggableMixin )
        .extend( baseMVC.HiddenUntilActivatedViewMixin ).extend({

-    tagName     : 'div',
-    className   : 'tags-display',
+    tagName      : 'div',
+    className    : 'tags-display',
+    select_width : '100%',
+    events: {},

    /** Set up listeners, parse options */
    initialize : function( options ){
        //console.debug( this, options );
        // only listen to the model only for changes to tags - re-render
+        this.show_editor = false;
        if (options.usePrompt === false) {
            this.label = '';
        } else {
            this.label = '<label class="prompt">' + _l( 'Tags' ) + '</label>';
        }
+        this.workflow_mode = options.workflow_mode || false;
+        if (this.workflow_mode) {
+            this.events.click = 'showEditor';
+            this.events.keydown = 'keydownHandler';
+        }
        this.hiddenUntilActivated( options.$activator, options );
    },

    /** Build the DOM elements, call select to on the created input, and set up behaviors */
    render : function(){
        var self = this;
-        this.$el.html( this._template() );
-
+        if (this.workflow_mode) {
+           this.$el.html(this._workflowTemplate());
+        } else {
+            this.$el.html(this._defaultTemplate());
+        }
        this.$input().select2({
            placeholder : 'Add tags',
-            width       : '100%',
+            width       : this.workflow_mode ? this.width : this.select_width,
            tags : function(){
                // initialize possible tags in the dropdown based on all the tags the user has used so far
                return self._getTagsUsed();
@@ -59,14 +70,61 @@ var TagsEditor = Backbone.View
    },

    /** @returns {String} the html text used to build the view's DOM */
-    _template : function(){
+    _defaultTemplate : function(){
        return [
            this.label,
-            // set up initial tags by adding as CSV to input vals (necc. to init select2)
-            '<input class="tags-input" value="', this.tagsToCSV(), '" />'
+            this._renderEditor()
        ].join( '' );
    },

+    _workflowTemplate : function(){
+        // Shows labels by default, event handler controls whether we show tags or editor
+        return [
+            this.show_editor ? this._renderEditor() : this._renderTags(),
+        ].join( ' ' );
+    },
+
+    keydownHandler : function (e) {
+        switch (e.which) {
+            // esc
+            case 27 :
+                // hide the tag editor when pressing escape
+                this.hideEditor();
+                break;
+        }
+    },
+
+    showEditor: function() {
+        this.show_editor = true;
+        this.render();
+    },
+
+    hideEditor: function() {
+        this.show_editor = false;
+        this.render();
+    },
+
+    _renderEditor: function(){
+        // set up initial tags by adding as CSV to input vals (necc. to init select2)
+        return '<input class="tags-input" value="' + this.tagsToCSV() + '"/>'
+    },
+
+    _renderTags : function(){
+        var tags = this.model.get('tags');
+        var addButton = 'static/images/fugue/tag--plus.png';
+        var renderedArray = [];
+        _.each(tags, function(tag) {
+            tag = tag.indexOf("name:") == 0 ? tag.slice(5) : tag ;
+            var renderString = '<span class="label label-info">' + tag + '</span>';
+            renderedArray.push( renderString );
+        });
+        if (renderedArray.length === 0) {
+            // If there are no tags to render we just show the add-tag-button
+            renderedArray.push('<img src=' + addButton + ' class="add-tag-button" title="Add tags"/>');
+        }
+        return renderedArray.join(" ");
+    },
+
    /** @returns {String} the sorted, comma-separated tags from the model */
    tagsToCSV : function(){
        var self = this;
@@ -132,8 +190,7 @@ var TagsEditor = Backbone.View
    toString : function(){ return [ 'TagsEditor(', this.model + '', ')' ].join(''); }
 });

-// =============================================================================
 return {
-    TagsEditor : TagsEditor
+    TagsEditor : TagsEditor,
 };
 });
@@ -77,6 +77,7 @@ define(['libs/bootstrap-tour'],function(BootstrapTour) {
        });
    };
    var ToursView = Backbone.View.extend({
+        title: "Tours",
        // initialize
        initialize: function() {
            var self = this;
@@ -91,7 +91,7 @@ define( [ 'mvc/form/form-view', 'mvc/ui/ui-misc', 'utils/query-string-parsing' ]

    /** View of the main user preference panel with links to individual user forms */
    var View = Backbone.View.extend({
-
+        title: "User Preferences",
        initialize: function() {
            this.model = new Model();
            this.setElement( '<div/>' );
@@ -0,0 +1,44 @@
+define([
+    "mvc/base-mvc",
+], function( baseMVC ){
+/* global Backbone */
+// workflow model
+
+var logNamespace = 'workflow';
+//==============================================================================
+/** @class model for a single workflow.
+ *  @name WorkflowItem
+ *  @augments Backbone.Model
+ */
+var WorkflowItem = Backbone.Model.extend( baseMVC.LoggableMixin ).extend({
+    _logNamespace : logNamespace,
+
+    urlRoot: '/api/workflows',
+
+    toJSON: function(){
+    // need to overwrite this as endpoint expects the 'workflow' key in payload
+    return {workflow : this.attributes};
+    },
+
+});
+
+//==============================================================================
+/** @class collection for workflows.
+ *  @name WorkflowCollection
+ *  @augments Backbone.Collection
+ */
+var WorkflowCollection = Backbone.Collection.extend({
+    model: WorkflowItem,
+    url: '/api/workflows',
+
+  });
+
+//==============================================================================
+
+return {
+    WorkflowItem: WorkflowItem,
+    WorkflowCollection: WorkflowCollection,
+};
+
+
+});
@@ -476,11 +476,29 @@ define(['mvc/workflow/workflow-globals'], function( Globals ) {
        initialize: function( attr ) {
            Terminal.prototype.initialize.call( this, attr );
            this.datatypes = attr.datatypes;
-            this.collectionType = new CollectionTypeDescription( attr.collection_type );
-            this.isCollection = true;
+            if( attr.collection_type ) {
+                this.collectionType = new CollectionTypeDescription( attr.collection_type );
+            } else {
+                var collectionTypeSource = attr.collection_type_source;
+                if( ! collectionTypeSource ) {
+                    console.log("Warning: No collection type or collection type source defined.");
+                }
+                this.collectionType = ANY_COLLECTION_TYPE_DESCRIPTION;
+            }
+            this.isCollection =  true;
        },
        update: function( output ) {
-            var newCollectionType = new CollectionTypeDescription( output.collection_type );
+            var newCollectionType;
+            if( output.collection_type ) {
+                newCollectionType = new CollectionTypeDescription( output.collection_type );
+            } else {
+                var collectionTypeSource = output.collection_type_source;
+                if( ! collectionTypeSource ) {
+                    console.log("Warning: No collection type or collection type source defined.");
+                }
+                newCollectionType = ANY_COLLECTION_TYPE_DESCRIPTION;
+            }
+
            if( newCollectionType.collectionType != this.collectionType.collectionType ) {
                _.each( this.connectors, function( connector ) {
                    // TODO: consider checking if connection valid before removing...
@@ -224,7 +224,8 @@ define(['mvc/workflow/workflow-globals', 'mvc/workflow/workflow-terminals',
        terminalMappingViewClass: TerminalMappingView,
        terminalForOutput: function( output ) {
            var collection_type = output.collection_type;
-            var terminal = new Terminals.OutputCollectionTerminal( { element: this.el, collection_type: collection_type, datatypes: output.extensions } );
+            var collection_type_source = output.collection_type_source;
+            var terminal = new Terminals.OutputCollectionTerminal( { element: this.el, collection_type: collection_type, collection_type_source: collection_type_source, datatypes: output.extensions } );
            return terminal;
        }
    });
@@ -1,94 +1,241 @@
 /** Workflow view */
-define( [ 'utils/utils', 'mvc/ui/ui-misc' ], function( Utils, Ui ) {
+define( [ "libs/toastr", "mvc/tag", "mvc/workflow/workflow-model"  ], function( mod_toastr, TAGS, WORKFLOWS ) {
+
+    /** View of the individual workflows */
+    var WorkflowItemView = Backbone.View.extend({
+        tagName: 'tr', // name of (orphan) root tag in this.el
+        initialize: function(){
+            _.bindAll(this, 'render', '_rowTemplate', 'renderTagEditor', '_templateActions', 'removeWorkflow', 'copyWorkflow'); // every function that uses 'this' as the current object should be in here
+            mod_toastr.options.timeOut = 1500;
+        },
+
+        events: {
+            'click #show-in-tool-panel': 'showInToolPanel',
+            'click #delete-workflow'   : 'removeWorkflow',
+            'click #rename-workflow'   : 'renameWorkflow',
+            'click #copy-workflow'     : 'copyWorkflow',
+        },
+
+        render: function(){
+            $(this.el).html(this._rowTemplate());
+            return this;
+        },
+
+        showInToolPanel: function(){
+            this.model.set('show_in_tool_panel', !this.model.get('show_in_tool_panel'));
+            this.model.save();
+            // This reloads the whole page, so that the workflow appears in the tool panel.
+            // Ideally we would notify only the tool panel of a change
+            window.location = Galaxy.root + 'workflow';
+        },
+
+        removeWorkflow: function(){
+            var wfName = this.model.get('name');
+            if (confirm( "Are you sure you want to delete workflow '" + wfName + "'?" )) {
+                this.model.destroy({
+                    success: function() {
+                        mod_toastr.success("Successfully deleted workflow '" + wfName + "'");
+                        }
+                });
+                this.remove();
+            };
+        },
+
+        renameWorkflow: function(){
+            var oldName = this.model.get('name');
+            var newName = prompt("Enter a new Name for workflow '" + oldName + "'", oldName );
+            if (newName) {
+                this.model.save(
+                    { 'name': newName },
+                    { success: function() {
+                        mod_toastr.success("Successfully renamed workflow '" + oldName + "' to '" + newName + "'")
+                    }
+                });
+                this.render();
+            }
+        },
+
+        copyWorkflow: function(){
+            self = this;
+            var oldName = this.model.get('name');
+            $.getJSON(this.model.urlRoot + '/' + this.model.id + '/download', function(wfJson) {
+                var newName = 'Copy of ' + oldName;
+                var currentOwner = self.model.get('owner');
+                if (currentOwner != Galaxy.user.attributes.username) {
+                    newName += ' shared by user ' + currentOwner;
+                }
+                wfJson.name = newName;
+                self.collection.create(wfJson, { at: 0,
+                                                 wait: true,
+                                                 success: function() {
+                                                     mod_toastr.success("Successfully copied workflow '" + oldName + "' to '" + newName + "'")
+                                                 },
+                                                 error : function(model, resp, options) {
+                                                     // signature seems to have changed over the course of backbone dev
+                                                     // see https://github.com/jashkenas/backbone/issues/2606#issuecomment-19289483
+                                                     mod_toastr.error(options.errorThrown);
+                                                 }
+                });
+            }).error(function(jqXHR, textStatus, errorThrown) {
+                  mod_toastr.error(jqXHR.responseJSON.err_msg);
+            })
+        },
+
+        _rowTemplate: function() {
+            var show = this.model.get("show_in_tool_panel");
+            var wfId = this.model.id;
+            var checkboxHtml = '<input id="show-in-tool-panel" type="checkbox" class="show-in-tool-panel" '+ ( show ? 'checked="' + show + '"' : "" ) +' value="' + wfId + '">';
+            var trHtml =  '<td>' +
+                             '<div class="dropdown">' +
+                                 '<button class="menubutton" type="button" data-toggle="dropdown">' +
+                                     _.escape( this.model.get("name") ) + '<span class="caret"></span>' +
+                                 '</button>' +
+                                 this._templateActions( ) +
+                             '</div>' +
+                          '</td>' +
+                          '<td><span>' + '<div class="' + wfId + ' tags-display"></div>' + '</td>' +
+                          '<td>' + ( this.model.get('owner') === Galaxy.user.attributes.username ? "You" : this.model.get('owner') ) +'</span></td>' +
+                          '<td>' + this.model.get("number_of_steps") + '</td>' +
+                          '<td>' + ( this.model.get("published") ? "Yes" : "No" ) + '</td>' +
+                          '<td>'+ checkboxHtml + '</td>';
+            return trHtml;
+        },
+
+        renderTagEditor: function(){
+            var TagEditor = new TAGS.TagsEditor({
+            model           : this.model,
+            el              : $.find( '.' + this.model.id + '.tags-display' ),
+            workflow_mode   : true });
+            TagEditor.toggle( true );
+            TagEditor.render();
+        },
+
+        /** Template for user actions for workflows */
+        _templateActions: function( ) {
+            if( this.model.get("owner") === Galaxy.user.attributes.username ) {
+                return '<ul class="dropdown-menu action-dpd">' +
+                           '<li><a href="'+ Galaxy.root +'workflow/editor?id='+ this.model.id +'">Edit</a></li>' +
+                           '<li><a href="'+ Galaxy.root +'workflow/run?id='+ this.model.id +'">Run</a></li>' +
+                           '<li><a href="'+ Galaxy.root +'workflow/sharing?id='+ this.model.id +'">Share</a></li>' +
+                           '<li><a href="'+ Galaxy.root +'api/workflows/'+ this.model.id +'/download?format=json-download">Download</a></li>' +
+                           '<li><a id="copy-workflow" style="cursor: pointer;">Copy</a></li>' +
+                           '<li><a id="rename-workflow" style="cursor: pointer;">Rename</a></li>' +
+                           '<li><a href="'+ Galaxy.root +'workflow/display_by_id?id='+ this.model.id +'">View</a></li>' +
+                           '<li><a id="delete-workflow" style="cursor: pointer;">Delete</a></li>' +
+                      '</ul>';
+            }
+            else {
+                return '<ul class="dropdown-menu action-dpd">' +
+                         '<li><a href="'+ Galaxy.root +'workflow/display_by_username_and_slug?username='+ workflow.owner +'&slug='+ workflow.slug +'">View</a></li>' +
+                         '<li><a href="'+ Galaxy.root +'workflow/run?id='+ this.model.id +'">Run</a></li>' +
+                         '<li><a id="copy-workflow" style="cursor: pointer;">Copy</a></li>' +
+                         '<li><a class="link-confirm-shared-'+ this.model.id +'" href="'+ Galaxy.root +'workflow/sharing?unshare_me=True&id='+ this.model.id +'">Remove</a></li>' +
+                      '</ul>';
+            }
+        },
+    });

-    /** Build messages after user action */
-    function build_messages() {
-        var $el_message = this.$( '.response-message' ),
-            response = {};
-        response = {
-            'status': Utils.getQueryString( 'status' ),
-            'message': _.escape( Utils.getQueryString( 'message' ) ),
-            'persistent': true,
-            'cls': Utils.getQueryString( 'status' ) + 'message'
-        };
-        $el_message.empty().html( new Ui.Message( response ).$el );
-    }
- 
    /** View of the main workflow list page */
-    var View = Backbone.View.extend({
-
+    var WorkflowListView = Backbone.View.extend({
+        title: "Workflows",
        initialize: function() {
            this.setElement( '<div/>' );
-            this.render();
+            _.bindAll(this, 'adjustActiondropdown')
+            this.collection = new WORKFLOWS.WorkflowCollection();
+            this.collection.fetch().done(this.render());
+            this.collection.bind('add', this.appendItem);
+            this.collection.on('sync', this.render, this);
        },

+        events: {
+            'dragleave' : 'unhighlightDropZone',
+            'drop' : 'drop',
+            'dragover': function(ev) {
+                $( '.hidden_description_layer' ).addClass( 'dragover' );
+                $('.menubutton').addClass('background-none');
+                ev.preventDefault();
+            }
+        },
+
+        unhighlightDropZone: function() {
+            $( '.hidden_description_layer' ).removeClass( 'dragover' );
+            $('.menubutton').removeClass('background-none');
+        },
+
+        drop: function(e) {
+            // TODO: check that file is valid galaxy workflow
+            this.unhighlightDropZone();
+            e.preventDefault();
+            var files = e.dataTransfer.files;
+            var self = this;
+            for (var i = 0, f; f = files[i]; i++) {
+                self.readWorkflowFiles(f);
+            }
+        },
+
+        readWorkflowFiles: function(f) {
+                var self = this;
+                var reader = new FileReader();
+                reader.onload = function(theFile) {
+                    try {
+                        var wf_json = JSON.parse(reader.result);
+                    } catch(e) {
+                        mod_toastr.error("Could not read file '" + f.name + "'. Verify it is a valid Galaxy workflow");
+                        wf_json = null;
+                    }
+                    if (wf_json) {
+                        self.collection.create(wf_json, {
+                            at: 0,
+                            wait: true,
+                            success: function() {
+                                mod_toastr.success("Successfully imported workflow '" + wf_json.name + "'")
+                            },
+                            error : function(model, resp, options) {
+                                mod_toastr.error(options.errorThrown);
+                            }
+                        });
+                    }
+                };
+                reader.readAsText(f, 'utf-8');
+            },
+
        render: function() {
-            var self = this,
-                min_query_length = 3;
-            $.getJSON( Galaxy.root + 'api/workflows/', function( workflows ) {
-                var $el_workflow = null;
-                // Add workflow header
-                self.$el.empty().append( self._templateHeader() );
-                // Add user actions message if any
-                build_messages();
-                $el_workflow = self.$( '.user-workflows' );
-                // Add the actions buttons
-                $el_workflow.append( self._templateActionButtons() );
-                if( workflows.length > 0) {
-                    $el_workflow.append( self._templateWorkflowTable( self, workflows) );
-                    self.adjust_actiondropdown( $el_workflow );
-                    // Register delete and run workflow events
-                    _.each( workflows, function( wf ) {
-                        self.confirm_delete( wf );
-                    });
-                    self.register_show_tool_menu();
-                    // Register search workflow event
-                    self.search_workflow( self.$( '.search-wf' ), self.$( '.workflow-search tr' ), min_query_length );
-                }
-                else {
-                    $el_workflow.append( self._templateNoWorkflow() );
-                }
-            });
+            // Add workflow header
+            var header = this._templateHeader();
+            // Add the actions buttons
+            var templateActions = this._templateActionButtons();
+            var tableTemplate = this._templateWorkflowTable();
+            this.$el.html( header + templateActions + tableTemplate);
+            var self = this;
+            _(this.collection.models).each(function(item){ // in case collection is not empty
+                self.appendItem(item);
+                self.confirmDelete(item);
+            }, this);
+            var minQueryLength = 3;
+            this.searchWorkflow( this.$( '.search-wf' ), this.$( '.workflow-search tr' ), minQueryLength );
+            this.adjustActiondropdown();
+            return this;
        },

-        // Save the workflow as an item in Tool panel
-        register_show_tool_menu: function() {
-            var $el_checkboxes = this.$( '.show-in-tool-panel' );
-            $el_checkboxes.on( 'click', function( e ) {
-                var ids = [];
-                // Look for all the checked checkboxes
-                for( var item = 0; item < $el_checkboxes.length; item++ ) {
-                    var checkbox = $el_checkboxes[ item ];
-                    if( checkbox.checked ) {
-                        ids.push( checkbox.value );
-                    }
-                }
-                // Save all the checked workflows
-                $.ajax({
-                    type: 'PUT',
-                    url: Galaxy.root + 'api/workflows/menu/',
-                    data: JSON.stringify( { 'workflow_ids': ids } ),
-                    contentType : 'application/json'
-                }).done( function( response ) {
-                    window.location = Galaxy.root + 'workflow';
-                });
+        appendItem: function(item){
+            var workflowItemView = new WorkflowItemView({
+                model: item,
+                collection: this.collection,
            });
+            $( '.workflow-search' ).append(workflowItemView.render().el);
+            workflowItemView.renderTagEditor();
        },

        /** Add confirm box before removing/unsharing workflow */
-        confirm_delete: function( workflow ) {
-            var $el_wf_link = this.$( '.link-confirm-' + workflow.id ),
-                $el_shared_wf_link = this.$( '.link-confirm-shared-' + workflow.id );
-            $el_wf_link.click( function() {
-                return confirm( "Are you sure you want to delete workflow '" + workflow.name + "'?" );
-            });
+        confirmDelete: function( workflow ) {
+            var $el_shared_wf_link = this.$( '.link-confirm-shared-' + workflow.id );
            $el_shared_wf_link.click( function() {
-                return confirm( "Are you sure you want to remove the shared workflow '" + workflow.name + "'?" );
+                return confirm( "Are you sure you want to remove the shared workflow '" + workflow.attributes.name + "'?" );
            });
        },

        /** Implement client side workflow search/filtering */
-        search_workflow: function( $el_searchinput, $el_tabletr, min_querylen ) {
+        searchWorkflow: function( $el_searchinput, $el_tabletr, min_querylen ) {
            $el_searchinput.on( 'keyup', function () {
                var query = $( this ).val();
                // Filter when query is at least 3 characters
@@ -110,13 +257,12 @@ define( [ 'utils/utils', 'mvc/ui/ui-misc' ], function( Utils, Ui ) {
        },

        /** Ajust the position of dropdown with respect to table */
-        adjust_actiondropdown: function( $el ) {
-            $el.on( 'show.bs.dropdown', function () {
-                $el.css( "overflow", "inherit" );
+        adjustActiondropdown: function( ) {
+            $(this.el).on( 'show.bs.dropdown', function () {
+                $(this.el).css( "overflow", "inherit" );
            });
-
-            $el.on( 'hide.bs.dropdown', function () {
-                $el.css( "overflow", "auto" );
+            $(this.el).on( 'hide.bs.dropdown', function () {
+                $(this.el).css( "overflow", "auto" );
            });
        },

@@ -143,57 +289,17 @@ define( [ 'utils/utils', 'mvc/ui/ui-misc' ], function( Utils, Ui ) {
        },

        /** Template for workflow table */
-        _templateWorkflowTable: function( self, workflows ) {
-            var tableHtml = "", trHtml = "";
-            tableHtml = tableHtml + '<table class="table colored"><thead>' +
+        _templateWorkflowTable: function( ) {
+            var tableHtml = '<table class="table colored"><thead>' +
                    '<tr class="header">' +
                        '<th>Name</th>' +
+                        '<th>Tags</th>' +
                        '<th>Owner</th>' +
                        '<th># of Steps</th>' +
                        '<th>Published</th>' +
                        '<th>Show in tools panel</th>' +
                    '</tr></thead>';
-            _.each( workflows, function( wf ) {
-                var checkbox_html = '<input type="checkbox" class="show-in-tool-panel" '+ ( wf.show_in_tool_panel ? 'checked="' + wf.show_in_tool_panel + '"' : "" ) +' value="' + wf.id + '">';
-                trHtml = trHtml + '<tr>' +
-                             '<td>' +
-                                 '<div class="dropdown">' +
-                                     '<button class="menubutton" type="button" data-toggle="dropdown">' +
-                                         _.escape( wf.name ) + '<span class="caret"></span>' +
-                                     '</button>' +
-                                     self._templateActions( wf ) +
-                                 '</div>' +
-                              '</td>' +
-                              '<td>' + ( wf.owner === Galaxy.user.attributes.username ? "You" : wf.owner ) +'</td>' +
-                              '<td>' + wf.number_of_steps + '</td>' +
-                              '<td>' + ( wf.published ? "Yes" : "No" ) + '</td>' +
-                              '<td>'+ checkbox_html +'</td>' +
-                         '</tr>';
-            });
-            return tableHtml + '<tbody class="workflow-search">' + trHtml + '</tbody></table>';
-        },
-
-        /** Template for user actions for workflows */
-        _templateActions: function( workflow ) {
-            if( workflow.owner === Galaxy.user.attributes.username ) {
-                return '<ul class="dropdown-menu action-dpd">' +
-                           '<li><a href="'+ Galaxy.root +'workflow/editor?id='+ workflow.id +'">Edit</a></li>' +
-                           '<li><a href="'+ Galaxy.root +'workflow/run?id='+ workflow.id +'">Run</a></li>' +
-                           '<li><a href="'+ Galaxy.root +'workflow/sharing?id='+ workflow.id +'">Share or Download</a></li>' +
-                           '<li><a href="'+ Galaxy.root +'workflow/copy?id='+ workflow.id +'">Copy</a></li>' +
-                           '<li><a href="'+ Galaxy.root +'workflow/rename?id='+ workflow.id +'">Rename</a></li>' +
-                           '<li><a href="'+ Galaxy.root +'workflow/display_by_id?id='+ workflow.id +'">View</a></li>' +
-                           '<li><a class="link-confirm-'+ workflow.id +'" href="'+ Galaxy.root +'workflow/delete?id='+ workflow.id +'">Delete</a></li>' +
-                      '</ul>';
-            }
-            else {
-                return '<ul class="dropdown-menu action-dpd">' +
-                         '<li><a href="'+ Galaxy.root +'workflow/display_by_username_and_slug?username='+ workflow.owner +'&slug='+ workflow.slug +'">View</a></li>' +
-                         '<li><a href="'+ Galaxy.root +'workflow/run?id='+ workflow.id +'">Run</a></li>' +
-                         '<li><a href="'+ Galaxy.root +'workflow/copy?id='+ workflow.id +'">Copy</a></li>' +
-                         '<li><a class="link-confirm-shared-'+ workflow.id +'" href="'+ Galaxy.root +'workflow/sharing?unshare_me=True&id='+ workflow.id +'">Remove</a></li>' +
-                      '</ul>';
-            }
+            return tableHtml + '<tbody class="workflow-search "><div class="hidden_description_layer"><p>Drop workflow files here to import</p>' + '</tbody></table></div>';
        },

        /** Main template */
@@ -264,11 +370,10 @@ define( [ 'utils/utils', 'mvc/ui/ui-misc' ], function( Utils, Ui ) {
                       "</div>" +
                   "</div>";
        },
-
    });

    return {
-        View  : View,
+        View  : WorkflowListView,
        ImportWorkflowView : ImportWorkflowView
    };
 });
@@ -65,11 +65,14 @@
            if (xhr.readyState == xhr.DONE) {
                // parse response
                var response = null;
+                var extra_info = "";
                if (xhr.responseText) {
                    try {
                        response = jQuery.parseJSON(xhr.responseText);
+                        extra_info = response.err_msg;
                    } catch (e) {
                        response = xhr.responseText;
+                        extra_info = response;
                    }
                }
                // pass any error to the error option
@@ -82,7 +85,7 @@
                    } else if (!text) {
                        text = cnf.error_default;
                    }
-                    cnf.error(text + ' (' + xhr.status + ')');
+                    cnf.error(text + ' (' + xhr.status + '). ' + extra_info);
                } else {
                    cnf.success(response);
                }
@@ -2,7 +2,7 @@
 * Galaxy utilities comprises small functions, which at this point
 * do not require their own classes/files
 */
-define( [], function() {
+define( ['utils/localization'], function(_l) {

    /** Builds a basic iframe */
    function iframe( src ) {
@@ -53,7 +53,7 @@ define( [], function() {
        return /^[\],:{}\s]*$/.test(text.replace(/\\["\\\/bfnrtu]/g, '@').
            replace(/"[^"\\\n\r]*"|true|false|null|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?/g, ']').
            replace(/(?:^|:|,)(?:\s*\[)+/g, ''));
-    };
+    }

    /**
     * Sanitize/escape a string
@@ -61,7 +61,7 @@ define( [], function() {
     */
    function sanitize(content) {
        return $('<div/>').text(content).html();
-    };
+    }

    /**
     * Checks if a value or list of values is `empty`
@@ -81,7 +81,7 @@ define( [], function() {
            }
        }
        return false;
-    };
+    }

    /**
     * Convert list to pretty string
@@ -97,7 +97,7 @@ define( [], function() {
            return lst;
        }
        return '';
-    };
+    }

    /**
     * Request handler for GET
@@ -125,7 +125,7 @@ define( [], function() {
                }
            });
        }
-    };
+    }

    /**
     * Request handler
@@ -142,7 +142,7 @@ define( [], function() {
            type        : options.type || 'GET',
            data        : options.data || {},
            url         : options.url
-        }
+        };
        // encode data into url
        if ( ajaxConfig.type == 'GET' || ajaxConfig.type == 'DELETE' ) {
            if ( !$.isEmptyObject(ajaxConfig.data) ) {
@@ -178,7 +178,7 @@ define( [], function() {
        }).always(function() {
            options.complete && options.complete();
        });
-    };
+    }

    /**
     * Read a property value from CSS
@@ -191,7 +191,7 @@ define( [], function() {
        var value = el.css(name);
        el.remove();
        return value;
-    };
+    }

    /**
     * Load a CSS file
@@ -201,7 +201,7 @@ define( [], function() {
        if (!$('link[href^="' + url + '"]').length) {
            $('<link href="' + Galaxy.root + url + '" rel="stylesheet">').appendTo('head');
        }
-    };
+    }

    /**
     * Safely merge to dictionaries
@@ -214,7 +214,7 @@ define( [], function() {
        } else {
            return optionsDefault;
        }
-    };
+    }


    /**
@@ -257,13 +257,13 @@ define( [], function() {
        } else {
            return '<strong>' + rounded + '</strong> ' + unit;
        }
-    };
+    }

    /** Create a unique id */
    function uid(){
        top.__utils__uid__ = top.__utils__uid__ || 0;
        return 'uid-' + top.__utils__uid__++;
-    };
+    }

    /** Create a time stamp */
    function time() {
@@ -275,7 +275,7 @@ define( [], function() {
                    + d.getFullYear() + ", "
                    + hours + ":"
                    + minutes;
-    };
+    }

    /** Append script and style tags to Galaxy main application */
    function appendScriptStyle( data ) {
@@ -287,12 +287,20 @@ define( [], function() {
        if( data.styles && data.styles !== "" ) {
            $( '<style/>', { type: 'text/css' } ).text( data.styles ).appendTo( 'head' );
        }
-    };
+    }

    /** Get querystrings from url */
    function getQueryString( key ) {
        return decodeURIComponent( window.location.search.replace(new RegExp("^(?:.*[&\\?]" + encodeURIComponent( key ).replace(/[\.\+\*]/g, "\\$&") + "(?:\\=([^&]*))?)?.*$", "i"), "$1") );
-    };
+    }
+
+    function setWindowTitle(title){
+        if (title) {
+            window.document.title = "Galaxy " + (window.Galaxy.config.brand ? " | " + window.Galaxy.config.brand : '') + " | " + _l(title);
+        } else {
+            window.document.title = "Galaxy " + (window.Galaxy.config.brand ? " | " + window.Galaxy.config.brand : '');
+        }
+    }

    return {
        cssLoadFile: cssLoadFile,
@@ -312,6 +320,7 @@ define( [], function() {
        clone: clone,
        linkify: linkify,
        appendScriptStyle: appendScriptStyle,
-        getQueryString: getQueryString
+        getQueryString: getQueryString,
+        setWindowTitle: setWindowTitle
    };
 });
@@ -1271,6 +1271,10 @@ a.action-button {
    }
 }

+.menubutton.background-none {
+    background: none;
+}
+
 // A split menu button, the main button has an action, the arrow causes the
 // popup menu to appear

@@ -1824,3 +1828,26 @@ div.toolTitleNoSection
 .other-options {
    margin-bottom: 2%;
 }
+
+.hidden_description_layer {
+  position: absolute;
+  top: 0;
+  bottom: 0;
+  left: 0;
+  right: 0;
+  background: rgba(200, 200, 200, 0.6);
+  visibility: hidden;
+  opacity: 0;
+  font-size: 2.0em;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+
+  /* transition effect. not necessary */
+  transition: opacity .2s, visibility .2s;
+}
+
+.hidden_description_layer.dragover {
+  visibility: visible;
+  opacity: 1;
+}
@@ -140,7 +140,13 @@ th.button_heading{
    margin-top: 2em;
    margin-bottom: 2em;
 }
+}

+// Extra style for Galaxy modal
+.modal-content #selected_history_content{
+    ul{
+        list-style-type: none;
+    }
 }

 // Follows the style for the deprecated admin libraries interface
@@ -9,7 +9,6 @@
    <datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true" description="blastz pairwise alignment format.  Each alignment block in an axt file contains three lines: a summary line and 2 sequence lines.  Blocks are separated from one another by blank lines.  The summary line contains chromosomal position and size information about the alignment. It consists of 9 required fields." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Axt"/>
    <datatype extension="fli" type="galaxy.datatypes.tabular:FeatureLocationIndex" display_in_upload="false"/>
    <datatype extension="bam" type="galaxy.datatypes.binary:Bam" mimetype="application/octet-stream" display_in_upload="true" description="A binary file compressed in the BGZF format with a '.bam' file extension." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#BAM">
-      <converter file="bam_to_bai.xml" target_datatype="bai"/>
      <converter file="bam_to_bigwig_converter.xml" target_datatype="bigwig"/>
      <display file="ucsc/bam.xml" />
      <display file="ensembl/ensembl_bam.xml" />
@@ -236,6 +235,9 @@
    <datatype extension="cps" type="galaxy.datatypes.binary:Binary" subclass="true" display_in_upload="true" />
    <datatype extension="ct" type="galaxy.datatypes.tabular:ConnectivityTable" display_in_upload="true"/>
    <datatype extension="searchgui_archive" type="galaxy.datatypes.binary:SearchGuiArchive" display_in_upload="true"/>
+    <datatype extension="fast5.tar" type="galaxy.datatypes.binary:Fast5Archive" display_in_upload="true"/>
+    <datatype extension="fast5.tar.gz" type="galaxy.datatypes.binary:Fast5ArchiveGz" display_in_upload="true"/>
+    <datatype extension="fast5.tar.bz2" type="galaxy.datatypes.binary:Fast5ArchiveBz2" display_in_upload="true"/>
    <datatype extension="peptideshaker_archive" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true"/>
    <datatype extension="percin" type="galaxy.datatypes.tabular:Tabular" subclass="true" />
    <datatype extension="percout" type="galaxy.datatypes.xml:GenericXml" subclass="true" />
@@ -505,9 +507,13 @@
    <datatype extension="plybinary" type="galaxy.datatypes.constructive_solid_geometry:PlyBinary" display_in_upload="true" />
    <datatype extension="vtkascii" type="galaxy.datatypes.constructive_solid_geometry:VtkAscii" display_in_upload="true" />
    <datatype extension="vtkbinary" type="galaxy.datatypes.constructive_solid_geometry:VtkBinary" display_in_upload="true" />
-    <!-- Metagenomic Datatype -->
+    <!-- Metagenomic Datatypes -->
    <datatype extension="biom1" type="galaxy.datatypes.text:Biom1" display_in_upload="true" subclass="true" mimetype="application/json">
        <display file="biom/biom_simple.xml" />
+        <converter file="biom1_to_biom2.xml" target_datatype="biom2"/>
+    </datatype>
+    <datatype extension="biom2" type="galaxy.datatypes.binary:Biom2" mimetype="application/octet-stream" display_in_upload="true">
+        <converter file="biom2_to_biom1.xml" target_datatype="biom1"/>
    </datatype>
    <!-- Strand-specific Coordinate Count Datatype used by the Center for Eukaryotic Gene Regulation labs at Penn State -->
    <datatype extension="scidx" type="galaxy.datatypes.interval:ScIdx" display_in_upload="true" />
@@ -662,6 +668,7 @@
    <sniffer type="galaxy.datatypes.binary:MzSQlite"/>
    <sniffer type="galaxy.datatypes.binary:IdpDB"/>
    <sniffer type="galaxy.datatypes.binary:SQlite"/>
+    <sniffer type="galaxy.datatypes.binary:Biom2"/>
    <sniffer type="galaxy.datatypes.binary:H5"/>
    <sniffer type="galaxy.datatypes.binary:Bam"/>
    <sniffer type="galaxy.datatypes.binary:CRAM"/>
@@ -669,6 +676,9 @@
    <sniffer type="galaxy.datatypes.binary:Sra"/>
    <sniffer type="galaxy.datatypes.binary:NetCDF"/>
    <sniffer type="galaxy.datatypes.binary:DMND" />
+    <sniffer type="galaxy.datatypes.binary:Fast5ArchiveGz" />
+    <sniffer type="galaxy.datatypes.binary:Fast5ArchiveBz2" />
+    <sniffer type="galaxy.datatypes.binary:Fast5Archive" />
    <sniffer type="galaxy.datatypes.triples:Rdf"/>
    <sniffer type="galaxy.datatypes.blast:BlastXml"/>
    <sniffer type="galaxy.datatypes.xml:Phyloxml"/>
@@ -14,6 +14,26 @@
  <!-- look for any version of the dependency installed via conda -->
  <conda versionless="true" />

+  <!-- LMOD dependency resolver (For the LMOD environment modules system - https://github.com/TACC/Lmod) -->
+  <!--
+  The LMOD dependency resolver attributes are:
+  * lmodexec - Path to the lmod executable on your system - Default: value of the "LMOD_CMD" environment variable
+  * settargexec - Path to the settarg executable on your system - Default: value of the "LMOD_SETTARG_CMD" environment variable
+  * modulepath - Path to the folder that contains the LMOD module files on your system - Default: value of the "MODULEPATH" environment variable
+  * versionless - Set it to true to resolve a dependency based on its name only (the version number is ignored) - Default: false
+  * mapping_files - Path to a Yaml configuration file that can be used to link tools requirements with existing LMOD modules - Default: config/lmod_modules_mapping.yml
+  Important notes:
+   - All the above attributes are optional
+   - The value of the lmodexec attribute can't just be "module" because module is actually a bash function and not the real LMOD binary (see the result of the "type module" command)
+   - The value of the modulepath attribute can also be a semicolon separated list of path
+   - In versionless mode, only modules marked as Default will be listed by the "avail" command (The -d option is used)
+   - If the config folder of your Galaxy instance contains a file called "lmod_modules_mapping.yml" (based on the lmod_modules_mapping.yml.sample file) it will be taken into consideration automatically
+  -->
+  <!--
+  <lmod />
+  <lmod versionless="true" />
+  -->
+
  <!-- Example configuration of modules dependency resolver, uses Environment Modules -->
  <!--
  <modules modulecmd="/opt/Modules/3.2.9/bin/modulecmd" />
@@ -25,6 +45,7 @@
  * prefetch - default: true - in the AvailModuleChecker prefetch module info with 'module avail'
  * default_indicator - default: '(default)' - what indicate to the AvailModuleChecker that a module is the default version
  -->
+
  <!-- other resolvers
  <tool_shed_tap />
  <homebrew />
@@ -40,3 +40,14 @@
 # their behalf.
 # - type: biostars
 #   user_submission: true
+
+# InfluxDB error reporting backend. You will need to `pip install
+# influxdb` in the galaxy virtualenv yourself. This sends well tagged
+# errors InfluxDB allowing you to notice relationships between tool errors and
+# other infrastructure issues.
+# - type: influxdb
+#   # All arguments prefixed with `influxdb_` are per http://influxdb-python.readthedocs.io/en/latest/api-documentation.html#influxdbclient
+#   influxdb_host: 127.0.0.1
+#   influxdb_port: 8086
+#   influxdb_database: galaxy
+#   influxdb_timeout: 2
@@ -1069,6 +1069,15 @@ use_interactive = True
 #expose_user_name = False
 #expose_user_email = False

+# Whitelist for local network addresses for "Upload from URL" dialog.
+# By default, Galaxy will deny access to the local network address space, to
+# prevent users making requests to services which the administrator did not
+# intend to expose. Previously, you could request any network service that
+# Galaxy might have had access to, even if the user could not normally access it.
+# It should be a comma separated list of IP addresses or IP address/mask, e.g.
+# 10.10.10.10,10.0.1.0/24,fd00::/8
+#fetch_url_whitelist=
+
 # -- Beta features

 # Enable new run workflow form
@@ -0,0 +1,47 @@
+# This is an example mapping file for the LMOD Dependency resolver (in YAML format)
+#
+# The goal of this file is to map tool's requirements to existing LMOD modules available on your system
+# Of course, if the name of a requirement and the name of a module match perfectly, there is no need to map them together through this mapping file.
+#
+# This is a sample file so the first thing to do to activate the mapping system is to create a copy of this file called "lmod_modules_mapping.yml".
+# The Lmod dependency resolver is programmed to search and use this YAML file automatically if it exists in the "config" folder of your Galaxy instance.
+# Alternatively, you can also use the "mapping_files" attribute of the <lmod /> resolver in the dependency_resolvers_conf.xml file to specify a custom mapping file
+#
+# Example 1:
+#
+# Let's say that one of the wrapper installed on your Galaxy instance has the following requirement:
+#
+#	<requirements>
+#		<requirement type="package" version="1.5.0">PIPITS</requirement>
+#	</requirements>
+#
+# But unfortunately, the name of the corresponding module file on your system is "pipits_pipeline/1.5.0"
+#
+# Then, to make Galaxy load/unload the appropriate module, you just have to add the following lines (without to the #) to the "lmod_modules_mapping.yml" file:
+#
+#- from:
+#    name: PIPITS
+#    version: 1.5.0
+#  to:
+#    name: pipits_pipeline
+#    version: 1.5.0.6
+#
+#
+# Example 2:
+#
+# The requirements section specify a requirement on the PIPITS tool but do not ask for a specific version of it:
+#
+#	<requirements>
+#		<requirement type="package">PIPITS</requirement>
+#	</requirements>
+#
+# Although, there is no version required you may want to force the loading of a version that is known to run well on your system.
+#
+# In that case you can add the following lines to the "lmod_modules_mapping.yml" file:
+#
+#- from:
+#    name: PIPITS
+#    unversioned: true
+#  to:
+#    name: pipits_pipeline
+#    version: 1.4.0
@@ -0,0 +1,11 @@
+# This file lists acceptable images to allow runing.
+#
+# This allows you, the admin, to create multiple flavours
+# for your users to run. E.g. maybe you need a specific branded flavour,
+# you can create the image based on our default image and add the
+# appropriate files.
+---
+-
+    image: bgruening/docker-hicbrowser
+    description: |
+        A simple web browser to visualize Hi-C and other genomic tracks
@@ -0,0 +1,46 @@
+[main]
+# Following options are ignored if using the Galaxy dynamic proxy but
+# are useful if mapping a range of ports for environment consumption.
+#password_auth = False
+#ssl = False
+
+[docker]
+# Command to launch docker container. For example `sudo docker` or `docker-lxc`.
+# If you need to use a command like `sg` you can do that here, just be sure to
+# wrap all of the docker portion in single quotes. E.g. `sg 'docker' 'docker {docker_args}'`
+#
+# It is recommended that you use command_inject if you need to inject
+# additional parameters. This command string is re-used for a `docker inspect`
+# command and will likely cause errors if it is extensively modified, past the
+# usual group/sudo changes.
+#command = docker {docker_args}
+
+# The image argument was moved to "allowed_images.yml.sample"
+
+# Additional arguments that are passed to the `docker run` command.
+command_inject = --sig-proxy=true -e DEBUG=false -e KILL_MODE=True -e DEFAULT_CONTAINER_RUNTIME=120
+
+# URL to access the Galaxy API with from the spawn Docker containter, if empty
+# this falls back to galaxy.ini's galaxy_infrastructure_url and finally to the
+# Docker host of the spawned container if that is also not set.
+#galaxy_url =
+
+# The Docker hostname. It can be useful to run the Docker daemon on a different
+# host than Galaxy.
+#docker_hostname = localhost
+
+# Try to set the tempdirectory to world execute - this can fix the issue 
+# where 'sudo docker' is not able to mount the folder otherwise.
+# "finalize namespace chdir to /import permission denied"
+#wx_tempdir = False
+
+# Overwride the IE tempdirectory. This can be useful if you regular tempdir is
+# located on an NFS share, which does not work well as Docker volume. In this case
+# you can have a shared sshfs share which you can use as temporary directory to
+# share data between the IE and Galaxy.
+#docker_galaxy_temp_dir = None
+
+# If your Docker container exposes more then one port, Galaxy needs to know to
+# which ports it needs to connect. With this option you can specify the port number
+# inside your container to which Galaxy should connect.
+docker_connect_port = 80
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE interactive_environment SYSTEM "../../interactive_environments.dtd">
+<interactive_environment name="HiCBrowser">
+    <data_sources>
+        <data_source>
+            <model_class>HistoryDatasetAssociation</model_class>
+            <test type="isinstance" test_attr="datatype" result_type="datatype">binary.CompressedArchive</test>
+            <to_param param_attr="id">dataset_id</to_param>
+        </data_source>
+    </data_sources>
+    <params>
+        <param type="dataset" var_name_in_template="hda" required="true">dataset_id</param>
+    </params>
+    <entry_point entry_point_type="mako">hicbrowser.mako</entry_point>
+</interactive_environment>
@@ -0,0 +1,12 @@
+// Load an interactive environment (IE) from a remote URL
+// @param {String} hicexplorer_access_url: the URL embeded in the page and loaded
+function load_hicexplorer(hicexplorer_access_url){
+    // When the page has completely loaded...
+    $( document ).ready(function() {
+        // Test if we can access the GIE, and if so, execute the function
+        // to load the GIE for the user.
+        test_ie_availability(hicexplorer_access_url, function(){
+            append_notebook(hicexplorer_access_url);
+        });
+    });
+}
@@ -0,0 +1,49 @@
+<%namespace name="ie" file="ie.mako" />
+<%
+    # Sets ID and sets up a lot of other variables
+    ie_request.load_deploy_config()
+
+    # Define a volume that will be mounted into the container.
+    # This is a useful way to provide access to large files in the container,
+    # if the user knows ahead of time that they will need it.
+
+    import os
+    mount_path = hda.file_name
+    data_vol = ie_request.volume(mount_path, '/data/data_pack.tar.gz', how='rw')
+
+    # Add all environment variables collected from Galaxy's IE infrastructure
+    # Launch the IE.
+
+    ie_request.launch(
+       image = trans.request.params.get('image_tag', None),
+       additional_ids = trans.request.params.get('additional_dataset_ids', None),
+       volumes = [data_vol]
+    )
+
+    # Only once the container is launched can we template our URLs. The ie_request
+    # doesn't have all of the information needed until the container is running.
+
+    url = ie_request.url_template('${PROXY_URL}')
+
+%>
+<html>
+<head>
+    ${ ie.load_default_js() }
+</head>
+<body>
+<script type="text/javascript">
+
+    ${ ie.default_javascript_variables() }
+    var url = '${ url }';
+    ${ ie.plugin_require_config() }
+
+
+    requirejs(['interactive_environments', 'plugin/hicbrowser'], function () {
+        load_hicexplorer(url);
+    });
+
+</script>
+<div id="main" width="100%" height="100%">
+</div>
+</body>
+</html>
@@ -70,3 +70,22 @@ When ``verbose="true" user_submission="true"``, the plugin will inform the user
 that ``Submitted bug report to Sentry. Your guru meditation number is
 dc907d44ce294f78b267a56f68e5cd1a``, using the same phrasing that is common to
 users from Galaxy internal server errors.
+
+InfluxDB
+--------
+
+This sends data directly to an InfluxDB server that you have available. If you wish to
+use this plugin you will first need to ``pip install influxdb`` in Galaxy's virtual environment.
+
+This plugin will send a value of ``1`` every time an error occurs, tagged with important information such as:
+
+- handler
+- tool_id
+- tool_version
+- exit_code
+
+This allows you to visualize the rate of bug reports (``group by time(30m)``,
+adjust as needed for how many error reports you see) in conjunction with any
+other data you're already tracking in InfluxDB/Grafana. This setup allows
+answering questions such as "did the change I make decrease the number of tool
+failures on average"
@@ -0,0 +1,198 @@
+Finding and improving slow Galaxy code
+--------------------------------------
+
+This is a short howto on how one can find slow code in galaxy (but this
+should apply to other projects as well).
+
+I will walk through how I have improved the tool form building speed in
+https://github.com/galaxyproject/galaxy/pull/4541.
+
+Identifying the problem
+~~~~~~~~~~~~~~~~~~~~~~~
+
+@bgruening mentioned that loading the tool form was slow on his server,
+and I checked ours and saw that for certain tools it took around 2-3
+seconds to load the tool form, while for others this was significantly
+faster.
+
+Identifying a rough entrypoint for profiling the code
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you know the part in the UI that is slow, you can identify the
+corresponding API endpoint (assuming the backend is slow) by looking at
+the network tab in Chrome's javascript console while doing the operation
+that is slow. When I clicked on hisat2 in the tool menu, I saw that
+there was a GET request to
+http://127.0.0.1:8080/api/tools/toolshed.g2.bx.psu.edu/repos/iuc/hisat2/hisat2/2.0.5.2/build?tool\_version=2.0.5.2
+that took 3 seconds to complete. Looking into `galaxy's API
+documentation <https://docs.galaxyproject.org/en/master/api/api.html#galaxy.webapps.galaxy.api.tools.ToolsController.build>`__
+we can match this URL to the actual code in
+`lib/galaxy/webapps/galaxy/api/tools.py <https://github.com/galaxyproject/galaxy/blob/release_17.05/lib/galaxy/webapps/galaxy/api/tools.py#L89>`__.
+
+Profiling
+~~~~~~~~~
+
+I like the profilehooks library, which provides a decorator for
+profiling specific functions like our ``build`` function. To use it,
+install profilehooks into galaxy's python environment ( sourcing
+galaxy's virtualenv and running ``pip install profilehooks`` should be
+enough) and import the profile function at the top of the file that
+contains the function you would like to profile
+(``from profilehooks import profile``), and then add an additional
+``@profile`` decorator just above the ``build`` function. You can now
+start galaxy, hit the API endpoint a few times and shut down galaxy
+again. You should see profilehooks output in your logs. This is the
+output I saw
+
+::
+
+               6585515 function calls (6497718 primitive calls) in 9.560 seconds
+
+       Ordered by: cumulative time, internal time, call count
+       List reduced from 997 to 40 due to restriction <40>
+
+       ncalls  tottime  percall  cumtime  percall filename:lineno(function)
+            4    0.000    0.000    9.572    2.393 decorators.py:227(decorator)
+            4    0.000    0.000    9.567    2.392 tools.py:89(build)
+            4    0.000    0.000    9.566    2.391 __init__.py:1786(to_json)
+        116/4    0.005    0.000    8.973    2.243 __init__.py:1877(populate_model)
+        27312    0.063    0.000    8.110    0.000 dataset_matcher.py:74(hda_match)
+         2184    0.005    0.000    6.426    0.003 dataset_matcher.py:163(hdca_match)
+       100/56    0.001    0.000    6.389    0.114 grouping.py:625(to_dict)
+      300/168    0.001    0.000    6.388    0.038 grouping.py:628(nested_to_dict)
+    3720/2184    0.014    0.000    6.387    0.003 dataset_matcher.py:170(dataset_collection_match)
+      768/324    0.002    0.000    6.386    0.020 {map}
+      200/112    0.001    0.000    6.384    0.057 grouping.py:643(to_dict)
+      468/308    0.001    0.000    6.381    0.021 grouping.py:646(input_to_dict)
+    5376/2760    0.018    0.000    5.949    0.002 dataset_matcher.py:146(__valid_element)
+          104    0.016    0.000    5.451    0.052 basic.py:1775(to_dict)
+        27312    0.076    0.000    4.980    0.000 dataset_matcher.py:34(hda_accessible)
+          328    0.011    0.000    4.965    0.015 query.py:2700(one)
+         3840    0.009    0.000    4.825    0.001 dataset_matcher.py:106(__can_access_dataset)
+          176    0.001    0.000    4.797    0.027 context.py:119(get_current_user_roles)
+          176    0.004    0.000    4.796    0.027 __init__.py:237(all_roles)
+          508    0.003    0.000    4.379    0.009 query.py:2756(__iter__)
+        10680    0.042    0.000    3.028    0.000 dataset_matcher.py:47(valid_hda_match)
+           32    0.002    0.000    2.975    0.093 basic.py:1943(to_dict)
+          208    0.015    .000    2.924    0.014 basic.py:1480(get_initial_value)
+         8144    0.021    0.000    2.645    0.000 __init__.py:2204(find_conversion_destination)
+         8144    0.010    0.000    2.584    0.000 data.py:611(find_conversion_destination)
+         8144    0.035    0.000    2.573    0.000 registry.py:818(find_conversion_destination_for_dataset_by_extensions)
+          508    0.009    0.000    2.534    0.005 query.py:3204(_compile_context)
+         8144    0.451    0.000    2.360    0.000 registry.py:798(get_converters_by_datatype)
+          508    0.003    0.000    2.083    0.004 query.py:3568(setup_context)
+     1580/508    0.054    0.000    2.080    0.004 loading.py:224(_setup_entity_query)
+    20660/9712    0.062    0.000    2.056    0.000 interfaces.py:498(setup)
+     1072/360    0.011    0.000    1.997    0.006 strategies.py:1114(setup_query)
+          816    0.004    0.000    1.928    0.002 basic.py:208(to_dict)
+          508    0.004    0.000    1.842    0.004 query.py:2770(_execute_and_instances)
+          508    0.002    0.000    1.786    0.004 base.py:846(execute)
+          508    0.001    0.000    1.783    0.004 elements.py:322(_execute_on_connection)
+          508    0.005    0.000    1.782    0.004 base.py:975(_execute_clauseelement)
+          128    0.002    0.000    1.781    0.014 basic.py:1878(match_multirun_collections)
+      13308    0.064    0.000    1.716    0.000 visitors.py:199(traverse)
+      13308    0.088    0.000    1.651    0.000 visitors.py:304(replacement_traverse)
+
+I loaded the tool form 4 times, as you can see on the second line of the
+output (ncalls=4). The table is sorted by the cumulative time that the
+functions ran while ``build`` was being evaluated.
+
+Optimizing the slow function calls
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The most valuable targets for optimization therefore should be
+relatively high in the table. I have seen two functions that I would not
+think of as expensive and that should not be that high in the table:
+
+``get_current_user_roles``, which took 4.8 seconds out of 9.5 seconds
+total and is being called 176 times and ``get_converters_by_datatype``
+which took 2.4 seconds out the 9.5 seconds total.
+
+``get_current_user_roles`` is called in the context of
+``hda_accessible`` and ``hda_accessible`` (these are also in our table)
+in
+https://github.com/galaxyproject/galaxy/blob/release\_17.05/lib/galaxy/tools/parameters/dataset\_matcher.py#L109
+We can see that the ``current_user_roles`` attribute of
+``DatasetMatcher`` instances are already being cached, however during
+the course of filling in parameters in the tool building process a new
+``DatasetMatcher`` instance is being created for each
+``DataToolParameter`` in
+https://github.com/galaxyproject/galaxy/blob/release\_17.05/lib/galaxy/tools/parameters/basic.py#L1456.
+This means that the cached roles will be lost for the next
+``DataToolParameter`` that we need to fill in. It would be great if we
+didn't need to redo this expensive operation for each data input.
+
+When building the tool interface we deal with a ``WorkRequestContext``
+instance, which inherits from ``ProvidesUserContext`` in
+https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/managers/context.py#L119.
+``ProvidesUserContext`` defines a ``get_current_user_roles`` method that
+gets the current users' roles from the database. We can implement a
+cached variant of this in ``WorkRequestContext``, which will be used
+when building the tool form. You can find this change in
+https://github.com/galaxyproject/galaxy/pull/4541/commits/d1a2007275f128fea051ead55fb47d2c2686abf5
+
+The slowness in ``get_converters_by_datatype`` can be circumvented by
+caching the result of this function, which I have done in
+https://github.com/galaxyproject/galaxy/pull/4541/commits/471707bd7dfa048a412aad9cdcc1d0b4aea70bc7
+(and fixed a mistake in
+https://github.com/galaxyproject/galaxy/pull/4541/commits/2d8b242e697a08775879fc873578b5f244d4d5cb)
+
+Checking how the changes affect the speed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+After these changes we can verify that the code is faster now and that
+the key functions we targeted are not as high anymore in the profiling
+table output:
+
+::
+
+             1384317 function calls (1363256 primitive calls) in 2.215 seconds
+
+       Ordered by: cumulative time, internal time, call count
+       List reduced from 991 to 40 due to restriction <40>
+
+       ncalls  tottime  percall  cumtime  percall filename:lineno(function)
+            4    0.000    0.000    2.220    0.555 decorators.py:227(decorator)
+            4    0.000    0.000    2.215    0.554 tools.py:89(build)
+            4    0.000    0.000    2.214    0.554 __init__.py:1786(to_json)
+        116/4    0.004    0.000    1.861    0.465 __init__.py:1877(populate_model)
+       100/56    0.000    0.000    1.578    0.028 grouping.py:625(to_dict)
+      300/168    0.000    0.000    1.577    0.009 grouping.py:628(nested_to_dict)
+      768/324    0.001    0.000    1.576    0.005 {map}
+      200/112    0.001    0.000    1.574    0.014 grouping.py:643(to_dict)
+      468/308    0.001    0.000    1.572    0.005 grouping.py:646(input_to_dict)
+         2184    0.003    0.000    1.055    0.000 dataset_matcher.py:163(hdca_match)
+       153436    0.147    0.000    1.048    0.000 attributes.py:229(__get__)
+    3720/2184    0.010    0.000    1.019    0.000 dataset_matcher.py:170(dataset_collection_match)
+           32    0.002    0.000    0.956    0.030 basic.py:1943(to_dict)
+     1924/964    0.005    0.000    0.901    0.001 attributes.py:561(get)
+          960    0.006    0.000    0.875    0.001 strategies.py:492(_load_for_state)
+        27312    0.051    0.000    0.860    0.000 dataset_matcher.py:74(hda_match)
+          324    0.001    0.000    0.857    0.003 <string>:1(<lambda>)
+          324    0.009    0.000    0.855    0.003 strategies.py:565(_emit_lazyload)
+          104    0.012    0.000    0.807    0.008 basic.py:1775(to_dict)
+          336    0.002    0.000    0.803    0.002 query.py:2756(__iter__)
+          208    0.011    0.000    0.752    0.004 basic.py:1480(get_initial_value)
+          180    0.004    0.000    0.666    0.004 query.py:2607(all)
+    5376/2760    0.014    0.000    0.612    0.000 dataset_matcher.py:146(__valid_element)
+          336    0.002    0.000    0.606    0.002 query.py:2770(_execute_and_instances)
+          336    0.001    0.000    0.570    0.002 base.py:846(execute)
+          336    0.001    0.000    0.569    0.002 elements.py:322(_execute_on_connection)
+          336    0.003    0.000    0.568    0.002 base.py:975(_execute_clauseelement)
+        10680    0.031    0.000    0.526    0.000 dataset_matcher.py:47(valid_hda_match)
+          128    0.001    0.000    0.525    0.004 basic.py:1878(match_multirun_collections)
+          156    0.003    0.000    0.446    0.003 query.py:2700(one)
+          816    0.004    0.000    0.426    0.001 basic.py:208(to_dict)
+          336    0.006    0.000    0.348    0.001 base.py:1061(_execute_context)
+          152    0.003    0.000    0.342    0.002 loading.py:161(load_on_ident)
+         44/4    0.001    0.000    0.324    0.081 __init__.py:219(populate_state)
+        16360    0.052    0.000    0.307    0.000 data.py:713(matches_any)
+         1300    0.007    0.000    0.302    0.000 loading.py:30(instances)
+        60/40    0.001    0.000    0.294    0.007 grouping.py:608(get_initial_value)
+    6168/3720    0.012    0.000    0.270    0.000 __init__.py:3222(populated)
+         336    0.001    0.000    0.268    0.001 default.py:449(do_execute)
+         336    0.262    0.001    0.267    0.001 {method 'execute' of 'psycopg2.extensions.cursor' objects}
+
+As before I hit the `build` endpoint 4 times.
+Both functions have disappeared from the table of the 40 longest running
+function calls, and the total time required has decreased from 9.5 seconds to 2.2 seconds.
@@ -4,6 +4,7 @@ Universe configuration builder.
 # absolute_import needed for tool_shed package.
 from __future__ import absolute_import

+import ipaddress
 import logging
 import logging.config
 import os
@@ -26,6 +27,7 @@ from galaxy.exceptions import ConfigurationError
 from galaxy.util import ExecutionTimer
 from galaxy.util import listify
 from galaxy.util import string_as_bool
+from galaxy.util import unicodify
 from galaxy.util.dbkeys import GenomeBuilds
 from galaxy.web.formatting import expand_pretty_datetime_format
 from galaxy.web.stack import register_postfork_function
@@ -226,6 +228,13 @@ class Configuration(object):
        self.remote_user_logout_href = kwargs.get("remote_user_logout_href", None)
        self.remote_user_secret = kwargs.get("remote_user_secret", None)
        self.require_login = string_as_bool(kwargs.get("require_login", "False"))
+        self.fetch_url_whitelist_ips = [
+            ipaddress.ip_network(unicodify(ip.strip()))  # If it has a slash, assume 127.0.0.1/24 notation
+            if '/' in ip else
+            ipaddress.ip_address(unicodify(ip.strip()))  # Otherwise interpret it as an ip address.
+            for ip in kwargs.get("fetch_url_whitelist", "").split(',')
+            if len(ip.strip()) > 0
+        ]
        self.allow_user_creation = string_as_bool(kwargs.get("allow_user_creation", "True"))
        self.allow_user_deletion = string_as_bool(kwargs.get("allow_user_deletion", "False"))
        self.allow_user_dataset_purge = string_as_bool(kwargs.get("allow_user_dataset_purge", "True"))
@@ -8,10 +8,13 @@ import os
 import shutil
 import struct
 import subprocess
+import sys
+import tarfile
 import tempfile
 import zipfile
 from json import dumps

+import h5py
 import pysam
 from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE

@@ -19,6 +22,7 @@ from galaxy import util
 from galaxy.datatypes import metadata
 from galaxy.datatypes.metadata import DictParameter, ListParameter, MetadataElement, MetadataParameter
 from galaxy.util import FILENAME_VALID_CHARS, nice_size, sqlite, which
+from galaxy.util.checkers import is_bz2, is_gzip
 from . import data, dataproviders


@@ -810,6 +814,90 @@ class H5(Binary):
            return "Binary HDF5 file (%s)" % (nice_size(dataset.get_size()))


+class Biom2(H5):
+    """
+    Class describing a biom2 file (http://biom-format.org/documentation/biom_format.html)
+    """
+    MetadataElement(name="id", default=None, desc="table id", readonly=True, visible=True, no_value=None)
+    MetadataElement(name="format_url", default=None, desc="format-url", readonly=True, visible=True, no_value=None)
+    MetadataElement(name="format_version", default=None, desc="format-version", readonly=True, visible=True, no_value=None)
+    MetadataElement(name="format", default=None, desc="format", readonly=True, visible=True, no_value=None)
+    MetadataElement(name="type", default=None, desc="table type", readonly=True, visible=True, no_value=None)
+    MetadataElement(name="generated_by", default=None, desc="generated by", readonly=True, visible=True, no_value=None)
+    MetadataElement(name="creation_date", default=None, desc="creation date", readonly=True, visible=True, no_value=None)
+    MetadataElement(name="nnz", default=-1, desc="nnz: The number of non-zero elements in the table", readonly=True, visible=True, no_value=-1)
+    MetadataElement(name="shape", default=(), desc="shape: The number of rows and columns in the dataset", readonly=True, visible=True, no_value=())
+
+    file_ext = "biom2"
+    edam_format = "format_3746"
+
+    def sniff(self, filename):
+        """
+        >>> from galaxy.datatypes.sniff import get_test_fname
+        >>> fname = get_test_fname( 'biom2_sparse_otu_table_hdf5.biom' )
+        >>> Biom2().sniff( fname )
+        True
+        >>> fname = get_test_fname( 'test.mz5' )
+        >>> Biom2().sniff( fname )
+        False
+        >>> fname = get_test_fname( 'wiggle.wig' )
+        >>> Biom2().sniff( fname )
+        False
+        """
+        if super(Biom2, self).sniff(filename):
+            try:
+                f = h5py.File(filename)
+                attributes = list(dict(f.attrs.items()))
+                required_fields = ['id', 'format-url', 'type', 'generated-by', 'creation-date', 'nnz', 'shape']
+                return set(required_fields).issubset(attributes)
+            except Exception:
+                return False
+        return False
+
+    def set_meta(self, dataset, overwrite=True, **kwd):
+        super(Biom2, self).set_meta(dataset, overwrite=overwrite, **kwd)
+        try:
+            f = h5py.File(dataset.file_name)
+            attributes = dict(f.attrs.items())
+
+            dataset.metadata.id = attributes['id']
+            dataset.metadata.format_url = attributes['format-url']
+            if 'format-version' in attributes:  # biom 2.1
+                dataset.metadata.format_version = '.'.join(map(str, list(attributes['format-version'])))
+            elif 'format' in attributes:  # biom 2.0
+                dataset.metadata.format = attributes['format']
+            dataset.metadata.type = attributes['type']
+            dataset.metadata.shape = tuple(attributes['shape'])
+            dataset.metadata.generated_by = attributes['generated-by']
+            dataset.metadata.creation_date = attributes['creation-date']
+            dataset.metadata.nnz = int(attributes['nnz'])
+
+        except Exception as e:
+            log.warning('%s, set_meta Exception: %s', self, e)
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        if not dataset.dataset.purged:
+            lines = ['Biom2 (HDF5) file']
+            try:
+                f = h5py.File(dataset.file_name)
+                for k, v in dict(f.attrs).items():
+                    lines.append('%s:  %s' % (k, v))
+            except Exception as e:
+                log.warning('%s, set_peek Exception: %s', self, e)
+            dataset.peek = '\n'.join(lines)
+            dataset.blurb = nice_size(dataset.get_size())
+        else:
+            dataset.peek = 'file does not exist'
+            dataset.blurb = 'file purged from disk'
+
+    def display_peek(self, dataset):
+        try:
+            return dataset.peek
+        except:
+            return "Biom2 (HDF5) file (%s)" % (nice_size(dataset.get_size()))
+
+
+Binary.register_sniffable_binary_format("biom2", "biom2", Biom2)
 Binary.register_sniffable_binary_format("h5", "h5", H5)


@@ -1488,6 +1576,111 @@ Binary.register_sniffable_binary_format("oxli.graphlabels", "oxligl",
                                        OxliGraphLabels)


+class Fast5Archive(CompressedArchive):
+    """
+    Class describing a FAST5 archive
+
+    >>> from galaxy.datatypes.sniff import get_test_fname
+    >>> fname = get_test_fname( 'test.fast5.tar' )
+    >>> Fast5Archive().sniff( fname )
+    True
+    """
+    MetadataElement(name="fast5_count", default='0', param=MetadataParameter, desc="Read Count",
+                    readonly=True, visible=True, no_value=None)
+    file_ext = "fast5.tar"
+
+    def set_meta(self, dataset, overwrite=True, **kwd):
+        super(Fast5Archive, self).set_meta(dataset, overwrite=overwrite, **kwd)
+        try:
+            if dataset and tarfile.is_tarfile(dataset.file_name):
+                with tarfile.open(dataset.file_name, 'r') as temptar:
+                    dataset.metadata.fast5_count = sum(
+                        1 for f in temptar if f.name.endswith('.fast5')
+                    )
+        except Exception as e:
+            log.warning('%s, set_meta Exception: %s', self, e)
+
+    def sniff(self, filename):
+        try:
+            if filename and tarfile.is_tarfile(filename):
+                with tarfile.open(filename, 'r') as temptar:
+                    for f in temptar:
+                        if not f.isfile():
+                            continue
+                        if f.name.endswith('.fast5'):
+                            return True
+                        else:
+                            return False
+        except Exception as e:
+            log.warning('%s, sniff Exception: %s', self, e)
+        return False
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        if not dataset.dataset.purged:
+            dataset.peek = "FAST5 Archive (%s)" % (nice_size(dataset.get_size()))
+            dataset.blurb = "%s sequences" % (dataset.metadata.fast5_count or 'unknown')
+        else:
+            dataset.peek = 'file does not exist'
+            dataset.blurb = 'file purged from disk'
+
+    def display_peek(self, dataset):
+        try:
+            return dataset.peek
+        except:
+            return "FAST5 Archive (%s)" % (nice_size(dataset.get_size()))
+
+
+class Fast5ArchiveGz(Fast5Archive):
+    """
+    Class describing a gzip-compressed FAST5 archive
+
+    >>> from galaxy.datatypes.sniff import get_test_fname
+    >>> fname = get_test_fname( 'test.fast5.tar.gz' )
+    >>> Fast5ArchiveGz().sniff( fname )
+    True
+    >>> fname = get_test_fname( 'test.fast5.tar.bz2' )
+    >>> Fast5ArchiveGz().sniff( fname )
+    False
+    >>> fname = get_test_fname( 'test.fast5.tar' )
+    >>> Fast5ArchiveGz().sniff( fname )
+    False
+    """
+    file_ext = "fast5.tar.gz"
+
+    def sniff(self, filename):
+        if not is_gzip(filename):
+            return False
+        return Fast5Archive.sniff(self, filename)
+
+
+class Fast5ArchiveBz2(Fast5Archive):
+    """
+    Class describing a bzip2-compressed FAST5 archive
+
+    >>> from galaxy.datatypes.sniff import get_test_fname
+    >>> fname = get_test_fname( 'test.fast5.tar.bz2' )
+    >>> Fast5ArchiveBz2().sniff( fname )
+    True
+    >>> fname = get_test_fname( 'test.fast5.tar.gz' )
+    >>> Fast5ArchiveBz2().sniff( fname )
+    False
+    >>> fname = get_test_fname( 'test.fast5.tar' )
+    >>> Fast5ArchiveBz2().sniff( fname )
+    False
+    """
+    file_ext = "fast5.tar.bz2"
+
+    def sniff(self, filename):
+        if not is_bz2(filename):
+            return False
+        return Fast5Archive.sniff(self, filename)
+
+
+Binary.register_sniffable_binary_format("fast5_archive_bz2", "fast5.tar.bz2", Fast5ArchiveBz2)
+Binary.register_sniffable_binary_format("fast5_archive_gz", "fast5.tar.gz", Fast5ArchiveGz)
+Binary.register_sniffable_binary_format("fast5_archive", "fast5.tar", Fast5Archive)
+
+
 class SearchGuiArchive(CompressedArchive):
    """Class describing a SearchGUI archive """
    MetadataElement(name="searchgui_version", default='1.28.0', param=MetadataParameter, desc="SearchGui Version",
@@ -1607,3 +1800,8 @@ class DMND(Binary):


 Binary.register_sniffable_binary_format("dmnd", "dmnd", DMND)
+
+
+if __name__ == '__main__':
+    import doctest
+    doctest.testmod(sys.modules[__name__])
@@ -1,14 +0,0 @@
-<tool id="CONVERTER_Bam_Bai_0" name="Bam to Bai" version="1.0.0" hidden="true">
-    <requirements>
-        <requirement type="package">samtools</requirement>
-    </requirements>
-    <command>samtools index '$input1' '$output1'</command>
-    <inputs>
-        <param format="bam" name="input1" type="data" label="Choose BAM"/>
-    </inputs>
-    <outputs>
-        <data format="bai" name="output1"/>
-    </outputs>
-    <help>
-    </help>
-</tool>
@@ -0,0 +1,14 @@
+<tool id="CONVERTER_biom1_to_biom2" name="Convert Biom1 to Biom2" version="2.1.5">
+    <requirements>
+        <requirement type="package" version="2.1.5">biom-format</requirement>
+    </requirements>
+    <command>biom convert -i '$input' -o '$output' --to-hdf5 </command>
+    <inputs>
+        <param name="input" type="data" format="biom1" label="Biom1 file"/>
+    </inputs>
+    <outputs>
+        <data name="output" format="biom2"/>
+    </outputs>
+    <help>
+    </help>
+</tool>
@@ -0,0 +1,14 @@
+<tool id="CONVERTER_biom2_to_biom1" name="Convert Biom2 to Biom1" version="2.1.5">
+    <requirements>
+        <requirement type="package" version="2.1.5">biom-format</requirement>
+    </requirements>
+    <command>biom convert -i '$input' -o '$output' --to-json </command>
+    <inputs>
+        <param name="input" type="data" format="biom2" label="Biom2 file"/>
+    </inputs>
+    <outputs>
+        <data name="output" format="biom1"/>
+    </outputs>
+    <help>
+    </help>
+</tool>
@@ -256,7 +256,6 @@ class LimitedOffsetDataProvider(FilteredDataProvider):
        """
        if self.limit is not None and self.limit <= 0:
            return
-            yield

        parent_gen = super(LimitedOffsetDataProvider, self).__iter__()
        for datum in parent_gen:
@@ -9,7 +9,6 @@ import logging
 import sys

 from bx import (
-    bbi as bx_bbi,
    seq as bx_seq,
    wiggle as bx_wig
 )
@@ -168,9 +167,9 @@ class ConvertedDatasetDataProvider(DatasetDataProvider):

    def __init__(self, dataset, **kwargs):
        raise NotImplementedError('Abstract class')
-        self.original_dataset = dataset
-        self.converted_dataset = self.convert_dataset(dataset, **kwargs)
-        super(ConvertedDatasetDataProvider, self).__init__(self.converted_dataset, **kwargs)
+        # self.original_dataset = dataset
+        # self.converted_dataset = self.convert_dataset(dataset, **kwargs)
+        # super(ConvertedDatasetDataProvider, self).__init__(self.converted_dataset, **kwargs)
        # NOTE: now self.converted_dataset == self.dataset

    def convert_dataset(self, dataset, **kwargs):
@@ -553,12 +552,12 @@ class BigWigDataProvider(base.LimitedOffsetDataProvider):
        raise NotImplementedError('Work in progress')
        # TODO: validate is a wig
        # still good to maintain a ref to the raw source bc Reader won't
-        self.raw_source = source
-        self.parser = bx_bbi.bigwig_file.BigWigFile(source)
-        super(BigWigDataProvider, self).__init__(self.parser, **kwargs)
+        # self.raw_source = source
+        # self.parser = bx_bbi.bigwig_file.BigWigFile(source)
+        # super(BigWigDataProvider, self).__init__(self.parser, **kwargs)

-        self.named_columns = named_columns
-        self.column_names = column_names or self.COLUMN_NAMES
+        # self.named_columns = named_columns
+        # self.column_names = column_names or self.COLUMN_NAMES

    def __iter__(self):
        parent_gen = super(BigWigDataProvider, self).__iter__()
@@ -586,8 +585,8 @@ class DatasetSubprocessDataProvider(external.SubprocessDataProvider):
        :type args: variadic function args
        """
        raise NotImplementedError('Abstract class')
-        super(DatasetSubprocessDataProvider, self).__init__(*args, **kwargs)
-        self.dataset = dataset
+        # super(DatasetSubprocessDataProvider, self).__init__(*args, **kwargs)
+        # self.dataset = dataset


 class SamtoolsDataProvider(line.RegexLineDataProvider):
@@ -699,7 +698,7 @@ class BcftoolsDataProvider(line.RegexLineDataProvider):
    def __init__(self, dataset, **kwargs):
        # TODO: as samtools
        raise NotImplementedError()
-        super(BcftoolsDataProvider, self).__init__(dataset, **kwargs)
+        # super(BcftoolsDataProvider, self).__init__(dataset, **kwargs)


 class BGzipTabixDataProvider(base.DataProvider):
@@ -712,7 +711,7 @@ class BGzipTabixDataProvider(base.DataProvider):
    def __init__(self, dataset, **kwargs):
        # TODO: as samtools - need more info on output format
        raise NotImplementedError()
-        super(BGzipTabixDataProvider, self).__init__(dataset, **kwargs)
+        # super(BGzipTabixDataProvider, self).__init__(dataset, **kwargs)


 class SQliteDataProvider(base.DataProvider):
@@ -156,8 +156,8 @@ class TempfileDataProvider(base.DataProvider):
        # TODO:
        raise NotImplementedError()
        # write the file here
-        self.create_file
-        super(TempfileDataProvider, self).__init__(self.tmp_file, **kwargs)
+        # self.create_file
+        # super(TempfileDataProvider, self).__init__(self.tmp_file, **kwargs)

    def create_file(self):
        self.tmp_file = tempfile.NamedTemporaryFile()
@@ -73,6 +73,9 @@ class Registry(object):
        self.datatype_elems = []
        self.sniffer_elems = []
        self.xml_filename = None
+        self._edam_formats_mapping = None
+        self._edam_data_mapping = None
+        self._converters_by_datatype = {}
        # Build sites
        self.build_sites = {}
        self.display_sites = {}
@@ -795,16 +798,18 @@ class Registry(object):

    def get_converters_by_datatype(self, ext):
        """Returns available converters by source type"""
-        converters = odict()
-        source_datatype = type(self.get_datatype_by_extension(ext))
-        for ext2, converters_dict in self.datatype_converters.items():
-            converter_datatype = type(self.get_datatype_by_extension(ext2))
-            if issubclass(source_datatype, converter_datatype):
-                converters.update(converters_dict)
-        # Ensure ext-level converters are present
-        if ext in self.datatype_converters.keys():
-            converters.update(self.datatype_converters[ext])
-        return converters
+        if ext not in self._converters_by_datatype:
+            converters = odict()
+            source_datatype = type(self.get_datatype_by_extension(ext))
+            for ext2, converters_dict in self.datatype_converters.items():
+                converter_datatype = type(self.get_datatype_by_extension(ext2))
+                if issubclass(source_datatype, converter_datatype):
+                    converters.update(converters_dict)
+            # Ensure ext-level converters are present
+            if ext in self.datatype_converters.keys():
+                converters.update(self.datatype_converters[ext])
+            self._converters_by_datatype[ext] = converters
+        return self._converters_by_datatype[ext]

    def get_converter_by_target_type(self, source_ext, target_ext):
        """Returns a converter based on source and target datatypes"""
@@ -853,15 +858,17 @@ class Registry(object):
    def edam_formats(self):
        """
        """
-        mapping = dict((k, v.edam_format) for k, v in self.datatypes_by_extension.items())
-        return mapping
+        if not self._edam_formats_mapping:
+            self._edam_formats_mapping = dict((k, v.edam_format) for k, v in self.datatypes_by_extension.items())
+        return self._edam_formats_mapping

    @property
    def edam_data(self):
        """
        """
-        mapping = dict((k, v.edam_data) for k, v in self.datatypes_by_extension.items())
-        return mapping
+        if not self._edam_data_mapping:
+            self._edam_data_mapping = dict((k, v.edam_data) for k, v in self.datatypes_by_extension.items())
+        return self._edam_data_mapping

    @property
    def integrated_datatypes_configs(self):
@@ -3,7 +3,6 @@ File format detector
 """
 from __future__ import absolute_import

-import bz2
 import codecs
 import gzip
 import logging
@@ -30,6 +29,11 @@ from galaxy.util.checkers import (
    is_gzip
 )

+if sys.version_info < (3, 3):
+    import bz2file as bz2
+else:
+    import bz2
+
 log = logging.getLogger(__name__)


@@ -386,6 +390,9 @@ def guess_ext(fname, sniff_order, is_multi_byte=False):
    >>> fname = get_test_fname('1.xls')
    >>> guess_ext(fname, sniff_order)
    'excel.xls'
+    >>> fname = get_test_fname('biom2_sparse_otu_table_hdf5.biom')
+    >>> guess_ext(fname, sniff_order)
+    'biom2'
    """
    file_ext = None
    for datatype in sniff_order:
@@ -170,6 +170,7 @@ class Biom1(Json):
        http://biom-format.org/documentation/format_versions/biom-1.0.html
    """
    file_ext = "biom1"
+    edam_format = "format_3746"

    MetadataElement(name="table_rows", default=[], desc="table_rows", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[])
    MetadataElement(name="table_matrix_element_type", default="", desc="table_matrix_element_type", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value="")
@@ -82,6 +82,9 @@ class ConditionalDependencies(object):
    def check_python_openid(self):
        return asbool(self.config["enable_openid"])

+    def check_chronos_python(self):
+        return "galaxy.jobs.runners.chronos:ChronosJobRunner" in self.job_runners
+
    def check_fluent_logger(self):
        return asbool(self.config["fluent_log"])

@@ -14,5 +14,8 @@ azure-storage==0.32.0
 # PyRods not in PyPI
 python-ldap==2.4.27

+# Chronos client
+chronos-python==0.38.0
+
 # Synnefo / Pithos+ object store client
 kamaki
@@ -13,6 +13,7 @@ uWSGI==2.0.15
 #python_lzo==1.8

 # pure Python packages
+bz2file==0.98; python_version < '3.3'
 Paste==2.0.2
 PasteDeploy==1.5.2
 docutils==0.12
@@ -32,6 +33,7 @@ six==1.10.0
 Whoosh==2.7.4
 testfixtures==4.10.0
 galaxy_sequence_utils==1.0.2
+h5py==2.7.1

 # pykwalify and dependencies
 pykwalify==1.5.1
@@ -76,8 +78,5 @@ ecdsa==0.13
 # Flexible BAM index naming
 pysam==0.8.4+gx5

-# Chronos client
-chronos-python==0.38.0
-
 # GenomeSpace dependencies
-python-genomespaceclient==0.1.8
+python-genomespaceclient==0.1.8
@@ -13,6 +13,7 @@ pycrypto
 #python_lzo

 # pure Python packages
+bz2file; python_version < '3.3'
 Paste
 PasteDeploy
 docutils
@@ -3,6 +3,7 @@ Job control via a command line interface (e.g. qsub/qstat), possibly over a remo
 """

 import logging
+import time

 from galaxy import model
 from galaxy.jobs import JobDestination
@@ -19,6 +20,7 @@ log = logging.getLogger(__name__)
 __all__ = ('ShellJobRunner', )

 DEFAULT_EMBED_METADATA_IN_JOB = True
+MAX_SUBMIT_RETRY = 3


 class ShellJobRunner(AsynchronousJobRunner):
@@ -94,15 +96,13 @@ class ShellJobRunner(AsynchronousJobRunner):

        log.debug("(%s) submitting file: %s" % (galaxy_id_tag, ajs.job_file))

-        cmd_out = shell.execute(job_interface.submit(ajs.job_file))
-        if cmd_out.returncode != 0:
-            log.error('(%s) submission failed (stdout): %s' % (galaxy_id_tag, cmd_out.stdout))
-            log.error('(%s) submission failed (stderr): %s' % (galaxy_id_tag, cmd_out.stderr))
+        returncode, stdout = self.submit(shell, job_interface, ajs.job_file, galaxy_id_tag, retry=MAX_SUBMIT_RETRY)
+        if returncode != 0:
            job_wrapper.fail("failure submitting job")
            return
        # Some job runners return something like 'Submitted batch job XXXX'
        # Strip and split to get job ID.
-        external_job_id = cmd_out.stdout.strip().split()[-1]
+        external_job_id = stdout.strip().split()[-1]
        if not external_job_id:
            log.error('(%s) submission did not return a job identifier, failing job' % galaxy_id_tag)
            job_wrapper.fail("failure submitting job")
@@ -121,6 +121,28 @@ class ShellJobRunner(AsynchronousJobRunner):
        # Add to our 'queue' of jobs to monitor
        self.monitor_queue.put(ajs)

+    def submit(self, shell, job_interface, job_file, galaxy_id_tag, retry=MAX_SUBMIT_RETRY, timeout=10):
+        """
+        Handles actual job script submission.
+
+        If submission fails will retry `retry` time with a timeout of `timeout` seconds.
+        Retuns the returncode of the submission and the stdout, which contains the external job_id.
+        """
+        cmd_out = shell.execute(job_interface.submit(job_file))
+        if cmd_out.returncode == 0:
+            return cmd_out.returncode, cmd_out.stdout
+        stdout = '(%s) submission failed (stdout): %s' % (galaxy_id_tag, cmd_out.stdout)
+        stderr = '(%s) submission failed (stderr): %s' % (galaxy_id_tag, cmd_out.stderr)
+        if retry > 0:
+            log.debug("%s, retrying in %s seconds", stdout, timeout)
+            log.debug("%s, retrying in %s seconds", stderr, timeout)
+            time.sleep(timeout)
+            return self.submit(shell, job_interface, job_file, galaxy_id_tag, retry=retry - 1, timeout=timeout)
+        else:
+            log.error(stdout)
+            log.error(stderr)
+            return cmd_out.returncode, cmd_out.stdout
+
    def check_watched_items(self):
        """
        Called by the monitor thread to look at each watched job and deal
@@ -1,4 +1,3 @@
-import re
 from ast import (
    Module,
    parse,
@@ -23,9 +22,6 @@ VALID_FUNCTIONS = BUILTIN_AND_MATH_FUNCTIONS + STRING_AND_LIST_METHODS
 def _check_name(ast_node, allowed_variables=[]):
    name = ast_node.id
    return name in (VALID_FUNCTIONS + allowed_variables)
-    if re.match(r'^c\d+$', name):
-        return True
-    return name in VALID_FUNCTIONS


 def _check_attribute(ast_node):
@@ -206,15 +206,6 @@ class TaskedJobRunner(BaseJobRunner):
    def _check_pid(self, pid):
        # DBTODO Need to check all subtask pids and return some sort of cumulative result.
        return True
-        try:
-            os.kill(pid, 0)
-            return True
-        except OSError as e:
-            if e.errno == errno.ESRCH:
-                log.debug("_check_pid(): PID %d is dead" % pid)
-            else:
-                log.warning("_check_pid(): Got errno %s when attempting to check PID %d: %s" % (errno.errorcode[e.errno], pid, e.strerror))
-            return False

    def _stop_pid(self, pid, job_id):
        """
@@ -1,4 +1,5 @@
 import logging
+import time

 import paramiko

@@ -57,6 +58,9 @@ class ParamikoShell(object):
        self.timeout = int(timeout) if timeout else timeout
        self.ssh = paramiko.SSHClient()
        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        self.connect()
+
+    def connect(self):
        self.ssh.connect(hostname=self.hostname,
                         port=self.port,
                         username=self.username,
@@ -65,10 +69,19 @@ class ParamikoShell(object):
                         timeout=self.timeout)

    def execute(self, cmd, timeout=60):
-        _, stdout, stderr = self.ssh.exec_command(cmd, timeout=timeout)
+        try:
+            _, stdout, stderr = self._execute(cmd, timeout)
+        except paramiko.SSHException as e:
+            log.error(e)
+            time.sleep(10)
+            self.connect()
+            _, stdout, stderr = self._execute(cmd, timeout)
        return_code = stdout.channel.recv_exit_status()
        return Bunch(stdout=stdout.read(), stderr=stderr.read(), returncode=return_code)

+    def _execute(self, cmd, timeout):
+        return self.ssh.exec_command(cmd, timeout=timeout)
+

 class GlobusSecureShell(SecureShell):

@@ -150,8 +150,8 @@ class LibraryManager(object):
        :param  check_accessible:        flag whether to check that user can access library
        :type   check_accessible:        bool

-        :returns:   the original folder
-        :rtype:     LibraryFolder
+        :returns:   the original library
+        :rtype:     Library
        """
        # all libraries are accessible to an admin
        if trans.user_is_admin():
@@ -0,0 +1,231 @@
+"""Manager and Serializer for library datasets."""
+import logging
+
+from galaxy import util
+
+from galaxy.exceptions import InternalServerError
+from galaxy.exceptions import InsufficientPermissionsException
+from galaxy.exceptions import ObjectNotFound
+from galaxy.exceptions import RequestParameterInvalidException
+from galaxy.managers import tags
+from galaxy.util import validation
+
+log = logging.getLogger(__name__)
+
+
+class LibraryDatasetsManager(object):
+    """Interface/service object for interacting with library datasets."""
+
+    def __init__(self, app):
+        self.app = app
+        self.tag_manager = tags.GalaxyTagManager(app.model.context)
+
+    def get(self, trans, decoded_library_dataset_id, check_accessible=True):
+        """
+        Get the library dataset from the DB.
+
+        :param  decoded_library_dataset_id: decoded library dataset id
+        :type   decoded_library_dataset_id: int
+        :param  check_accessible:           flag whether to check that user can access item
+        :type   check_accessible:           bool
+
+        :returns:   the requested library dataset
+        :rtype:     galaxy.model.LibraryDataset
+        """
+        try:
+            ld = trans.sa_session.query(trans.app.model.LibraryDataset).filter(trans.app.model.LibraryDataset.table.c.id == decoded_library_dataset_id).one()
+        except Exception as e:
+            raise InternalServerError('Error loading from the database.' + str(e))
+        ld = self.secure(trans, ld, check_accessible)
+        return ld
+
+    def update(self, trans, ld, payload):
+        """
+        Update the given library dataset - the latest linked ldda.
+        Updating older lddas (versions) is not allowed.
+
+        :param  ld:                 library dataset to change
+        :type   ld:                 LibraryDataset
+        :param  payload:            dictionary structure containing::
+            :param name:            new ld's name, must be longer than 0
+            :type  name:            str
+            :param misc_info:       new ld's misc info
+            :type  misc_info:       str
+            :param file_ext:        new ld's extension, must exist in the Galaxy registry
+            :type  file_ext:        str
+            :param genome_build:    new ld's genome build
+            :type  genome_build:    str
+        :type   payload: dict
+
+        :returns:   the changed library dataset
+        :rtype:     galaxy.model.LibraryDataset
+        """
+        self.check_modifiable(trans, ld)
+        # we are going to operate on the actual latest ldda
+        ldda = ld.library_dataset_dataset_association
+        payload = self._validate_and_parse_update_payload(payload)
+        self._set_from_dict(trans, ldda, payload)
+        return ld
+
+    def _set_from_dict(self, trans, ldda, new_data):
+        changed = False
+        new_name = new_data.get('name', None)
+        if new_name is not None and new_name != ldda.name:
+            ldda.name = new_name
+            changed = True
+        new_misc_info = new_data.get('misc_info', None)
+        if new_misc_info is not None and new_misc_info != ldda.info:
+            ldda.info = new_misc_info
+            changed = True
+        new_file_ext = new_data.get('file_ext', None)
+        if new_file_ext is not None and new_file_ext != ldda.extension:
+            ldda.extension = new_file_ext
+            # TODO trigger set metadata here
+            changed = True
+        new_genome_build = new_data.get('genome_build', None)
+        if new_genome_build is not None and new_genome_build != ldda.dbkey:
+            ldda.dbkey = new_genome_build
+            changed = True
+        if changed:
+            trans.sa_session.add(ldda)
+            trans.sa_session.flush()
+        return changed
+
+    def _validate_and_parse_update_payload(self, payload):
+        MINIMUM_STRING_LENGTH = 1
+        validated_payload = {}
+        for key, val in payload.items():
+            if val is None:
+                continue
+            if key in ('name'):
+                if len(val) < MINIMUM_STRING_LENGTH:
+                    raise RequestParameterInvalidException('%s must have at least length of %s' % (key, MINIMUM_STRING_LENGTH))
+                val = validation.validate_and_sanitize_basestring(key, val)
+                validated_payload[key] = val
+            if key in ('misc_info'):
+                val = validation.validate_and_sanitize_basestring(key, val)
+                validated_payload[key] = val
+            if key in ('file_ext'):
+                datatype = self.app.datatypes_registry.get_datatype_by_extension(val)
+                if datatype is None:
+                    raise RequestParameterInvalidException('This Galaxy does not recognize the datatype of: %s' % (val))
+                validated_payload[key] = val
+            if key in ('genome_build'):
+                if len(val) < MINIMUM_STRING_LENGTH:
+                    raise RequestParameterInvalidException('%s must have at least length of %s' % (key, MINIMUM_STRING_LENGTH))
+                val = validation.validate_and_sanitize_basestring(key, val)
+                validated_payload[key] = val
+        return validated_payload
+
+    def secure(self, trans, ld, check_accessible=True, check_ownership=False):
+        """
+        Check if library dataset is accessible to current user or the user is an admin.
+
+        :param  ld:         library dataset
+        :type   ld:         galaxy.model.LibraryDataset
+        :param  check_accessible:        flag whether to check that user can access library dataset
+        :type   check_accessible:        bool
+
+        :returns:   the original library dataset
+        :rtype:     galaxy.model.LibraryDataset
+        """
+        if trans.user_is_admin():
+            # all operations are available to an admin
+            return ld
+        if check_accessible:
+            ld = self.check_accessible(trans, ld)
+        return ld
+
+    def check_accessible(self, trans, ld):
+        """
+        Check whether the current user has permissions to access library dataset.
+
+        :param  ld: library dataset
+        :type   ld: galaxy.model.LibraryDataset
+
+        :returns:   the original library dataset
+        :rtype:     galaxy.model.LibraryDataset
+
+        :raises:    ObjectNotFound
+        """
+        if not trans.app.security_agent.can_access_library_item(trans.get_current_user_roles(), ld, trans.user):
+            raise ObjectNotFound('Library dataset with the id provided was not found.')
+        elif ld.deleted:
+            raise ObjectNotFound('Library dataset with the id provided is deleted.')
+        else:
+            return ld
+
+    def check_modifiable(self, trans, ld):
+        """
+        Check whether the current user has permissions to modify library dataset.
+
+        :param  ld: library dataset
+        :type   ld: galaxy.model.LibraryDataset
+
+        :returns:   the original library dataset
+        :rtype:     galaxy.model.LibraryDataset
+
+        :raises:    ObjectNotFound
+        """
+        if ld.deleted:
+            raise ObjectNotFound('Library dataset with the id provided is deleted.')
+        elif trans.user_is_admin():
+            return ld
+        if not trans.app.security_agent.can_modify_library_item(trans.get_current_user_roles(), ld):
+            raise InsufficientPermissionsException('You do not have proper permission to modify this library dataset.')
+        else:
+            return ld
+
+    def serialize(self, trans, ld):
+        """Serialize the library dataset into a dictionary."""
+        current_user_roles = trans.get_current_user_roles()
+
+        # Build the full path for breadcrumb purposes.
+        full_path = self._build_path(trans, ld.folder)
+        dataset_item = (trans.security.encode_id(ld.id), ld.name)
+        full_path.insert(0, dataset_item)
+        full_path = full_path[::-1]
+
+        # Find expired versions of the library dataset
+        expired_ldda_versions = []
+        for expired_ldda in ld.expired_datasets:
+            expired_ldda_versions.append((trans.security.encode_id(expired_ldda.id), expired_ldda.name))
+
+        rval = trans.security.encode_all_ids(ld.to_dict())
+        if len(expired_ldda_versions) > 0:
+            rval['has_versions'] = True
+            rval['expired_versions'] = expired_ldda_versions
+        rval['deleted'] = ld.deleted
+        rval['folder_id'] = 'F' + rval['folder_id']
+        rval['full_path'] = full_path
+        rval['file_size'] = util.nice_size(int(ld.library_dataset_dataset_association.get_size()))
+        rval['date_uploaded'] = ld.library_dataset_dataset_association.create_time.strftime("%Y-%m-%d %I:%M %p")
+        rval['can_user_modify'] = trans.app.security_agent.can_modify_library_item(current_user_roles, ld) or trans.user_is_admin()
+        rval['is_unrestricted'] = trans.app.security_agent.dataset_is_public(ld.library_dataset_dataset_association.dataset)
+        rval['tags'] = self.tag_manager.get_tags_str(ld.library_dataset_dataset_association.tags)
+
+        #  Manage dataset permission is always attached to the dataset itself, not the the ld or ldda to maintain consistency
+        rval['can_user_manage'] = trans.app.security_agent.can_manage_dataset(current_user_roles, ld.library_dataset_dataset_association.dataset) or trans.user_is_admin()
+        return rval
+
+    def _build_path(self, trans, folder):
+        """
+        Search the path upwards recursively and load the whole route of
+        names and ids for breadcrumb building purposes.
+
+        :param folder: current folder for navigating up
+        :param type:   Galaxy LibraryFolder
+
+        :returns:   list consisting of full path to the library
+        :type:      list
+        """
+        path_to_root = []
+        if folder.parent_id is None:
+            # We are almost in root
+            path_to_root.append(('F' + trans.security.encode_id(folder.id), folder.name))
+        else:
+            # We add the current folder and traverse up one folder.
+            path_to_root.append(('F' + trans.security.encode_id(folder.id), folder.name))
+            upper_folder = trans.sa_session.query(trans.app.model.LibraryFolder).get(folder.parent_id)
+            path_to_root.extend(self._build_path(trans, upper_folder))
+        return path_to_root
@@ -8,6 +8,7 @@ import json
 import uuid

 from sqlalchemy import and_
+from sqlalchemy.orm import joinedload, subqueryload

 from galaxy import model
 from galaxy import util
@@ -30,7 +31,7 @@ log = logging.getLogger(__name__)


 class WorkflowsManager(object):
-    """ Handle CRUD type operaitons related to workflows. More interesting
+    """ Handle CRUD type operations related to workflows. More interesting
    stuff regarding workflow execution, step sorting, etc... can be found in
    the galaxy.workflow module.
    """
@@ -45,16 +46,17 @@ class WorkflowsManager(object):
        if util.is_uuid(workflow_id):
            # see if they have passed in the UUID for a workflow that is attached to a stored workflow
            workflow_uuid = uuid.UUID(workflow_id)
-            stored_workflow = trans.sa_session.query(trans.app.model.StoredWorkflow).filter(and_(
+            workflow_query = trans.sa_session.query(trans.app.model.StoredWorkflow).filter(and_(
                trans.app.model.StoredWorkflow.latest_workflow_id == trans.app.model.Workflow.id,
                trans.app.model.Workflow.uuid == workflow_uuid
-            )).first()
-            if stored_workflow is None:
-                raise exceptions.ObjectNotFound("Workflow not found: %s" % workflow_id)
+            ))
        else:
            workflow_id = decode_id(self.app, workflow_id)
-            query = trans.sa_session.query(trans.app.model.StoredWorkflow)
-            stored_workflow = query.get(workflow_id)
+            workflow_query = trans.sa_session.query(trans.app.model.StoredWorkflow).\
+                filter(trans.app.model.StoredWorkflow.id == workflow_id)
+        stored_workflow = workflow_query.options(joinedload('annotations'),
+                                                 joinedload('tags'),
+                                                 subqueryload('workflows').joinedload('steps').joinedload('*')).first()
        if stored_workflow is None:
            raise exceptions.ObjectNotFound("No such workflow found.")
        return stored_workflow
@@ -203,7 +205,7 @@ class WorkflowContentsManager(UsesAnnotations):
        exact_tools=False,
    ):
        # Put parameters in workflow mode
-        trans.workflow_building_mode = True
+        trans.workflow_building_mode = workflow_building_modes.ENABLED
        # If there's a source, put it in the workflow name.
        if source and source != 'API':
            name = "%s (imported from %s)" % (data['name'], source)
@@ -229,6 +231,8 @@ class WorkflowContentsManager(UsesAnnotations):
            if data['annotation']:
                annotation = sanitize_html(data['annotation'], 'utf-8', 'text/html')
                self.add_item_annotation(trans.sa_session, stored.user, stored, annotation)
+            workflow_tags = data.get('tags', [])
+            trans.app.tag_handler.set_tags_from_list(user=trans.user, item=stored, new_tags_list=workflow_tags)

            # Persist
            trans.sa_session.add(stored)
@@ -255,7 +259,7 @@ class WorkflowContentsManager(UsesAnnotations):

    def update_workflow_from_dict(self, trans, stored_workflow, workflow_data):
        # Put parameters in workflow mode
-        trans.workflow_building_mode = True
+        trans.workflow_building_mode = workflow_building_modes.ENABLED

        workflow, missing_tool_tups = self._workflow_from_dict(
            trans,
@@ -449,10 +453,7 @@ class WorkflowContentsManager(UsesAnnotations):
                else:
                    data['upgrade_messages'][step.order_index] = {module.tool.name: "\n".join(module.version_changes)}
            # Get user annotation.
-            step_annotation = self.get_item_annotation_obj(trans.sa_session, trans.user, step)
-            annotation_str = ""
-            if step_annotation:
-                annotation_str = step_annotation.annotation
+            annotation_str = self.get_item_annotation_str(trans.sa_session, trans.user, step) or ''
            config_form = None
            if trans.history:
                # If in a web session, attach form html. No reason to do
@@ -547,16 +548,17 @@ class WorkflowContentsManager(UsesAnnotations):
            workflow = stored.latest_workflow

        annotation_str = ""
+        tag_str = ""
        if stored is not None:
-            workflow_annotation = self.get_item_annotation_obj(trans.sa_session, trans.user, stored)
-            if workflow_annotation:
-                annotation_str = workflow_annotation.annotation
+            annotation_str = self.get_item_annotation_str(trans.sa_session, trans.user, stored) or ''
+            tag_str = stored.make_tag_string_list()
        # Pack workflow data into a dictionary and return
        data = {}
        data['a_galaxy_workflow'] = 'true'  # Placeholder for identifying galaxy workflow
        data['format-version'] = "0.1"
        data['name'] = workflow.name
        data['annotation'] = annotation_str
+        data['tags'] = tag_str
        if workflow.uuid is not None:
            data['uuid'] = str(workflow.uuid)
        data['steps'] = {}
@@ -567,10 +569,7 @@ class WorkflowContentsManager(UsesAnnotations):
            if not module:
                return None
            # Get user annotation.
-            step_annotation = self.get_item_annotation_obj(trans.sa_session, trans.user, step)
-            annotation_str = ""
-            if step_annotation:
-                annotation_str = step_annotation.annotation
+            annotation_str = self.get_item_annotation_str(trans.sa_session, trans.user, step) or ''
            content_id = module.get_content_id()
            # Export differences for backward compatibility
            if module.type == 'tool':
@@ -594,13 +593,12 @@ class WorkflowContentsManager(UsesAnnotations):
            }
            # Add tool shed repository information and post-job actions to step dict.
            if module.type == 'tool':
-                if module.tool.tool_shed_repository:
-                    tsr = module.tool.tool_shed_repository
+                if module.tool and module.tool.tool_shed:
                    step_dict["tool_shed_repository"] = {
-                        'name': tsr.name,
-                        'owner': tsr.owner,
-                        'changeset_revision': tsr.changeset_revision,
-                        'tool_shed': tsr.tool_shed
+                        'name': module.tool.repository_name,
+                        'owner': module.tool.repository_owner,
+                        'changeset_revision': module.tool.changeset_revision,
+                        'tool_shed': module.tool.tool_shed
                    }
                pja_dict = {}
                for pja in step.post_job_actions:
@@ -668,10 +666,12 @@ class WorkflowContentsManager(UsesAnnotations):
                        data_input_names[prefixed_name] = True
                # FIXME: this updates modules silently right now; messages from updates should be provided.
                module.check_and_update_state()
-                visit_input_values(module.tool.inputs, module.state.inputs, callback)
-                # Filter
-                # FIXME: this removes connection without displaying a message currently!
-                input_connections = [conn for conn in input_connections if (conn.input_name in data_input_names or conn.non_data_connection)]
+                if module.tool:
+                    # If the tool is installed we attempt to verify input values
+                    # and connections, otherwise the last known state will be dumped without modifications.
+                    visit_input_values(module.tool.inputs, module.state.inputs, callback)
+                    # FIXME: this removes connection without displaying a message currently!
+                    input_connections = [conn for conn in input_connections if (conn.input_name in data_input_names or conn.non_data_connection)]

            # Encode input connections as dictionary
            input_conn_dict = {}
@@ -743,7 +743,6 @@ class WorkflowContentsManager(UsesAnnotations):
        for step in workflow.steps:
            steps_to_order_index[step.id] = step.order_index
        for step in workflow.steps:
-            step_uuid = str(step.uuid) if step.uuid else None
            step_id = step.id if legacy else step.order_index
            step_type = step.type
            step_dict = {'id': step_id,
@@ -100,7 +100,15 @@ class UsesAnnotations:

    def get_item_annotation_str(self, db_session, user, item):
        """ Returns a user's annotation string for an item. """
-        annotation_obj = self.get_item_annotation_obj(db_session, user, item)
+        if hasattr(item, 'annotations'):
+            # If we already have an annotations object we use it.
+            annotation_obj = None
+            for annotation in item.annotations:
+                if annotation.user == user:
+                    annotation_obj = annotation
+                    break
+        else:
+            annotation_obj = self.get_item_annotation_obj(db_session, user, item)
        if annotation_obj:
            return galaxy.util.unicodify(annotation_obj.annotation)
        return None
@@ -781,7 +781,6 @@ class JobExternalOutputMetadataWrapper(object):
            config_root = os.path.abspath(os.getcwd())
        if datatypes_config is None:
            raise Exception('In setup_external_metadata, the received datatypes_config is None.')
-            datatypes_config = 'datatypes_conf.xml'
        metadata_files_list = []
        for dataset in datasets:
            key = self.get_dataset_metadata_key(dataset)
@@ -427,8 +427,6 @@ class AzureBlobObjectStore(ObjectStore):
        #     return cache_path
        raise ObjectNotFound('objectstore.get_filename, no cache_path: %s, kwargs: %s' % (str(obj), str(kwargs)))

-        return cache_path  # Until the upload tool does not explicitly create the dataset, return expected path
-
    def update_from_file(self, obj, file_name=None, create=False, **kwargs):
        if create is True:
            self.create(obj, **kwargs)
@@ -63,7 +63,6 @@ class PulsarObjectStore(ObjectStore):
    def __build_kwds(self, obj, **kwds):
        kwds['object_id'] = obj.id
        return kwds
-        pass

    def __build_pulsar_client(self, config_xml):
        if ObjectStoreClientManager is None:
@@ -53,7 +53,8 @@ from galaxy.tools.parameters.basic import (
    DataToolParameter,
    HiddenToolParameter,
    SelectToolParameter,
-    ToolParameter
+    ToolParameter,
+    workflow_building_modes,
 )
 from galaxy.tools.parameters.grouping import Conditional, ConditionalWhen, Repeat, Section, UploadDataset
 from galaxy.tools.parameters.input_translation import ToolInputTranslator
@@ -419,6 +420,7 @@ class Tool(object, Dictifiable):
        self.repository_owner = None
        self.changeset_revision = None
        self.installed_changeset_revision = None
+        self.sharable_url = None
        # The tool.id value will be the value of guid, but we'll keep the
        # guid attribute since it is useful to have.
        self.guid = guid
@@ -472,18 +474,6 @@ class Tool(object, Dictifiable):
                                                            installed_changeset_revision=self.installed_changeset_revision)
        return None

-    @property
-    def produces_collections_of_unknown_type(self):
-
-        def output_is_dynamic_collection(output):
-            if not output.collection:
-                return False
-            if output.structure.collection_type:
-                return False
-            return True
-
-        return any(map(output_is_dynamic_collection, self.outputs.values()))
-
    @property
    def produces_collections_with_unknown_structure(self):

@@ -1103,6 +1093,7 @@ class Tool(object, Dictifiable):
                    self.repository_owner = tool_shed_repository.owner
                    self.changeset_revision = tool_shed_repository.changeset_revision
                    self.installed_changeset_revision = tool_shed_repository.installed_changeset_revision
+                    self.sharable_url = tool_shed_repository.get_sharable_url(self.app)

    @property
    def help(self):
@@ -1192,10 +1183,6 @@ class Tool(object, Dictifiable):
        if self.tool_type.startswith('data_source'):
            return False

-        if self.produces_collections_of_unknown_type:
-            # Getting there...
-            return False
-
        if hasattr(tool_source, "root"):
            root = tool_source.root
            if not string_as_bool(root.get("workflow_compatible", "True")):
@@ -1802,17 +1789,19 @@ class Tool(object, Dictifiable):
        """
        history_id = kwd.get('history_id', None)
        history = None
-        try:
-            if history_id is not None:
-                history = self.history_manager.get_owned(trans.security.decode_id(history_id), trans.user, current_history=trans.history)
-            else:
-                history = trans.get_history()
-            if history is None and job is not None:
-                history = self.history_manager.get_owned(job.history.id, trans.user, current_history=trans.history)
-            if history is None:
-                raise exceptions.MessageException('History unavailable. Please specify a valid history id')
-        except Exception as e:
-            raise exceptions.MessageException('[history_id=%s] Failed to retrieve history. %s.' % (history_id, str(e)))
+        if workflow_building_mode is workflow_building_modes.USE_HISTORY or workflow_building_mode is workflow_building_modes.DISABLED:
+            # We don't need a history when exporting a workflow for the workflow editor or when downloading a workflow
+            try:
+                if history_id is not None:
+                    history = self.history_manager.get_owned(trans.security.decode_id(history_id), trans.user, current_history=trans.history)
+                else:
+                    history = trans.get_history()
+                if history is None and job is not None:
+                    history = self.history_manager.get_owned(job.history.id, trans.user, current_history=trans.history)
+                if history is None:
+                    raise exceptions.MessageException('History unavailable. Please specify a valid history id')
+            except Exception as e:
+                raise exceptions.MessageException('[history_id=%s] Failed to retrieve history. %s.' % (history_id, str(e)))

        # build request context
        request_context = WorkRequestContext(app=trans.app, user=trans.user, history=history, workflow_building_mode=workflow_building_mode)
@@ -1868,7 +1857,7 @@ class Tool(object, Dictifiable):
            'help'          : tool_help,
            'citations'     : bool(self.citations),
            'biostar_url'   : self.app.config.biostar_url,
-            'sharable_url'  : self.tool_shed_repository.get_sharable_url(self.app) if self.tool_shed_repository else None,
+            'sharable_url'  : self.sharable_url,
            'message'       : tool_message,
            'warnings'      : tool_warnings,
            'versions'      : tool_versions,
@@ -1877,7 +1866,7 @@ class Tool(object, Dictifiable):
            'state_inputs'  : params_to_strings(self.inputs, state_inputs, self.app),
            'job_id'        : trans.security.encode_id(job.id) if job else None,
            'job_remap'     : self._get_job_remap(job),
-            'history_id'    : trans.security.encode_id(history.id),
+            'history_id'    : trans.security.encode_id(history.id) if history else None,
            'display'       : self.display_interface,
            'action'        : url_for(self.action),
            'method'        : self.method,
@@ -2018,7 +2007,7 @@ class Tool(object, Dictifiable):
                        else:
                            message += 'You can re-run the job with this tool version, which is a different version of the original tool.'
                else:
-                    new_tool_shed_url = '%s/%s/' % (tool.tool_shed_repository.get_sharable_url(tool.app), tool.tool_shed_repository.changeset_revision)
+                    new_tool_shed_url = '%s/%s/' % (tool.sharable_url, tool.changeset_revision)
                    old_tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(self.app, tool_id.split('/repos/')[0])
                    old_tool_shed_url = '%s/view/%s/%s/' % (old_tool_shed_url, tool.repository_owner, tool.repository_name)
                    message = 'This job was run with <a href=\"%s\" target=\"_blank\">tool id \"%s\"</a>, version "%s", which is not available.  ' % (old_tool_shed_url, tool_id, tool_version)
@@ -19,7 +19,7 @@ class UploadToolAction(ToolAction):

        persisting_uploads_timer = ExecutionTimer()
        precreated_datasets = upload_common.get_precreated_datasets(trans, incoming, trans.app.model.HistoryDatasetAssociation)
-        incoming = upload_common.persist_uploads(incoming)
+        incoming = upload_common.persist_uploads(incoming, trans)
        log.debug("Persisted uploads %s" % persisting_uploads_timer)
        # We can pass an empty string as the cntrller here since it is used to check whether we
        # are in an admin view, and this tool is currently not used there.
@@ -1,6 +1,8 @@
+import ipaddress
 import logging
 import os
 import shlex
+import socket
 import subprocess
 import tempfile
 from cgi import FieldStorage
@@ -8,16 +10,103 @@ from json import dumps

 from six import StringIO
 from sqlalchemy.orm import eagerload_all
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse

 from galaxy import datatypes, util
 from galaxy.exceptions import ObjectInvalid
 from galaxy.managers import tags
+from galaxy.util import unicodify
 from galaxy.util.odict import odict

 log = logging.getLogger(__name__)


-def persist_uploads(params):
+def validate_url(url, ip_whitelist):
+    # If it doesn't look like a URL, ignore it.
+    if not (url.lstrip().startswith('http://') or url.lstrip().startswith('https://')):
+        return url
+
+    # Extract hostname component
+    parsed_url = urlparse(url).netloc
+    # If credentials are in this URL, we need to strip those.
+    if parsed_url.count('@') > 0:
+        # credentials.
+        parsed_url = parsed_url[parsed_url.rindex('@') + 1:]
+    # Percent encoded colons and other characters will not be resolved as such
+    # so we don't have to either.
+
+    # Sometimes the netloc will contain the port which is not desired, so we
+    # need to extract that.
+    port = None
+    # However, it could ALSO be an IPv6 address they've supplied.
+    if ':' in parsed_url:
+        # IPv6 addresses have colons in them already (it seems like always more than two)
+        if parsed_url.count(':') >= 2:
+            # Since IPv6 already use colons extensively, they wrap it in
+            # brackets when there is a port, e.g. http://[2001:db8:1f70::999:de8:7648:6e8]:100/
+            # However if it ends with a ']' then there is no port after it and
+            # they've wrapped it in brackets just for fun.
+            if ']' in parsed_url and not parsed_url.endswith(']'):
+                # If this +1 throws a range error, we don't care, their url
+                # shouldn't end with a colon.
+                idx = parsed_url.rindex(':')
+                # We parse as an int and let this fail ungracefully if parsing
+                # fails because we desire to fail closed rather than open.
+                port = int(parsed_url[idx + 1:])
+                parsed_url = parsed_url[:idx]
+            else:
+                # Plain ipv6 without port
+                pass
+        else:
+            # This should finally be ipv4 with port. It cannot be IPv6 as that
+            # was caught by earlier cases, and it cannot be due to credentials.
+            idx = parsed_url.rindex(':')
+            port = int(parsed_url[idx + 1:])
+            parsed_url = parsed_url[:idx]
+
+    # safe to log out, no credentials/request path, just an IP + port
+    log.debug("parsed url, port: %s : %s", parsed_url, port)
+    # Call getaddrinfo to resolve hostname into tuples containing IPs.
+    addrinfo = socket.getaddrinfo(parsed_url, port)
+    # Get the IP addresses that this entry resolves to (uniquely)
+    # We drop:
+    #   AF_* family: It will resolve to AF_INET or AF_INET6, getaddrinfo(3) doesn't even mention AF_UNIX,
+    #   socktype: We don't care if a stream/dgram/raw protocol
+    #   protocol: we don't care if it is tcp or udp.
+    addrinfo_results = set([info[4][0] for info in addrinfo])
+    # There may be multiple (e.g. IPv4 + IPv6 or DNS round robin). Any one of these
+    # could resolve to a local addresses (and could be returned by chance),
+    # therefore we must check them all.
+    for raw_ip in addrinfo_results:
+        # Convert to an IP object so we can tell if it is in private space.
+        ip = ipaddress.ip_address(unicodify(raw_ip))
+        # If this is a private address
+        if ip.is_private:
+            results = []
+            # If this IP is not anywhere in the whitelist
+            for whitelisted in ip_whitelist:
+                # If it's an IP address range (rather than a single one...)
+                if hasattr(whitelisted, 'subnets'):
+                    results.append(ip in whitelisted)
+                else:
+                    results.append(ip == whitelisted)
+
+            if any(results):
+                # If we had any True, then THIS (and ONLY THIS) IP address that
+                # that specific DNS entry resolved to is in whitelisted and
+                # safe to access. But we cannot exit here, we must ensure that
+                # all IPs that that DNS entry resolves to are likewise safe.
+                pass
+            else:
+                # Otherwise, we deny access.
+                raise Exception("Access to this address in not permitted by server configuration")
+    return url
+
+
+def persist_uploads(params, trans):
    """
    Turn any uploads in the submitted form to persisted files.
    """
@@ -35,7 +124,10 @@ def persist_uploads(params):
            elif type(f) == dict and 'local_filename' not in f:
                raise Exception('Uploaded file was encoded in a way not understood by Galaxy.')
            if upload_dataset['url_paste'] and upload_dataset['url_paste'].strip() != '':
-                upload_dataset['url_paste'], is_multi_byte = datatypes.sniff.stream_to_file(StringIO(upload_dataset['url_paste']), prefix="strio_url_paste_")
+                upload_dataset['url_paste'], is_multi_byte = datatypes.sniff.stream_to_file(
+                    StringIO(validate_url(upload_dataset['url_paste'], trans.app.config.fetch_url_whitelist_ips)),
+                    prefix="strio_url_paste_"
+                )
            else:
                upload_dataset['url_paste'] = None
            new_files.append(upload_dataset)
@@ -323,20 +415,26 @@ def create_paramfile(trans, uploaded_datasets):
        else:
            try:
                is_binary = uploaded_dataset.datatype.is_binary
-            except:
+            except Exception:
                is_binary = None
            try:
                link_data_only = uploaded_dataset.link_data_only
-            except:
+            except Exception:
                link_data_only = 'copy_files'
            try:
                uuid_str = uploaded_dataset.uuid
-            except:
+            except Exception:
                uuid_str = None
            try:
                purge_source = uploaded_dataset.purge_source
-            except:
+            except Exception:
                purge_source = True
+            try:
+                user_ftp_dir = os.path.abspath(trans.user_ftp_dir)
+            except Exception:
+                user_ftp_dir = None
+            if user_ftp_dir and uploaded_dataset.path.startswith(user_ftp_dir):
+                uploaded_dataset.type = 'ftp_import'
            json = dict(file_type=uploaded_dataset.file_type,
                        ext=uploaded_dataset.ext,
                        name=uploaded_dataset.name,
@@ -77,7 +77,7 @@ class RequirementSpecification(object):

    @staticmethod
    def from_dict(dict):
-        uri = dict.get["uri"]
+        uri = dict.get("uri")
        version = dict.get("version", None)
        return RequirementSpecification(uri=uri, version=version)

@@ -0,0 +1,183 @@
+"""
+This is a prototype dependency resolver to be able to use the "LMOD environment modules system" from TACC to solve package requirements
+
+LMOD official website: https://www.tacc.utexas.edu/research-development/tacc-projects/lmod
+
+LMOD @ Github: https://github.com/TACC/Lmod
+
+"""
+import logging
+from os import getenv
+from os.path import exists
+from subprocess import PIPE, Popen
+
+from six import StringIO
+
+from ..resolvers import (
+    Dependency,
+    DependencyResolver,
+    MappableDependencyResolver,
+    NullDependency,
+)
+
+log = logging.getLogger(__name__)
+
+DEFAULT_LMOD_PATH = getenv('LMOD_CMD')
+DEFAULT_SETTARG_PATH = getenv('LMOD_SETTARG_CMD')
+DEFAULT_MODULEPATH = getenv('MODULEPATH')
+DEFAULT_MAPPING_FILE = 'config/lmod_modules_mapping.yml'
+INVALID_LMOD_PATH_MSG = "The following LMOD executable could not be found: %s. Either your LMOD Dependency Resolver is misconfigured or LMOD is improperly installed on your system !"
+EMPTY_MODULEPATH_MSG = "No valid LMOD MODULEPATH defined ! Either your LMOD Dependency Resolver is misconfigured or LMOD is improperly installed on your system !"
+
+
+class LmodDependencyResolver(DependencyResolver, MappableDependencyResolver):
+    """Dependency resolver based on the LMOD environment modules system"""
+
+    dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'modulepath']
+    resolver_type = "lmod"
+
+    def __init__(self, dependency_manager, **kwds):
+        # Mapping file management
+        self._set_default_mapping_file(kwds)
+        self._setup_mapping(dependency_manager, **kwds)
+
+        # Other attributes
+        self.versionless = _string_as_bool(kwds.get('versionless', 'false'))
+        self.lmodexec = kwds.get('lmodexec', DEFAULT_LMOD_PATH)
+        self.settargexec = kwds.get('settargexec', DEFAULT_SETTARG_PATH)
+        self.modulepath = kwds.get('modulepath', DEFAULT_MODULEPATH)
+        self.module_checker = AvailModuleChecker(self, self.modulepath)
+
+    def _set_default_mapping_file(self, resolver_attributes):
+        if 'mapping_files' not in resolver_attributes:
+            if exists(DEFAULT_MAPPING_FILE):
+                resolver_attributes['mapping_files'] = DEFAULT_MAPPING_FILE
+
+    def resolve(self, requirement, **kwds):
+        requirement = self._expand_mappings(requirement)
+        name, version, type = requirement.name, requirement.version, requirement.type
+
+        if type != "package":
+            return NullDependency(version=version, name=name)
+
+        if self.__has_module(name, version):
+            return LmodDependency(self, name, version, exact=True)
+        elif self.versionless and self.__has_module(name, None):
+            return LmodDependency(self, name, None, exact=False)
+
+        return NullDependency(version=version, name=name)
+
+    def __has_module(self, name, version):
+        return self.module_checker.has_module(name, version)
+
+
+class AvailModuleChecker(object):
+    """Parses the output of Lmod 'module avail' command to get the list of available modules."""
+
+    def __init__(self, lmod_dependency_resolver, modulepath):
+        self.lmod_dependency_resolver = lmod_dependency_resolver
+        self.modulepath = modulepath
+
+    def has_module(self, module, version):
+        # When version is None (No specific version required by the wrapper -or- versionless is set to 'true'), we only get the list of default modules
+        # We get the full list of modules otherwise
+        if version is None:
+            available_modules = self.__get_list_of_available_modules(True)
+        else:
+            available_modules = self.__get_list_of_available_modules(False)
+
+        # Is the required module in the list of avaialable modules ?
+        for module_name, module_version in available_modules:
+            names_match = module == module_name
+            module_match = names_match and (version is None or module_version == version)
+            if module_match:
+                return True
+        return False
+
+    def __get_list_of_available_modules(self, default_version_only=False):
+        # Get the results of the "module avail" command in an easy to parse format
+        # Note that since "module" is actually a bash function, we are directy executing the underlying executable instead
+        raw_output = self.__get_module_avail_command_output(default_version_only).decode("utf-8")
+
+        # Parse the result
+        for line in StringIO(raw_output):
+            # Clean line and discard non-module lines
+            line = line and line.strip()
+            if not line or line.startswith("/"):
+                continue
+
+            # Split module lines by / to separate the module name from the module version
+            # Module without version are discarded
+            module_parts = line.split('/')
+            if len(module_parts) == 2:
+                yield module_parts[0], module_parts[1]
+
+    def __get_module_avail_command_output(self, default_version_only=False):
+        # Check if the LMOD executable is available (ie. if both LMOD and the lmod dependency resolver are both setup properly)
+        lmodexec = self.lmod_dependency_resolver.lmodexec
+        if not exists(lmodexec):
+            raise Exception(INVALID_LMOD_PATH_MSG % lmodexec)
+
+        # Check if the MODULEPATH environment
+        if self.modulepath == "" or self.modulepath is None:
+            raise Exception(EMPTY_MODULEPATH_MSG)
+
+        # Build command line
+        if default_version_only:
+            module_avail_command = [lmodexec, '-t', '-d', 'avail']
+        else:
+            module_avail_command = [lmodexec, '-t', 'avail']
+
+        # The list of avaialable modules is actually printed on stderr and not stdout for module commands
+        return Popen(module_avail_command, stdout=PIPE, stderr=PIPE, env={'MODULEPATH': self.modulepath}, close_fds=True).communicate()[1]
+
+
+class LmodDependency(Dependency):
+    """Prepare the commands required to solve the dependency and add them to the script used to run a tool in Galaxy."""
+
+    dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['module_name', 'module_version']
+    dependency_type = 'lmod'
+
+    def __init__(self, lmod_dependency_resolver, module_name, module_version=None, exact=True):
+        self.lmod_dependency_resolver = lmod_dependency_resolver
+        self.module_name = module_name
+        self.module_version = module_version
+        self._exact = exact
+
+    @property
+    def name(self):
+        return self.module_name
+
+    @property
+    def version(self):
+        return self.module_version
+
+    @property
+    def exact(self):
+        return self._exact
+
+    def shell_commands(self, requirement):
+        # Get the full module name in the form "tool_name/tool_version"
+        module_to_load = self.module_name
+        if self.module_version:
+            module_to_load = '%s/%s' % (self.module_name, self.module_version)
+
+        # Build the list of command to add to run script
+        # Note that since "module" is actually a bash function, we are directy executing the underlying executable instead
+        # - Set the MODULEPATH environment variable
+        command = 'MODULEPATH=%s; ' % (self.lmod_dependency_resolver.modulepath)
+        command += 'export MODULEPATH; '
+        # - Execute the "module load" command (or rather the "/path/to/lmod load" command)
+        command += 'eval `%s load %s` ' % (self.lmod_dependency_resolver.lmodexec, module_to_load)
+        # - Execute the "settarg" command in addition if needed
+        if self.lmod_dependency_resolver.settargexec is not None:
+            command += '&& eval `%s -s sh`' % (self.lmod_dependency_resolver.settargexec)
+
+        return command
+
+
+def _string_as_bool(value):
+    return str(value).lower() == "true"
+
+
+__all__ = ('LmodDependencyResolver', )
@@ -0,0 +1,64 @@
+"""The module describes the ``influxdb`` error plugin plugin."""
+from __future__ import absolute_import
+
+import datetime
+import logging
+
+from galaxy.util import unicodify
+
+from ..plugins import ErrorPlugin
+
+try:
+    import influxdb
+except ImportError:
+    # This middleware will never be used without influxdb.
+    influxdb = None
+
+log = logging.getLogger(__name__)
+
+
+class InfluxDBPlugin(ErrorPlugin):
+    """Send error report to InfluxDB
+    """
+    plugin_type = "influxdb"
+
+    def __init__(self, **kwargs):
+        if not influxdb:
+            raise ImportError("Could not find InfluxDB Client, this error reporter will be disabled; please pip install influxdb")
+
+        self.app = kwargs['app']
+        # Neither of these matter
+        self.verbose = False
+        self.user_submission = False
+        # Anything with influxdb_ gets sent to the client initialization
+        influx_args = {
+            k[len('influxdb_'):]: v
+            for (k, v)
+            in kwargs.items()
+            if k.startswith('influxdb_')
+        }
+        print(influx_args)
+        self.client = influxdb.InfluxDBClient(**influx_args)
+
+    def submit_report(self, dataset, job, tool, **kwargs):
+        """Submit the error report to sentry
+        """
+        self.client.write_points([{
+            'measurement': 'galaxy_tool_error',
+            'time': datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'),
+            'fields': {
+                'value': 1
+            },
+            'tags': {
+                'exit_code': job.exit_code,
+                'tool_id': unicodify(job.tool_id),
+                'tool_version': unicodify(job.tool_version),
+                'tool_xml': unicodify(tool.config_file) if tool else None,
+                'destination_id': unicodify(job.destination_id),
+                'handler': unicodify(job.handler),
+            }
+        }])
+        return ('Submitted to InfluxDB', 'success')
+
+
+__all__ = ('InfluxDBPlugin', )
@@ -35,6 +35,7 @@ class JsonPlugin(ErrorPlugin):
                'info' : job.info,
                'id' : job.id,
                'command_line' : job.command_line,
+                'destination_id': job.destination_id,
                'stderr' : job.stderr,
                'traceback': job.traceback,
                'exit_code': job.exit_code,
@@ -42,6 +42,7 @@ class SentryPlugin(ErrorPlugin):
                'info': job.info,
                'id': job.id,
                'command_line': unicodify(job.command_line),
+                'destination_id': unicodify(job.destination_id),
                'stderr': unicodify(job.stderr),
                'traceback': unicodify(job.traceback),
                'exit_code': job.exit_code,
@@ -70,6 +70,7 @@ class JobImportHistoryArchiveWrapper(object, UsesAnnotations):
        if jiha:
            try:
                archive_dir = jiha.archive_dir
+                archive_dir = os.path.realpath(archive_dir)
                user = jiha.job.user

                # Bioblend previous to 17.01 exported histories with an extra subdir.
@@ -122,7 +123,7 @@ class JobImportHistoryArchiveWrapper(object, UsesAnnotations):
                datasets_usage_counts = {}
                for dataset_attrs in datasets_attrs:
                    temp_dataset_file_name = \
-                        os.path.abspath(os.path.join(archive_dir, dataset_attrs['file_name']))
+                        os.path.realpath(os.path.join(archive_dir, dataset_attrs['file_name']))
                    if (temp_dataset_file_name not in datasets_usage_counts):
                        datasets_usage_counts[temp_dataset_file_name] = 0
                    datasets_usage_counts[temp_dataset_file_name] += 1
@@ -39,7 +39,7 @@ def rst_invalid(text):
    """
    invalid_rst = False
    try:
-        rst_to_html(text)
+        rst_to_html(text, error=True)
    except Exception as e:
        invalid_rst = str(e)
    return invalid_rst
@@ -835,7 +835,7 @@ class SelectToolParameter(ToolParameter):
        workflow_building_mode = trans.workflow_building_mode
        for context_value in other_values.values():
            if is_runtime_value(context_value):
-                workflow_building_mode = True
+                workflow_building_mode = workflow_building_modes.ENABLED
                break
        if len(list(legal_values)) == 0 and workflow_building_mode:
            if self.multiple:
@@ -1062,18 +1062,18 @@ of ``type`` ``data``.</xs:documentation>
  </xs:complexType>
  <xs:complexType name="TestCompositeData">
    <xs:annotation>
-      <xs:documentation xml:lang="en">Define extra composite input files for test input.</xs:documentation>
+      <xs:documentation xml:lang="en">Define extra composite input files for test
+input. The specified ``ftype`` on the parent ``param`` should specify a composite
+datatype with defined static composite files. The order of the defined composite
+files on the datatype must match the order specified with these elements and All
+non-optional composite inputs must be specified as part of the ``param``.
+</xs:documentation>
    </xs:annotation>
    <xs:attribute name="value" type="xs:string" use="required">
      <xs:annotation>
        <xs:documentation xml:lang="en">Path relative to test-data of composite file.</xs:documentation>
      </xs:annotation>
    </xs:attribute>
-    <xs:attribute name="ftype" type="xs:string">
-      <xs:annotation>
-        <xs:documentation xml:lang="en">Optional datatype of composite file for test input.</xs:documentation>
-      </xs:annotation>
-    </xs:attribute>
  </xs:complexType>
  <xs:complexType name="TestCollection">
    <xs:annotation>
@@ -760,7 +760,7 @@ class Params(object):
        self.__dict__.update(values)


-def rst_to_html(s):
+def rst_to_html(s, error=False):
    """Convert a blob of reStructuredText to HTML"""
    log = logging.getLogger("docutils")

@@ -770,6 +770,8 @@ def rst_to_html(s):
    class FakeStream(object):
        def write(self, str):
            if len(str) > 0 and not str.isspace():
+                if error:
+                    raise Exception(str)
                log.warning(str)

    settings_overrides = {
@@ -1,6 +1,6 @@
-import bz2
 import gzip
 import re
+import sys
 import zipfile

 from six import StringIO
@@ -8,6 +8,11 @@ from six import StringIO
 from galaxy import util
 from galaxy.util.image_util import image_type

+if sys.version_info < (3, 3):
+    import bz2file as bz2
+else:
+    import bz2
+
 HTML_CHECK_LINES = 100


@@ -1,5 +1,5 @@
-import bz2
 import gzip
+import sys
 import zipfile

 from .checkers import (
@@ -7,6 +7,11 @@ from .checkers import (
    is_gzip
 )

+if sys.version_info < (3, 3):
+    import bz2file as bz2
+else:
+    import bz2
+

 def get_fileobj(filename, mode="r", gzip_only=False, bz2_only=False, zip_only=False):
    """
@@ -1205,10 +1205,8 @@ class UsesStoredWorkflowMixin(SharableItemSecurityMixin, UsesAnnotations):
    def get_stored_workflow(self, trans, id, check_ownership=True, check_accessible=False):
        """ Get a StoredWorkflow from the database by id, verifying ownership. """
        # Load workflow from database
-        try:
-            workflow = trans.sa_session.query(trans.model.StoredWorkflow).get(trans.security.decode_id(id))
-        except TypeError:
-            workflow = None
+        workflow_contents_manager = workflows.WorkflowsManager(self.app)
+        workflow = workflow_contents_manager.get_stored_workflow(trans=trans, workflow_id=id)

        if not workflow:
            error("Workflow not found")
@@ -1,6 +1,4 @@
-"""
-API operations on the library datasets.
-"""
+"""API operations on the library datasets."""
 import glob
 import logging
 import os
@@ -16,7 +14,8 @@ from galaxy import exceptions
 from galaxy import util
 from galaxy import web
 from galaxy.exceptions import ObjectNotFound
-from galaxy.managers import folders, roles, tags
+from galaxy.managers import base as managers_base
+from galaxy.managers import folders, library_datasets, roles
 from galaxy.tools.actions import upload_common
 from galaxy.tools.parameters import populate_state
 from galaxy.util.streamball import StreamBall
@@ -31,81 +30,47 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):

    def __init__(self, app):
        super(LibraryDatasetsController, self).__init__(app)
+        self.app = app
        self.folder_manager = folders.FolderManager()
        self.role_manager = roles.RoleManager(app)
+        self.ld_manager = library_datasets.LibraryDatasetsManager(app)

    @expose_api_anonymous
    def show(self, trans, id, **kwd):
        """
-        show( self, trans, id, **kwd )
-        * GET /api/libraries/datasets/{encoded_dataset_id}
-            Displays information about the dataset identified by the encoded ID.
+        Show the details of a library dataset.

-        :param  id:      the encoded id of the dataset to query
+        * GET /api/libraries/datasets/{encoded_dataset_id}
+
+        :param  id:      the encoded id of the library dataset to query
        :type   id:      an encoded id string

-        :returns:   detailed dataset information from base controller
+        :returns:   detailed library dataset information
        :rtype:     dictionary
-
-        .. seealso:: :attr:`galaxy.web.base.controller.UsesLibraryMixinItems.get_library_dataset`
        """
-        try:
-            library_dataset = self.get_library_dataset(trans, id=id, check_ownership=False, check_accessible=True)
-        except Exception:
-            raise exceptions.ObjectNotFound('Requested library_dataset was not found.')
-
-        current_user_roles = trans.get_current_user_roles()
-
-        tag_manager = tags.GalaxyTagManager(trans.sa_session)
-
-        # Build the full path for breadcrumb purposes.
-        full_path = self._build_path(trans, library_dataset.folder)
-        dataset_item = (trans.security.encode_id(library_dataset.id), library_dataset.name)
-        full_path.insert(0, dataset_item)
-        full_path = full_path[::-1]
-
-        # Find expired versions of the library dataset
-        expired_ldda_versions = []
-        for expired_ldda in library_dataset.expired_datasets:
-            expired_ldda_versions.append((trans.security.encode_id(expired_ldda.id), expired_ldda.name))
-
-        rval = trans.security.encode_all_ids(library_dataset.to_dict())
-        if len(expired_ldda_versions) > 0:
-            rval['has_versions'] = True
-            rval['expired_versions'] = expired_ldda_versions
-        rval['deleted'] = library_dataset.deleted
-        rval['folder_id'] = 'F' + rval['folder_id']
-        rval['full_path'] = full_path
-        rval['file_size'] = util.nice_size(int(library_dataset.library_dataset_dataset_association.get_size()))
-        rval['date_uploaded'] = library_dataset.library_dataset_dataset_association.create_time.strftime("%Y-%m-%d %I:%M %p")
-        rval['can_user_modify'] = trans.app.security_agent.can_modify_library_item(current_user_roles, library_dataset) or trans.user_is_admin()
-        rval['is_unrestricted'] = trans.app.security_agent.dataset_is_public(library_dataset.library_dataset_dataset_association.dataset)
-        rval['tags'] = tag_manager.get_tags_str(library_dataset.library_dataset_dataset_association.tags)
-
-        #  Manage dataset permission is always attached to the dataset itself, not the the ld or ldda to maintain consistency
-        rval['can_user_manage'] = trans.app.security_agent.can_manage_dataset(current_user_roles, library_dataset.library_dataset_dataset_association.dataset) or trans.user_is_admin()
-        return rval
+        ld = self.ld_manager.get(trans, managers_base.decode_id(self.app, id))
+        serialized = self.ld_manager.serialize(trans, ld)
+        return serialized

    @expose_api_anonymous
    def show_version(self, trans, encoded_dataset_id, encoded_ldda_id, **kwd):
        """
-        show_version( self, trans, encoded_dataset_id, encoded_ldda_id, **kwd ):
-        * GET /api/libraries/datasets/{encoded_dataset_id}/versions/{encoded_ldda_id}
-            Displays information about specific version of the library_dataset (i.e. ldda).
+        Display a specific version of a library dataset (i.e. ldda).

-        :param  encoded_dataset_id:      the encoded id of the dataset to query
+        * GET /api/libraries/datasets/{encoded_dataset_id}/versions/{encoded_ldda_id}
+
+        :param  encoded_dataset_id:      the encoded id of the related library dataset
        :type   encoded_dataset_id:      an encoded id string

        :param  encoded_ldda_id:      the encoded id of the ldda to query
        :type   encoded_ldda_id:      an encoded id string

-        :rtype:     dictionary
        :returns:   dict of ldda's details
+        :rtype:     dictionary
+
+        :raises: ObjectNotFound
        """
-        try:
-            library_dataset = self.get_library_dataset(trans, id=encoded_dataset_id, check_ownership=False, check_accessible=True)
-        except Exception:
-            raise exceptions.ObjectNotFound('Requested library_dataset was not found.')
+        library_dataset = self.ld_manager.get(trans, managers_base.decode_id(self.app, encoded_dataset_id))

        try:
            ldda = self.get_library_dataset_dataset_association(trans, id=encoded_ldda_id, check_ownership=False, check_accessible=False)
@@ -121,10 +86,9 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):
    @expose_api
    def show_roles(self, trans, encoded_dataset_id, **kwd):
        """
-        show_roles( self, trans, id, **kwd ):
+        Display information about current or available roles for a given dataset permission.
+
        * GET /api/libraries/datasets/{encoded_dataset_id}/permissions
-            Displays information about current or available roles
-            for a given dataset permission.

        :param  encoded_dataset_id:      the encoded id of the dataset to query
        :type   encoded_dataset_id:      an encoded id string
@@ -132,16 +96,14 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):
        :param  scope:      either 'current' or 'available'
        :type   scope:      string

+        :returns:   either dict of current roles for all permission types
+                    or dict of available roles to choose from (is the same for any permission type)
        :rtype:     dictionary
-        :returns:   either dict of current roles for all permission types or
-                           dict of available roles to choose from (is the same for any permission type)
-        """

+        :raises: InsufficientPermissionsException
+        """
        current_user_roles = trans.get_current_user_roles()
-        try:
-            library_dataset = self.get_library_dataset(trans, id=encoded_dataset_id, check_ownership=False, check_accessible=False)
-        except Exception as e:
-            raise exceptions.ObjectNotFound('Requested dataset was not found.' + str(e))
+        library_dataset = self.ld_manager.get(trans, managers_base.decode_id(self.app, encoded_dataset_id))
        dataset = library_dataset.library_dataset_dataset_association.dataset

        # User has to have manage permissions permission in order to see the roles.
@@ -203,11 +165,39 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):

        return dict(access_dataset_roles=access_dataset_role_list, modify_item_roles=modify_item_role_list, manage_dataset_roles=manage_dataset_role_list)

+    @expose_api
+    def update(self, trans, encoded_dataset_id, payload=None, **kwd):
+        """Update the given library dataset (the latest linked ldda).
+
+        * PATCH /api/libraries/datasets/{encoded_dataset_id}
+
+        :param  encoded_dataset_id: the encoded id of the library dataset to update
+        :type   encoded_dataset_id: an encoded id string
+        :param  payload:            dictionary structure containing::
+            :param name:            new ld's name, must be longer than 0
+            :type  name:            str
+            :param misc_info:       new ld's misc info
+            :type  misc_info:       str
+            :param file_ext:        new ld's extension, must exist in the Galaxy registry
+            :type  file_ext:        str
+            :param genome_build:    new ld's genome build
+            :type  genome_build:    str
+        :type   payload: dict
+
+        :returns:   detailed library dataset information
+        :rtype:     dictionary
+        """
+        library_dataset = self.ld_manager.get(trans, managers_base.decode_id(self.app, encoded_dataset_id))
+        updated = self.ld_manager.update(trans, library_dataset, payload)
+        serialized = self.ld_manager.serialize(trans, updated)
+        return serialized
+
    @expose_api
    def update_permissions(self, trans, encoded_dataset_id, payload=None, **kwd):
        """
+        Set permissions of the given library dataset to the given role ids.
+
        *POST /api/libraries/datasets/{encoded_dataset_id}/permissions
-            Set permissions of the given dataset to the given role ids.

        :param  encoded_dataset_id:      the encoded id of the dataset to update permissions of
        :type   encoded_dataset_id:      an encoded id string
@@ -222,6 +212,7 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):
            :param  modify_ids[]:      list of Role.id defining roles that should have modify permission on the library dataset item
            :type   modify_ids[]:      string or list
        :type:      dictionary
+
        :returns:   dict of current roles for all available permission types
        :rtype:     dictionary

@@ -230,10 +221,8 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):
        """
        if payload:
            kwd.update(payload)
-        try:
-            library_dataset = self.get_library_dataset(trans, id=encoded_dataset_id, check_ownership=False, check_accessible=False)
-        except Exception as e:
-            raise exceptions.ObjectNotFound('Requested dataset was not found.' + str(e))
+        library_dataset = self.ld_manager.get(trans, managers_base.decode_id(self.app, encoded_dataset_id))
+        # Some permissions are attached directly to the underlying dataset.
        dataset = library_dataset.library_dataset_dataset_association.dataset
        current_user_roles = trans.get_current_user_roles()
        can_manage = trans.app.security_agent.can_manage_dataset(current_user_roles, dataset) or trans.user_is_admin()
@@ -266,7 +255,7 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):
                trans.app.security_agent.make_dataset_public(dataset)
            else:
                for role_id in new_access_roles_ids:
-                    role = self.role_manager.get(trans, self.__decode_id(trans, role_id, 'role'))
+                    role = self.role_manager.get(trans, managers_base.decode_id(self.app, role_id))
                    #  Check whether role is in the set of allowed roles
                    valid_roles, total_roles = trans.app.security_agent.get_valid_roles(trans, dataset)
                    if role in valid_roles:
@@ -288,7 +277,7 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):
            active_access_roles = dataset.get_access_roles(trans)

            for role_id in new_manage_roles_ids:
-                role = self.role_manager.get(trans, self.__decode_id(trans, role_id, 'role'))
+                role = self.role_manager.get(trans, managers_base.decode_id(self.app, role_id))
                #  Check whether role is in the set of access roles
                if role in active_access_roles:
                    valid_manage_roles.append(role)
@@ -310,7 +299,7 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):
            active_access_roles = dataset.get_access_roles(trans)

            for role_id in new_modify_roles_ids:
-                role = self.role_manager.get(trans, self.__decode_id(trans, role_id, 'role'))
+                role = self.role_manager.get(trans, managers_base.decode_id(self.app, role_id))
                #  Check whether role is in the set of access roles
                if role in active_access_roles:
                    valid_modify_roles.append(role)
@@ -332,62 +321,59 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):
    @expose_api
    def delete(self, trans, encoded_dataset_id, **kwd):
        """
-        delete( self, trans, encoded_dataset_id, **kwd ):
+        Mark the dataset deleted or undeleted.
+
        * DELETE /api/libraries/datasets/{encoded_dataset_id}
-            Marks the dataset deleted or undeleted based on the value
-            of the undelete flag.
-            If the flag is not present it is considered False and the
-            item is marked deleted.

        :param  encoded_dataset_id:      the encoded id of the dataset to change
        :type   encoded_dataset_id:      an encoded id string
+        :param  undelete:                flag whether to undeleted instead of deleting
+        :type   undelete:                bool

        :returns:   dict containing information about the dataset
        :rtype:     dictionary
        """
        undelete = util.string_as_bool(kwd.get('undelete', False))
-        try:
-            dataset = self.get_library_dataset(trans, id=encoded_dataset_id, check_ownership=False, check_accessible=False)
-        except Exception as e:
-            raise exceptions.ObjectNotFound('Requested dataset was not found.' + str(e))
+        library_dataset = self.ld_manager.get(trans, managers_base.decode_id(self.app, encoded_dataset_id))
        current_user_roles = trans.get_current_user_roles()
-        allowed = trans.app.security_agent.can_modify_library_item(current_user_roles, dataset)
+        allowed = trans.app.security_agent.can_modify_library_item(current_user_roles, library_dataset)
        if (not allowed) and (not trans.user_is_admin()):
            raise exceptions.InsufficientPermissionsException('You do not have proper permissions to delete this dataset.')

        if undelete:
-            dataset.deleted = False
+            library_dataset.deleted = False
        else:
-            dataset.deleted = True
+            library_dataset.deleted = True

-        trans.sa_session.add(dataset)
+        trans.sa_session.add(library_dataset)
        trans.sa_session.flush()

-        rval = trans.security.encode_all_ids(dataset.to_dict())
-        nice_size = util.nice_size(int(dataset.library_dataset_dataset_association.get_size()))
+        rval = trans.security.encode_all_ids(library_dataset.to_dict())
+        nice_size = util.nice_size(int(library_dataset.library_dataset_dataset_association.get_size()))
        rval['file_size'] = nice_size
-        rval['update_time'] = dataset.update_time.strftime("%Y-%m-%d %I:%M %p")
-        rval['deleted'] = dataset.deleted
+        rval['update_time'] = library_dataset.update_time.strftime("%Y-%m-%d %I:%M %p")
+        rval['deleted'] = library_dataset.deleted
        rval['folder_id'] = 'F' + rval['folder_id']
        return rval

    @expose_api
    def load(self, trans, payload=None, **kwd):
        """
+        Load dataset(s) from the given source into the library.
+
        * POST /api/libraries/datasets
-        Load dataset from the given source into the library.
-        Source can be:
-            user directory - root folder specified in galaxy.ini as "$user_library_import_dir"
-                example path: path/to/galaxy/$user_library_import_dir/user@example.com/{user can browse everything here}
-                the folder with the user login has to be created beforehand
-            (admin)import directory - root folder specified in galaxy ini as "$library_import_dir"
-                example path: path/to/galaxy/$library_import_dir/{admin can browse everything here}
-            (admin)any absolute or relative path - option allowed with "allow_library_path_paste" in galaxy.ini

        :param   payload: dictionary structure containing:
            :param  encoded_folder_id:      the encoded id of the folder to import dataset(s) to
            :type   encoded_folder_id:      an encoded id string
            :param  source:                 source the datasets should be loaded from
+                    Source can be:
+                        user directory - root folder specified in galaxy.ini as "$user_library_import_dir"
+                            example path: path/to/galaxy/$user_library_import_dir/user@example.com/{user can browse everything here}
+                            the folder with the user login has to be created beforehand
+                        (admin)import directory - root folder specified in galaxy ini as "$library_import_dir"
+                            example path: path/to/galaxy/$library_import_dir/{admin can browse everything here}
+                        (admin)any absolute or relative path - option allowed with "allow_library_path_paste" in galaxy.ini
            :type   source:                 str
            :param  link_data:              flag whether to link the dataset to data or copy it to Galaxy, defaults to copy
                                            while linking is set to True all symlinks will be resolved _once_
@@ -402,8 +388,10 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):
            :param  tag_using_filenames:    flag whether to generate dataset tags from filenames
            :type   tag_using_filenames:    bool
        :type   dictionary
+
        :returns:   dict containing information about the created upload job
        :rtype:     dictionary
+
        :raises: RequestParameterMissingException, AdminRequiredException, ConfigDoesNotAllowException, RequestParameterInvalidException
                    InsufficientPermissionsException, ObjectNotFound
        """
@@ -510,10 +498,10 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):
    #  TODO convert to expose_api
    def download(self, trans, format, **kwd):
        """
-        download( self, trans, format, **kwd )
+        Download requested datasets (identified by encoded IDs) in requested format.
+
        * GET /api/libraries/datasets/download/{format}
        * POST /api/libraries/datasets/download/{format}
-            Downloads requested datasets (identified by encoded IDs) in requested format.

        example: ``GET localhost:8080/api/libraries/datasets/download/tbz?ld_ids%255B%255D=a0d84b45643a2678&ld_ids%255B%255D=fe38c84dcd46c828``

@@ -526,8 +514,8 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):
        :param  folder_ids[]:      an array of encoded folder ids
        :type   folder_ids[]:      an array

-        :rtype:   file
        :returns: either archive with the requested datasets packed inside or a single uncompressed dataset
+        :rtype:   file

        :raises: MessageException, ItemDeletionException, ItemAccessibilityException, HTTPBadRequest, OSError, IOError, ObjectNotFound
        """
@@ -636,7 +624,8 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):
                zpath = os.path.split(path)[-1]  # comes as base_name/fname
                outfname, zpathext = os.path.splitext(zpath)

-                if is_composite:  # need to add all the components from the extra_files_path to the zip
+                if is_composite:
+                    # need to add all the components from the extra_files_path to the zip
                    if zpathext == '':
                        zpath = '%s.html' % zpath  # fake the real nature of the html file
                    try:
@@ -671,10 +660,9 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):
                            log.exception("Requested dataset %s does not exist on the host.", fpath)
                            raise exceptions.ObjectNotFound("Requested dataset not found.")
                        except Exception as e:
-                            log.exception("Unable to add %s to temporary library download archive %s", fname, outfname)
+                            log.exception("Unable to add %s to temporary library download archive %s" % (fname, outfname))
                            raise exceptions.InternalServerError("Unable to add dataset to temporary library download archive . " + str(e))
-
-                else:  # simple case
+                else:
                    try:
                        if format == 'zip':
                            archive.add(ldda.dataset.file_name, path)
@@ -724,39 +712,3 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin):
                    raise exceptions.InternalServerError("This dataset contains no content.")
        else:
            raise exceptions.RequestParameterInvalidException("Wrong format parameter specified")
-
-    def _build_path(self, trans, folder):
-        """
-        Search the path upwards recursively and load the whole route of
-        names and ids for breadcrumb building purposes.
-
-        :param folder: current folder for navigating up
-        :param type:   Galaxy LibraryFolder
-
-        :returns:   list consisting of full path to the library
-        :type:      list
-        """
-        path_to_root = []
-        # We are almost in root
-        if folder.parent_id is None:
-            path_to_root.append(('F' + trans.security.encode_id(folder.id), folder.name))
-        else:
-            # We add the current folder and traverse up one folder.
-            path_to_root.append(('F' + trans.security.encode_id(folder.id), folder.name))
-            upper_folder = trans.sa_session.query(trans.app.model.LibraryFolder).get(folder.parent_id)
-            path_to_root.extend(self._build_path(trans, upper_folder))
-        return path_to_root
-
-    def __decode_id(self, trans, encoded_id, object_name=None):
-        """
-        Try to decode the id.
-
-        :param  object_name:      Name of the object the id belongs to. (optional)
-        :type   object_name:      str
-        """
-        try:
-            return trans.security.decode_id(encoded_id)
-        except TypeError:
-            raise exceptions.MalformedId('Malformed %s id specified, unable to decode.' % object_name if object_name is not None else '')
-        except ValueError:
-            raise exceptions.MalformedId('Wrong %s id specified, unable to decode.' % object_name if object_name is not None else '')
@@ -358,11 +358,13 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
        stored_workflow = self.__get_stored_accessible_workflow(trans, workflow_id)

        style = kwd.get("style", "export")
+        download_format = kwd.get('format')
        ret_dict = self.workflow_contents_manager.workflow_to_dict(trans, stored_workflow, style=style)
-        if not ret_dict:
-            # This workflow has a tool that's missing from the distribution
-            message = "Workflow cannot be exported due to missing tools."
-            raise exceptions.MessageException(message)
+        if download_format == 'json-download':
+            sname = stored_workflow.name
+            sname = ''.join(c in util.FILENAME_VALID_CHARS and c or '_' for c in sname)[0:150]
+            trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy-Workflow-%s.ga"' % (sname)
+            trans.response.set_content_type('application/galaxy-archive')
        return ret_dict

    @expose_api
@@ -436,15 +438,18 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
        :returns:   serialized version of the workflow
        """
        stored_workflow = self.__get_stored_workflow(trans, id)
-        if 'workflow' in payload:
-            stored_workflow.name = sanitize_html(payload['name']) if ('name' in payload) else stored_workflow.name
+        workflow_dict = payload.get('workflow')
+        if workflow_dict:
+            new_workflow_name = payload.get('name') or workflow_dict.get('name')
+            if new_workflow_name:
+                stored_workflow.name = sanitize_html(new_workflow_name)

            if 'annotation' in payload:
                newAnnotation = sanitize_html(payload['annotation'])
                self.add_item_annotation(trans.sa_session, trans.get_user(), stored_workflow, newAnnotation)

-            if 'menu_entry' in payload:
-                if payload['menu_entry']:
+            if 'menu_entry' in payload or 'show_in_tool_panel' in workflow_dict:
+                if payload.get('menu_entry') or workflow_dict.get('show_in_tool_panel'):
                    menuEntry = model.StoredWorkflowMenuEntry()
                    menuEntry.stored_workflow = stored_workflow
                    trans.get_user().stored_workflow_menu_entries.append(menuEntry)
@@ -453,15 +458,21 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
                    entries = {x.stored_workflow_id: x for x in trans.get_user().stored_workflow_menu_entries}
                    if (trans.security.decode_id(id) in entries):
                        trans.get_user().stored_workflow_menu_entries.remove(entries[trans.security.decode_id(id)])
+            # set tags
+            trans.app.tag_handler.set_tags_from_list(user=trans.user, item=stored_workflow, new_tags_list=workflow_dict.get('tags', []))

-            try:
-                workflow, errors = self.workflow_contents_manager.update_workflow_from_dict(
-                    trans,
-                    stored_workflow,
-                    payload['workflow'],
-                )
-            except workflows.MissingToolsException:
-                raise exceptions.MessageException("This workflow contains missing tools. It cannot be saved until they have been removed from the workflow or installed.")
+            if 'steps' in workflow_dict:
+                try:
+                    workflow, errors = self.workflow_contents_manager.update_workflow_from_dict(
+                        trans,
+                        stored_workflow,
+                        workflow_dict,
+                    )
+                except workflows.MissingToolsException:
+                    raise exceptions.MessageException("This workflow contains missing tools. It cannot be saved until they have been removed from the workflow or installed.")
+            else:
+                # We only adjusted tags and menu entry
+                return payload
        else:
            message = "Updating workflow requires dictionary containing 'workflow' attribute with new JSON description."
            raise exceptions.RequestParameterInvalidException(message)
@@ -537,7 +548,8 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn

        item = workflow.to_dict(value_mapper={'id': trans.security.encode_id})
        item['url'] = url_for('workflow', id=encoded_id)
-
+        item['owner'] = workflow.user.username
+        item['number_of_steps'] = len(workflow.latest_workflow.steps)
        rval.append(item)

        #
@@ -652,43 +652,49 @@ def populate_api_routes(webapp, app):

    webapp.mapper.connect('show_ld_item',
                          '/api/libraries/datasets/{id}',
-                          controller='lda_datasets',
+                          controller='library_datasets',
                          action='show',
                          conditions=dict(method=["GET"]))

    webapp.mapper.connect('load_ld',
                          '/api/libraries/datasets/',
-                          controller='lda_datasets',
+                          controller='library_datasets',
                          action='load',
                          conditions=dict(method=["POST"]))

    webapp.mapper.connect('show_version_of_ld_item',
                          '/api/libraries/datasets/{encoded_dataset_id}/versions/{encoded_ldda_id}',
-                          controller='lda_datasets',
+                          controller='library_datasets',
                          action='show_version',
                          conditions=dict(method=["GET"]))

-    webapp.mapper.connect('show_legitimate_lda_roles',
+    webapp.mapper.connect('update_ld',
+                          '/api/libraries/datasets/{encoded_dataset_id}',
+                          controller='library_datasets',
+                          action='update',
+                          conditions=dict(method=["PATCH"]))
+
+    webapp.mapper.connect('show_legitimate_ld_roles',
                          '/api/libraries/datasets/{encoded_dataset_id}/permissions',
-                          controller='lda_datasets',
+                          controller='library_datasets',
                          action='show_roles',
                          conditions=dict(method=["GET"]))

-    webapp.mapper.connect('update_lda_permissions',
+    webapp.mapper.connect('update_ld_permissions',
                          '/api/libraries/datasets/{encoded_dataset_id}/permissions',
-                          controller='lda_datasets',
+                          controller='library_datasets',
                          action='update_permissions',
                          conditions=dict(method=["POST"]))

-    webapp.mapper.connect('delete_lda_item',
+    webapp.mapper.connect('delete_ld_item',
                          '/api/libraries/datasets/{encoded_dataset_id}',
-                          controller='lda_datasets',
+                          controller='library_datasets',
                          action='delete',
                          conditions=dict(method=["DELETE"]))

-    webapp.mapper.connect('download_lda_items',
+    webapp.mapper.connect('download_ld_items',
                          '/api/libraries/datasets/download/{format}',
-                          controller='lda_datasets',
+                          controller='library_datasets',
                          action='download',
                          conditions=dict(method=["POST", "GET"]))

@@ -1255,7 +1255,6 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP
                    'status' : 'info',
                    'inputs' : [build_select_input('in_roles', 'Roles', all_roles, in_roles),
                                build_select_input('in_users', 'Users', all_users, in_users)]}
-            return {'message' : 'Not showing associated datasets, there are too many.', 'info' : 'info'}
        else:
            in_users = [trans.sa_session.query(trans.app.model.User).get(trans.security.decode_id(x)) for x in util.listify(payload.get('in_users'))]
            in_roles = [trans.sa_session.query(trans.app.model.Role).get(trans.security.decode_id(x)) for x in util.listify(payload.get('in_roles'))]
@@ -1043,11 +1043,11 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt
        # histories looks like: { userA: [ historyX, historyY ], userB: [ historyY ] }
        histories = histories or {}
        msg = ""
+        shared_histories = []
        if not histories:
            send_to_err += "No users have been specified or no histories can be sent without changing permissions or associating a sharing role.  "
        else:
            for send_to_user, send_to_user_histories in histories.items():
-                shared_histories = []
                for history in send_to_user_histories:
                    share = trans.app.model.HistoryUserShareAssociation()
                    share.history = history
@@ -1096,7 +1096,7 @@ class LibraryCommon(BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMeta
        if response_code == 200:
            precreated_datasets = upload_common.get_precreated_datasets(trans, tool_params, trans.app.model.LibraryDatasetDatasetAssociation, controller=cntrller)
            if upload_option == 'upload_file':
-                tool_params = upload_common.persist_uploads(tool_params)
+                tool_params = upload_common.persist_uploads(tool_params, trans)
                uploaded_datasets = upload_common.get_uploaded_datasets(trans, cntrller, tool_params, precreated_datasets, dataset_upload_inputs, library_bunch=library_bunch)
            elif upload_option == 'upload_directory':
                uploaded_datasets, response_code, message = self.get_server_dir_uploaded_datasets(trans, cntrller, kwd, full_dir, import_dir_desc, library_bunch, response_code, message)
@@ -7,6 +7,7 @@ import sgmllib
 import urllib2

 from sqlalchemy import and_
+from sqlalchemy.orm import joinedload
 from sqlalchemy.sql import expression
 from markupsafe import escape

@@ -18,7 +19,8 @@ from galaxy import web
 from galaxy.managers import workflows
 from galaxy.model.item_attrs import UsesItemRatings
 from galaxy.model.mapping import desc
-from galaxy.util import unicodify, FILENAME_VALID_CHARS
+from galaxy.tools.parameters.basic import workflow_building_modes
+from galaxy.util import unicodify
 from galaxy.util.sanitize_html import sanitize_html
 from galaxy.web import error, url_for
 from galaxy.web.base.controller import BaseUIController, SharableMixin, UsesStoredWorkflowMixin
@@ -620,9 +622,12 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
        if not id:
            error("Invalid workflow id")
        stored = self.get_stored_workflow(trans, id)
+        # The following query loads all user-owned workflows,
+        # So that they can be copied or inserted in the workflow editor.
        workflows = trans.sa_session.query(model.StoredWorkflow) \
            .filter_by(user=trans.user, deleted=False) \
            .order_by(desc(model.StoredWorkflow.table.c.update_time)) \
+            .options(joinedload('latest_workflow').joinedload('steps')) \
            .all()
        return trans.fill_template("workflow/editor.mako", workflows=workflows, stored=stored, annotation=self.get_item_annotation_str(trans.sa_session, trans.user, stored))

@@ -633,7 +638,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
        encode it as a json string that can be read by the workflow editor
        web interface.
        """
-        trans.workflow_building_mode = True
+        trans.workflow_building_mode = workflow_building_modes.ENABLED
        stored = self.get_stored_workflow(trans, id, check_ownership=True, check_accessible=False)
        workflow_contents_manager = workflows.WorkflowContentsManager(trans.app)
        return workflow_contents_manager.workflow_to_dict(trans, stored, style="editor")
@@ -644,7 +649,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
        """
        Exports a workflow to myExperiment website.
        """
-        trans.workflow_building_mode = True
+        trans.workflow_building_mode = workflow_building_modes.ENABLED
        stored = self.get_stored_workflow(trans, id, check_ownership=False, check_accessible=True)

        # Convert workflow to dict.
@@ -708,32 +713,6 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
        stored = self.get_stored_workflow(trans, id, check_ownership=False, check_accessible=True)
        return self._workflow_to_dict(trans, stored)

-    @web.json_pretty
-    def export_to_file(self, trans, id):
-        """
-        Get the latest Workflow for the StoredWorkflow identified by `id` and
-        encode it as a json string that can be imported back into Galaxy
-
-        This has slightly different information than the above. In particular,
-        it does not attempt to decode forms and build UIs, it just stores
-        the raw state.
-        """
-
-        # Get workflow.
-        stored = self.get_stored_workflow(trans, id, check_ownership=False, check_accessible=True)
-
-        # Stream workflow to file.
-        stored_dict = self._workflow_to_dict(trans, stored)
-        if not stored_dict:
-            # This workflow has a tool that's missing from the distribution
-            trans.response.status = 400
-            return "Workflow cannot be exported due to missing tools."
-        sname = stored.name
-        sname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in sname)[0:150]
-        trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy-Workflow-%s.ga"' % (sname)
-        trans.response.set_content_type('application/galaxy-archive')
-        return stored_dict
-
    @web.expose
    @web.json
    def upload_import_workflow(self, trans, cntrller='workflow', **kwd):
@@ -22,6 +22,7 @@ class WorkRequestContext(ProvidesAppContext, ProvidesUserContext, ProvidesHistor
        self.app = app
        self.security = app.security
        self.__user = user
+        self.__user_current_roles = None
        self.__history = history
        self.api_inherit_admin = False
        self.workflow_building_mode = workflow_building_mode
@@ -40,6 +41,11 @@ class WorkRequestContext(ProvidesAppContext, ProvidesUserContext, ProvidesHistor
        """Return the current user if logged in or None."""
        return self.__user

+    def get_current_user_roles(self):
+        if self.__user_current_roles is None:
+            self.__user_current_roles = super(WorkRequestContext, self).get_current_user_roles()
+        return self.__user_current_roles
+
    def set_user(self, user):
        """Set the current user."""
        raise NotImplementedError("Cannot change users from a work request context.")
@@ -608,7 +608,7 @@ class ToolModule(WorkflowModule):
                    if not old_tool_shed_url:  # a tool from a different tool_shed has been found, but the original tool shed has been deactivated
                        old_tool_shed_url = "http://" + old_tool_shed  # let's just assume it's either http, or a http is forwarded to https.
                    old_url = old_tool_shed_url + "/view/%s/%s/" % (module.tool.repository_owner, module.tool.repository_name)
-                    new_url = module.tool.tool_shed_repository.get_sharable_url(module.tool.app) + '/%s/' % module.tool.tool_shed_repository.changeset_revision
+                    new_url = module.tool.sharable_url + '/%s/' % module.tool.changeset_revision
                    new_tool_shed_url = new_url.split("/view")[0]
                    message += "The tool \'%s\', version %s by the owner %s installed from <a href=\"%s\" target=\"_blank\">%s</a> is not available. " % (module.tool.name, tool_version, module.tool.repository_owner, old_url, old_tool_shed_url)
                    message += "A derivation of this tool installed from <a href=\"%s\" target=\"_blank\">%s</a> will be used instead. " % (new_url, new_tool_shed_url)
@@ -617,6 +617,8 @@ class ToolModule(WorkflowModule):
            if message:
                log.debug(message)
                module.version_changes.append(message)
+        else:
+            log.warning("The tool '%s' is missing. Cannot build workflow module." % tool_id)
        return module

    # ---- Saving in various forms ------------------------------------------
@@ -686,6 +688,7 @@ class ToolModule(WorkflowModule):
                if tool_output.collection:
                    extra_kwds["collection"] = True
                    extra_kwds["collection_type"] = tool_output.structure.collection_type
+                    extra_kwds["collection_type_source"] = tool_output.structure.collection_type_source
                    formats = ['input']  # TODO: fix
                elif tool_output.format_source is not None:
                    formats = ['input']  # default to special name "input" which remove restrictions on connections
@@ -156,6 +156,10 @@ class WorkflowInvoker(object):
            # invocations.
            return self.progress.outputs

+        if workflow_invocation.history.deleted:
+            log.info("Cancelled workflow evaluation due to deleted history")
+            raise modules.CancelWorkflowEvaluation()
+
        remaining_steps = self.progress.remaining_steps()
        delayed_steps = False
        for step in remaining_steps:
@@ -42,6 +42,14 @@ class InstalledRepositoryManager(object):
        self.tool_configs = self.app.config.tool_configs
        if self.app.config.migrated_tools_config not in self.tool_configs:
            self.tool_configs.append(self.app.config.migrated_tools_config)
+
+        self.tool_trees = []
+        for tool_config in self.tool_configs:
+            tree, error_message = xml_util.parse_xml(tool_config)
+            if error_message:
+                log.error(error_message)
+            self.tool_trees.append(tree)
+
        self.installed_repository_dicts = []
        # Keep an in-memory dictionary whose keys are tuples defining tool_shed_repository objects (whose status is 'Installed')
        # and whose values are a list of tuples defining tool_shed_repository objects (whose status can be anything) required by
@@ -572,8 +580,7 @@ class InstalledRepositoryManager(object):
                str(repository.installed_changeset_revision))

    def get_repository_install_dir(self, tool_shed_repository):
-        for tool_config in self.tool_configs:
-            tree, error_message = xml_util.parse_xml(tool_config)
+        for tree in self.tool_trees:
            if tree is None:
                return None
            root = tree.getroot()
@@ -264,7 +264,6 @@ class Repository(RecipeTag, SyncDatabase):
            message = "Unable to retrieve required tool_dependencies.xml file from the Tool Shed for revision "
            message += "%s of installed repository %s owned by %s." % (str(changeset_revision), str(name), str(owner))
            raise Exception(message)
-            return None

    def create_tool_dependency_with_initialized_env_sh_file(self, dependent_install_dir, tool_shed_repository,
                                                            required_repository, package_name, package_version,
@@ -116,7 +116,6 @@ class RepositoryGrid(grids.Grid):
                rev_label, rev_date = option_items.split(' ')
                rev_date = '<i><font color="#666666">%s</font></i>' % rev_date
                return '%s %s' % (rev_label, rev_date)
-                return select_field.options[0][0]
            return ''

    class LatestInstallableRevisionColumn(grids.GridColumn):
@@ -1,9 +1,9 @@
-import bz2
 import gzip
 import json
 import logging
 import os
 import shutil
+import sys
 import tempfile
 from collections import namedtuple

@@ -14,6 +14,11 @@ from galaxy.util import checkers, safe_relpath
 from tool_shed.tools import data_table_manager
 from tool_shed.util import basic_util, hg_util, shed_util_common as suc

+if sys.version_info < (3, 3):
+    import bz2file as bz2
+else:
+    import bz2
+
 log = logging.getLogger(__name__)

 UNDESIRABLE_DIRS = ['.hg', '.svn', '.git', '.cvs']
@@ -1,4 +1,5 @@
 import logging
+import os
 import sys
 import tempfile
 import xml.etree.ElementTree
@@ -118,6 +119,9 @@ def indent(elem, level=0):
 def parse_xml(file_name):
    """Returns a parsed xml tree with comments intact."""
    error_message = ''
+    if not os.path.exists(file_name):
+        return None, "File does not exist %s" % str(file_name)
+
    fobj = open(file_name, 'r')
    if using_python_27:
        try:
--- a/Show More
+++ b/Show More